indigo/LinkDesk
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Init Repo

Browse Source
This commit is contained in:
indigo
2026-02-28 03:22:04 +08:00
commit de59b57ee7
883 changed files with 156857 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/services/__init__.py
+29
View File
@@ -0,0 +1,29 @@
"""
Services module for VFX Project Management System.
This module contains business logic services for handling complex operations
like soft deletion and recovery of shots and assets.
"""
from .shot_soft_deletion import ShotSoftDeletionService, DeletionInfo, DeletionResult
from .asset_soft_deletion import AssetSoftDeletionService, AssetDeletionInfo, AssetDeletionResult
from .recovery_service import (
RecoveryService, RecoveryInfo, RecoveryResult, DeletedShot, DeletedAsset,
PermanentDeleteResult, BulkPermanentDeleteResult
)
__all__ = [
'ShotSoftDeletionService',
'DeletionInfo',
'DeletionResult',
'AssetSoftDeletionService',
'AssetDeletionInfo',
'AssetDeletionResult',
'RecoveryService',
'RecoveryInfo',
'RecoveryResult',
'DeletedShot',
'DeletedAsset',
'PermanentDeleteResult',
'BulkPermanentDeleteResult'
]
backend/services/asset_soft_deletion.py
+483
View File
@@ -0,0 +1,483 @@
"""
Asset Soft Deletion Service
This service handles the soft deletion of assets and all related data including:
- Tasks associated with the asset
- Submissions for those tasks
- Production notes for those tasks
- Task attachments for those tasks
- Reviews for those submissions
All operations are performed within database transactions to ensure atomicity.
"""
from datetime import datetime
from typing import List, Dict, Any, Optional
from sqlalchemy.orm import Session
from sqlalchemy.exc import SQLAlchemyError
from sqlalchemy import func
import sys
import os
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from models.asset import Asset
from models.task import Task, Submission, Review, ProductionNote, TaskAttachment
from models.user import User
from models.activity import Activity, ActivityType
from utils.activity import ActivityService
class AssetDeletionInfo:
"""Information about what will be deleted when an asset is soft deleted."""
def __init__(self):
self.asset_id: int = 0
self.asset_name: str = ""
self.asset_category: str = ""
self.project_name: str = ""
# Counts of items that will be marked as deleted
self.task_count: int = 0
self.submission_count: int = 0
self.attachment_count: int = 0
self.note_count: int = 0
self.review_count: int = 0
# File information (preserved, not deleted)
self.total_file_size: int = 0
self.file_count: int = 0
# Affected users
self.affected_users: List[Dict[str, Any]] = []
# Timestamps
self.last_activity_date: Optional[str] = None
self.created_at: str = ""
class AssetDeletionResult:
"""Result of an asset soft deletion operation."""
def __init__(self):
self.success: bool = False
self.asset_id: int = 0
self.asset_name: str = ""
# Database update results
self.marked_deleted_tasks: int = 0
self.marked_deleted_submissions: int = 0
self.marked_deleted_attachments: int = 0
self.marked_deleted_notes: int = 0
self.marked_deleted_reviews: int = 0
# Timing
self.operation_duration: float = 0.0
self.deleted_at: str = ""
self.deleted_by: int = 0
# Errors
self.errors: List[str] = []
self.warnings: List[str] = []
class AssetSoftDeletionService:
"""Service for handling asset soft deletion operations."""
def __init__(self):
self.activity_service = ActivityService()
def get_deletion_info(self, asset_id: int, db: Session) -> Optional[AssetDeletionInfo]:
"""
Get information about what will be deleted when an asset is soft deleted.
Args:
asset_id: ID of the asset to analyze
db: Database session
Returns:
AssetDeletionInfo object with counts and affected users, or None if asset not found
"""
try:
# Get the asset (only if not already deleted)
asset = db.query(Asset).filter(
Asset.id == asset_id,
Asset.deleted_at.is_(None)
).first()
if not asset:
return None
info = AssetDeletionInfo()
info.asset_id = asset.id
info.asset_name = asset.name
info.asset_category = asset.category.value if asset.category else ""
info.project_name = asset.project.name if asset.project else ""
info.created_at = asset.created_at.isoformat() if asset.created_at else ""
# Get all active tasks for this asset
tasks = db.query(Task).filter(
Task.asset_id == asset_id,
Task.deleted_at.is_(None)
).all()
info.task_count = len(tasks)
if not tasks:
return info
task_ids = [task.id for task in tasks]
# Count submissions
submissions = db.query(Submission).filter(
Submission.task_id.in_(task_ids),
Submission.deleted_at.is_(None)
).all()
info.submission_count = len(submissions)
# Count attachments and calculate file sizes
attachments = db.query(TaskAttachment).filter(
TaskAttachment.task_id.in_(task_ids),
TaskAttachment.deleted_at.is_(None)
).all()
info.attachment_count = len(attachments)
info.file_count = len(attachments) + len(submissions)
info.total_file_size = sum(att.file_size for att in attachments if att.file_size)
# Count production notes
notes = db.query(ProductionNote).filter(
ProductionNote.task_id.in_(task_ids),
ProductionNote.deleted_at.is_(None)
).all()
info.note_count = len(notes)
# Count reviews
if submissions:
submission_ids = [sub.id for sub in submissions]
reviews = db.query(Review).filter(
Review.submission_id.in_(submission_ids),
Review.deleted_at.is_(None)
).all()
info.review_count = len(reviews)
# Get affected users
info.affected_users = self._get_affected_users(tasks, submissions, notes, db)
# Get last activity date
info.last_activity_date = self._get_last_activity_date(asset_id, task_ids, db)
return info
except SQLAlchemyError as e:
raise Exception(f"Database error while getting deletion info: {str(e)}")
def soft_delete_asset_cascade(self, asset_id: int, db: Session, current_user: User) -> AssetDeletionResult:
"""
Perform cascading soft deletion of an asset and all related data.
Args:
asset_id: ID of the asset to soft delete
db: Database session
current_user: User performing the deletion
Returns:
AssetDeletionResult with operation details
"""
start_time = datetime.utcnow()
result = AssetDeletionResult()
result.asset_id = asset_id
result.deleted_by = current_user.id
try:
# Get the asset (only if not already deleted)
asset = db.query(Asset).filter(
Asset.id == asset_id,
Asset.deleted_at.is_(None)
).first()
if not asset:
result.errors.append("Asset not found or already deleted")
return result
result.asset_name = asset.name
deleted_at = datetime.utcnow()
result.deleted_at = deleted_at.isoformat()
# Get deletion info for logging
deletion_info = self.get_deletion_info(asset_id, db)
# Mark related data as deleted
self._mark_related_data_deleted(asset_id, db, current_user, deleted_at, result)
# Mark the asset as deleted
asset.deleted_at = deleted_at
asset.deleted_by = current_user.id
db.flush()
# Log the deletion activity
self._log_asset_deletion(asset, current_user, deletion_info, db)
result.success = True
except SQLAlchemyError as e:
result.errors.append(f"Database error during deletion: {str(e)}")
except Exception as e:
result.errors.append(f"Unexpected error during deletion: {str(e)}")
# Calculate operation duration
end_time = datetime.utcnow()
result.operation_duration = (end_time - start_time).total_seconds()
return result
def _mark_related_data_deleted(self, asset_id: int, db: Session, current_user: User,
deleted_at: datetime, result: AssetDeletionResult) -> None:
"""
Mark all data related to an asset as deleted.
Args:
asset_id: ID of the asset
db: Database session
current_user: User performing the deletion
deleted_at: Timestamp for deletion
result: AssetDeletionResult to update with counts
"""
# Get all active tasks for this asset
tasks = db.query(Task).filter(
Task.asset_id == asset_id,
Task.deleted_at.is_(None)
).all()
if not tasks:
return
task_ids = [task.id for task in tasks]
# Mark tasks as deleted
task_update_count = db.query(Task).filter(
Task.asset_id == asset_id,
Task.deleted_at.is_(None)
).update({
Task.deleted_at: deleted_at,
Task.deleted_by: current_user.id
}, synchronize_session=False)
result.marked_deleted_tasks = task_update_count
# Mark submissions as deleted
submission_update_count = db.query(Submission).filter(
Submission.task_id.in_(task_ids),
Submission.deleted_at.is_(None)
).update({
Submission.deleted_at: deleted_at,
Submission.deleted_by: current_user.id
}, synchronize_session=False)
result.marked_deleted_submissions = submission_update_count
# Mark attachments as deleted
attachment_update_count = db.query(TaskAttachment).filter(
TaskAttachment.task_id.in_(task_ids),
TaskAttachment.deleted_at.is_(None)
).update({
TaskAttachment.deleted_at: deleted_at,
TaskAttachment.deleted_by: current_user.id
}, synchronize_session=False)
result.marked_deleted_attachments = attachment_update_count
# Mark production notes as deleted
note_update_count = db.query(ProductionNote).filter(
ProductionNote.task_id.in_(task_ids),
ProductionNote.deleted_at.is_(None)
).update({
ProductionNote.deleted_at: deleted_at,
ProductionNote.deleted_by: current_user.id
}, synchronize_session=False)
result.marked_deleted_notes = note_update_count
# Mark reviews as deleted (for submissions that belong to these tasks)
# First get submission IDs
submission_ids = [sub.id for sub in db.query(Submission.id).filter(
Submission.task_id.in_(task_ids)
).all()]
if submission_ids:
review_update_count = db.query(Review).filter(
Review.submission_id.in_(submission_ids),
Review.deleted_at.is_(None)
).update({
Review.deleted_at: deleted_at,
Review.deleted_by: current_user.id
}, synchronize_session=False)
result.marked_deleted_reviews = review_update_count
db.flush()
def _get_affected_users(self, tasks: List[Task], submissions: List[Submission],
notes: List[ProductionNote], db: Session) -> List[Dict[str, Any]]:
"""
Get list of users affected by the asset deletion.
Args:
tasks: List of tasks that will be deleted
submissions: List of submissions that will be deleted
notes: List of production notes that will be deleted
db: Database session
Returns:
List of affected user information
"""
user_data = {}
# Collect user IDs and their involvement
for task in tasks:
if task.assigned_user_id:
if task.assigned_user_id not in user_data:
user_data[task.assigned_user_id] = {
'task_count': 0,
'submission_count': 0,
'note_count': 0,
'last_activity_date': None
}
user_data[task.assigned_user_id]['task_count'] += 1
if task.updated_at:
current_date = user_data[task.assigned_user_id]['last_activity_date']
if not current_date or task.updated_at > current_date:
user_data[task.assigned_user_id]['last_activity_date'] = task.updated_at
for submission in submissions:
if submission.user_id not in user_data:
user_data[submission.user_id] = {
'task_count': 0,
'submission_count': 0,
'note_count': 0,
'last_activity_date': None
}
user_data[submission.user_id]['submission_count'] += 1
if submission.submitted_at:
current_date = user_data[submission.user_id]['last_activity_date']
if not current_date or submission.submitted_at > current_date:
user_data[submission.user_id]['last_activity_date'] = submission.submitted_at
for note in notes:
if note.user_id not in user_data:
user_data[note.user_id] = {
'task_count': 0,
'submission_count': 0,
'note_count': 0,
'last_activity_date': None
}
user_data[note.user_id]['note_count'] += 1
if note.updated_at:
current_date = user_data[note.user_id]['last_activity_date']
if not current_date or note.updated_at > current_date:
user_data[note.user_id]['last_activity_date'] = note.updated_at
# Get user details
affected_users = []
if user_data:
users = db.query(User).filter(User.id.in_(user_data.keys())).all()
for user in users:
data = user_data[user.id]
affected_users.append({
'id': user.id,
'name': f"{user.first_name} {user.last_name}",
'email': user.email,
'role': user.role,
'task_count': data['task_count'],
'submission_count': data['submission_count'],
'note_count': data['note_count'],
'last_activity_date': data['last_activity_date'].isoformat() if data['last_activity_date'] else None
})
return affected_users
def _get_last_activity_date(self, asset_id: int, task_ids: List[int], db: Session) -> Optional[str]:
"""
Get the most recent activity date for the asset and its tasks.
Args:
asset_id: ID of the asset
task_ids: List of task IDs
db: Database session
Returns:
ISO formatted date string of last activity, or None
"""
try:
# Get the most recent activity from various sources
dates = []
# Asset updated_at
asset = db.query(Asset.updated_at).filter(Asset.id == asset_id).first()
if asset and asset.updated_at:
dates.append(asset.updated_at)
if task_ids:
# Task updated_at
task_dates = db.query(func.max(Task.updated_at)).filter(
Task.id.in_(task_ids)
).scalar()
if task_dates:
dates.append(task_dates)
# Submission submitted_at
submission_dates = db.query(func.max(Submission.submitted_at)).filter(
Submission.task_id.in_(task_ids)
).scalar()
if submission_dates:
dates.append(submission_dates)
# Production note updated_at
note_dates = db.query(func.max(ProductionNote.updated_at)).filter(
ProductionNote.task_id.in_(task_ids)
).scalar()
if note_dates:
dates.append(note_dates)
if dates:
return max(dates).isoformat()
return None
except SQLAlchemyError:
return None
def _log_asset_deletion(self, asset: Asset, current_user: User, deletion_info: Optional[AssetDeletionInfo], db: Session) -> None:
"""
Log the asset deletion activity.
Args:
asset: The asset that was deleted
current_user: User who performed the deletion
deletion_info: Information about what was deleted
db: Database session
"""
try:
# Create activity record
activity = Activity(
type=ActivityType.ASSET_UPDATED, # We'll use ASSET_UPDATED for now, could add ASSET_DELETED later
user_id=current_user.id,
project_id=asset.project_id,
asset_id=asset.id,
description=f"Asset '{asset.name}' was deleted by {current_user.first_name} {current_user.last_name}",
activity_metadata={
'action': 'soft_delete',
'asset_name': asset.name,
'asset_category': asset.category.value if asset.category else None,
'project_name': asset.project.name if asset.project else None,
'deleted_counts': {
'tasks': deletion_info.task_count if deletion_info else 0,
'submissions': deletion_info.submission_count if deletion_info else 0,
'attachments': deletion_info.attachment_count if deletion_info else 0,
'notes': deletion_info.note_count if deletion_info else 0,
'reviews': deletion_info.review_count if deletion_info else 0
},
'affected_users_count': len(deletion_info.affected_users) if deletion_info else 0
}
)
db.add(activity)
db.flush()
except SQLAlchemyError as e:
# Don't fail the deletion if logging fails, just add a warning
pass
backend/services/batch_operations.py
+902
View File
@@ -0,0 +1,902 @@
#!/usr/bin/env python3
"""
Batch Operations Service
This service provides efficient batch operations for soft deletion and recovery
of multiple shots and assets. It optimizes database operations by using bulk
updates and proper transaction management.
"""
from datetime import datetime
from typing import List, Dict, Any, Optional, Tuple
from sqlalchemy.orm import Session
from sqlalchemy.exc import SQLAlchemyError
from sqlalchemy import func, and_, or_
import sys
import os
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from models.shot import Shot
from models.asset import Asset
from models.task import Task, Submission, Review, ProductionNote, TaskAttachment
from models.user import User
from models.activity import Activity, ActivityType
from services.shot_soft_deletion import ShotSoftDeletionService, DeletionResult
from services.asset_soft_deletion import AssetSoftDeletionService, AssetDeletionResult
from services.recovery_service import RecoveryService, RecoveryResult
class BatchDeletionItem:
"""Information about an item in a batch deletion."""
def __init__(self):
self.id: int = 0
self.name: str = ""
self.type: str = "" # 'shot' or 'asset'
self.success: bool = False
self.error: Optional[str] = None
self.deleted_counts: Dict[str, int] = {}
class BatchDeletionResult:
"""Result of a batch deletion operation."""
def __init__(self):
self.total_items: int = 0
self.successful_deletions: int = 0
self.failed_deletions: int = 0
self.operation_duration: float = 0.0
self.items: List[BatchDeletionItem] = []
# Aggregate counts
self.total_deleted_tasks: int = 0
self.total_deleted_submissions: int = 0
self.total_deleted_attachments: int = 0
self.total_deleted_notes: int = 0
self.total_deleted_reviews: int = 0
class BatchRecoveryItem:
"""Information about an item in a batch recovery."""
def __init__(self):
self.id: int = 0
self.name: str = ""
self.type: str = "" # 'shot' or 'asset'
self.success: bool = False
self.error: Optional[str] = None
self.recovered_counts: Dict[str, int] = {}
class BatchRecoveryResult:
"""Result of a batch recovery operation."""
def __init__(self):
self.total_items: int = 0
self.successful_recoveries: int = 0
self.failed_recoveries: int = 0
self.operation_duration: float = 0.0
self.items: List[BatchRecoveryItem] = []
# Aggregate counts
self.total_recovered_tasks: int = 0
self.total_recovered_submissions: int = 0
self.total_recovered_attachments: int = 0
self.total_recovered_notes: int = 0
self.total_recovered_reviews: int = 0
class BatchOperationsService:
"""Service for efficient batch operations on shots and assets."""
def __init__(self):
self.shot_deletion_service = ShotSoftDeletionService()
self.asset_deletion_service = AssetSoftDeletionService()
self.recovery_service = RecoveryService()
def batch_delete_shots(self, shot_ids: List[int], db: Session, current_user: User,
batch_size: int = 50) -> BatchDeletionResult:
"""
Efficiently delete multiple shots in batches.
Args:
shot_ids: List of shot IDs to delete
db: Database session
current_user: User performing the deletion
batch_size: Number of shots to process in each batch
Returns:
BatchDeletionResult with operation details
"""
start_time = datetime.utcnow()
result = BatchDeletionResult()
result.total_items = len(shot_ids)
try:
# Process shots in batches to avoid memory issues
for i in range(0, len(shot_ids), batch_size):
batch_shot_ids = shot_ids[i:i + batch_size]
# Process each shot in the batch
for shot_id in batch_shot_ids:
item = BatchDeletionItem()
item.id = shot_id
item.type = "shot"
try:
# Get deletion info first
deletion_info = self.shot_deletion_service.get_deletion_info(shot_id, db)
if not deletion_info:
item.success = False
item.error = "Shot not found or already deleted"
result.failed_deletions += 1
result.items.append(item)
continue
# Perform deletion manually without nested transaction
deleted_at = datetime.utcnow()
# Get the shot
shot = db.query(Shot).filter(
Shot.id == shot_id,
Shot.deleted_at.is_(None)
).first()
if not shot:
item.success = False
item.error = "Shot not found or already deleted"
result.failed_deletions += 1
result.items.append(item)
continue
# Mark related data as deleted
tasks = db.query(Task).filter(
Task.shot_id == shot_id,
Task.deleted_at.is_(None)
).all()
task_ids = [task.id for task in tasks] if tasks else []
# Update tasks
task_update_count = db.query(Task).filter(
Task.shot_id == shot_id,
Task.deleted_at.is_(None)
).update({
Task.deleted_at: deleted_at,
Task.deleted_by: current_user.id
}, synchronize_session=False)
# Update related data
submission_update_count = 0
attachment_update_count = 0
note_update_count = 0
review_update_count = 0
if task_ids:
submission_update_count = db.query(Submission).filter(
Submission.task_id.in_(task_ids),
Submission.deleted_at.is_(None)
).update({
Submission.deleted_at: deleted_at,
Submission.deleted_by: current_user.id
}, synchronize_session=False)
attachment_update_count = db.query(TaskAttachment).filter(
TaskAttachment.task_id.in_(task_ids),
TaskAttachment.deleted_at.is_(None)
).update({
TaskAttachment.deleted_at: deleted_at,
TaskAttachment.deleted_by: current_user.id
}, synchronize_session=False)
note_update_count = db.query(ProductionNote).filter(
ProductionNote.task_id.in_(task_ids),
ProductionNote.deleted_at.is_(None)
).update({
ProductionNote.deleted_at: deleted_at,
ProductionNote.deleted_by: current_user.id
}, synchronize_session=False)
# Update reviews
submission_ids = [s.id for s in db.query(Submission.id).filter(
Submission.task_id.in_(task_ids)
).all()]
if submission_ids:
review_update_count = db.query(Review).filter(
Review.submission_id.in_(submission_ids),
Review.deleted_at.is_(None)
).update({
Review.deleted_at: deleted_at,
Review.deleted_by: current_user.id
}, synchronize_session=False)
# Mark the shot as deleted
shot.deleted_at = deleted_at
shot.deleted_by = current_user.id
db.flush()
# Success
item.success = True
item.name = shot.name
item.deleted_counts = {
'tasks': task_update_count,
'submissions': submission_update_count,
'attachments': attachment_update_count,
'notes': note_update_count,
'reviews': review_update_count
}
result.successful_deletions += 1
result.total_deleted_tasks += task_update_count
result.total_deleted_submissions += submission_update_count
result.total_deleted_attachments += attachment_update_count
result.total_deleted_notes += note_update_count
result.total_deleted_reviews += review_update_count
except Exception as e:
item.success = False
item.error = str(e)
result.failed_deletions += 1
result.items.append(item)
# Commit batch to avoid long-running transactions
db.commit()
except Exception as e:
db.rollback()
# Mark remaining items as failed
for shot_id in shot_ids[len(result.items):]:
item = BatchDeletionItem()
item.id = shot_id
item.type = "shot"
item.success = False
item.error = f"Batch operation failed: {str(e)}"
result.items.append(item)
result.failed_deletions += 1
# Calculate operation duration
end_time = datetime.utcnow()
result.operation_duration = (end_time - start_time).total_seconds()
return result
def batch_delete_assets(self, asset_ids: List[int], db: Session, current_user: User,
batch_size: int = 50) -> BatchDeletionResult:
"""
Efficiently delete multiple assets in batches.
Args:
asset_ids: List of asset IDs to delete
db: Database session
current_user: User performing the deletion
batch_size: Number of assets to process in each batch
Returns:
BatchDeletionResult with operation details
"""
start_time = datetime.utcnow()
result = BatchDeletionResult()
result.total_items = len(asset_ids)
try:
# Process assets in batches to avoid memory issues
for i in range(0, len(asset_ids), batch_size):
batch_asset_ids = asset_ids[i:i + batch_size]
# Process each asset in the batch
for asset_id in batch_asset_ids:
item = BatchDeletionItem()
item.id = asset_id
item.type = "asset"
try:
# Get deletion info first
deletion_info = self.asset_deletion_service.get_deletion_info(asset_id, db)
if not deletion_info:
item.success = False
item.error = "Asset not found or already deleted"
result.failed_deletions += 1
result.items.append(item)
continue
# Perform deletion manually without nested transaction
deleted_at = datetime.utcnow()
# Get the asset
asset = db.query(Asset).filter(
Asset.id == asset_id,
Asset.deleted_at.is_(None)
).first()
if not asset:
item.success = False
item.error = "Asset not found or already deleted"
result.failed_deletions += 1
result.items.append(item)
continue
# Mark related data as deleted
tasks = db.query(Task).filter(
Task.asset_id == asset_id,
Task.deleted_at.is_(None)
).all()
task_ids = [task.id for task in tasks] if tasks else []
# Update tasks
task_update_count = db.query(Task).filter(
Task.asset_id == asset_id,
Task.deleted_at.is_(None)
).update({
Task.deleted_at: deleted_at,
Task.deleted_by: current_user.id
}, synchronize_session=False)
# Update related data
submission_update_count = 0
attachment_update_count = 0
note_update_count = 0
review_update_count = 0
if task_ids:
submission_update_count = db.query(Submission).filter(
Submission.task_id.in_(task_ids),
Submission.deleted_at.is_(None)
).update({
Submission.deleted_at: deleted_at,
Submission.deleted_by: current_user.id
}, synchronize_session=False)
attachment_update_count = db.query(TaskAttachment).filter(
TaskAttachment.task_id.in_(task_ids),
TaskAttachment.deleted_at.is_(None)
).update({
TaskAttachment.deleted_at: deleted_at,
TaskAttachment.deleted_by: current_user.id
}, synchronize_session=False)
note_update_count = db.query(ProductionNote).filter(
ProductionNote.task_id.in_(task_ids),
ProductionNote.deleted_at.is_(None)
).update({
ProductionNote.deleted_at: deleted_at,
ProductionNote.deleted_by: current_user.id
}, synchronize_session=False)
# Update reviews
submission_ids = [s.id for s in db.query(Submission.id).filter(
Submission.task_id.in_(task_ids)
).all()]
if submission_ids:
review_update_count = db.query(Review).filter(
Review.submission_id.in_(submission_ids),
Review.deleted_at.is_(None)
).update({
Review.deleted_at: deleted_at,
Review.deleted_by: current_user.id
}, synchronize_session=False)
# Mark the asset as deleted
asset.deleted_at = deleted_at
asset.deleted_by = current_user.id
db.flush()
# Success
item.success = True
item.name = asset.name
item.deleted_counts = {
'tasks': task_update_count,
'submissions': submission_update_count,
'attachments': attachment_update_count,
'notes': note_update_count,
'reviews': review_update_count
}
result.successful_deletions += 1
result.total_deleted_tasks += task_update_count
result.total_deleted_submissions += submission_update_count
result.total_deleted_attachments += attachment_update_count
result.total_deleted_notes += note_update_count
result.total_deleted_reviews += review_update_count
except Exception as e:
item.success = False
item.error = str(e)
result.failed_deletions += 1
result.items.append(item)
# Commit batch to avoid long-running transactions
db.commit()
except Exception as e:
db.rollback()
# Mark remaining items as failed
for asset_id in asset_ids[len(result.items):]:
item = BatchDeletionItem()
item.id = asset_id
item.type = "asset"
item.success = False
item.error = f"Batch operation failed: {str(e)}"
result.items.append(item)
result.failed_deletions += 1
# Calculate operation duration
end_time = datetime.utcnow()
result.operation_duration = (end_time - start_time).total_seconds()
return result
def batch_recover_shots(self, shot_ids: List[int], db: Session, current_user: User,
batch_size: int = 50) -> BatchRecoveryResult:
"""
Efficiently recover multiple shots in batches.
Args:
shot_ids: List of shot IDs to recover
db: Database session
current_user: User performing the recovery
batch_size: Number of shots to process in each batch
Returns:
BatchRecoveryResult with operation details
"""
start_time = datetime.utcnow()
result = BatchRecoveryResult()
result.total_items = len(shot_ids)
try:
# Process shots in batches to avoid memory issues
for i in range(0, len(shot_ids), batch_size):
batch_shot_ids = shot_ids[i:i + batch_size]
# Process each shot in the batch
for shot_id in batch_shot_ids:
item = BatchRecoveryItem()
item.id = shot_id
item.type = "shot"
try:
# Get the deleted shot
shot = db.query(Shot).filter(
Shot.id == shot_id,
Shot.deleted_at.isnot(None)
).first()
if not shot:
item.success = False
item.error = "Shot not found or not deleted"
result.failed_recoveries += 1
result.items.append(item)
continue
# Get deleted task IDs
task_ids = [t.id for t in db.query(Task.id).filter(
Task.shot_id == shot_id,
Task.deleted_at.isnot(None)
).all()]
# Recover tasks
recovered_tasks = db.query(Task).filter(
Task.shot_id == shot_id,
Task.deleted_at.isnot(None)
).update({
Task.deleted_at: None,
Task.deleted_by: None
}, synchronize_session=False)
# Recover related data
recovered_submissions = 0
recovered_attachments = 0
recovered_notes = 0
recovered_reviews = 0
if task_ids:
recovered_submissions = db.query(Submission).filter(
Submission.task_id.in_(task_ids),
Submission.deleted_at.isnot(None)
).update({
Submission.deleted_at: None,
Submission.deleted_by: None
}, synchronize_session=False)
recovered_attachments = db.query(TaskAttachment).filter(
TaskAttachment.task_id.in_(task_ids),
TaskAttachment.deleted_at.isnot(None)
).update({
TaskAttachment.deleted_at: None,
TaskAttachment.deleted_by: None
}, synchronize_session=False)
recovered_notes = db.query(ProductionNote).filter(
ProductionNote.task_id.in_(task_ids),
ProductionNote.deleted_at.isnot(None)
).update({
ProductionNote.deleted_at: None,
ProductionNote.deleted_by: None
}, synchronize_session=False)
# Recover reviews
submission_ids = [s.id for s in db.query(Submission.id).filter(
Submission.task_id.in_(task_ids)
).all()]
if submission_ids:
recovered_reviews = db.query(Review).filter(
Review.submission_id.in_(submission_ids),
Review.deleted_at.isnot(None)
).update({
Review.deleted_at: None,
Review.deleted_by: None
}, synchronize_session=False)
# Recover the shot
shot.deleted_at = None
shot.deleted_by = None
db.flush()
# Success
item.success = True
item.name = shot.name
item.recovered_counts = {
'tasks': recovered_tasks,
'submissions': recovered_submissions,
'attachments': recovered_attachments,
'notes': recovered_notes,
'reviews': recovered_reviews
}
result.successful_recoveries += 1
result.total_recovered_tasks += recovered_tasks
result.total_recovered_submissions += recovered_submissions
result.total_recovered_attachments += recovered_attachments
result.total_recovered_notes += recovered_notes
result.total_recovered_reviews += recovered_reviews
except Exception as e:
item.success = False
item.error = str(e)
result.failed_recoveries += 1
result.items.append(item)
# Commit batch to avoid long-running transactions
db.commit()
except Exception as e:
db.rollback()
# Mark remaining items as failed
for shot_id in shot_ids[len(result.items):]:
item = BatchRecoveryItem()
item.id = shot_id
item.type = "shot"
item.success = False
item.error = f"Batch operation failed: {str(e)}"
result.items.append(item)
result.failed_recoveries += 1
# Calculate operation duration
end_time = datetime.utcnow()
result.operation_duration = (end_time - start_time).total_seconds()
return result
def batch_recover_assets(self, asset_ids: List[int], db: Session, current_user: User,
batch_size: int = 50) -> BatchRecoveryResult:
"""
Efficiently recover multiple assets in batches.
Args:
asset_ids: List of asset IDs to recover
db: Database session
current_user: User performing the recovery
batch_size: Number of assets to process in each batch
Returns:
BatchRecoveryResult with operation details
"""
start_time = datetime.utcnow()
result = BatchRecoveryResult()
result.total_items = len(asset_ids)
try:
# Process assets in batches to avoid memory issues
for i in range(0, len(asset_ids), batch_size):
batch_asset_ids = asset_ids[i:i + batch_size]
# Process each asset in the batch
for asset_id in batch_asset_ids:
item = BatchRecoveryItem()
item.id = asset_id
item.type = "asset"
try:
# Get the deleted asset
asset = db.query(Asset).filter(
Asset.id == asset_id,
Asset.deleted_at.isnot(None)
).first()
if not asset:
item.success = False
item.error = "Asset not found or not deleted"
result.failed_recoveries += 1
result.items.append(item)
continue
# Get deleted task IDs
task_ids = [t.id for t in db.query(Task.id).filter(
Task.asset_id == asset_id,
Task.deleted_at.isnot(None)
).all()]
# Recover tasks
recovered_tasks = db.query(Task).filter(
Task.asset_id == asset_id,
Task.deleted_at.isnot(None)
).update({
Task.deleted_at: None,
Task.deleted_by: None
}, synchronize_session=False)
# Recover related data
recovered_submissions = 0
recovered_attachments = 0
recovered_notes = 0
recovered_reviews = 0
if task_ids:
recovered_submissions = db.query(Submission).filter(
Submission.task_id.in_(task_ids),
Submission.deleted_at.isnot(None)
).update({
Submission.deleted_at: None,
Submission.deleted_by: None
}, synchronize_session=False)
recovered_attachments = db.query(TaskAttachment).filter(
TaskAttachment.task_id.in_(task_ids),
TaskAttachment.deleted_at.isnot(None)
).update({
TaskAttachment.deleted_at: None,
TaskAttachment.deleted_by: None
}, synchronize_session=False)
recovered_notes = db.query(ProductionNote).filter(
ProductionNote.task_id.in_(task_ids),
ProductionNote.deleted_at.isnot(None)
).update({
ProductionNote.deleted_at: None,
ProductionNote.deleted_by: None
}, synchronize_session=False)
# Recover reviews
submission_ids = [s.id for s in db.query(Submission.id).filter(
Submission.task_id.in_(task_ids)
).all()]
if submission_ids:
recovered_reviews = db.query(Review).filter(
Review.submission_id.in_(submission_ids),
Review.deleted_at.isnot(None)
).update({
Review.deleted_at: None,
Review.deleted_by: None
}, synchronize_session=False)
# Recover the asset
asset.deleted_at = None
asset.deleted_by = None
db.flush()
# Success
item.success = True
item.name = asset.name
item.recovered_counts = {
'tasks': recovered_tasks,
'submissions': recovered_submissions,
'attachments': recovered_attachments,
'notes': recovered_notes,
'reviews': recovered_reviews
}
result.successful_recoveries += 1
result.total_recovered_tasks += recovered_tasks
result.total_recovered_submissions += recovered_submissions
result.total_recovered_attachments += recovered_attachments
result.total_recovered_notes += recovered_notes
result.total_recovered_reviews += recovered_reviews
except Exception as e:
item.success = False
item.error = str(e)
result.failed_recoveries += 1
result.items.append(item)
# Commit batch to avoid long-running transactions
db.commit()
except Exception as e:
db.rollback()
# Mark remaining items as failed
for asset_id in asset_ids[len(result.items):]:
item = BatchRecoveryItem()
item.id = asset_id
item.type = "asset"
item.success = False
item.error = f"Batch operation failed: {str(e)}"
result.items.append(item)
result.failed_recoveries += 1
# Calculate operation duration
end_time = datetime.utcnow()
result.operation_duration = (end_time - start_time).total_seconds()
return result
def get_batch_deletion_preview(self, shot_ids: List[int], asset_ids: List[int],
db: Session) -> Dict[str, Any]:
"""
Get preview information for a batch deletion operation.
Args:
shot_ids: List of shot IDs to delete
asset_ids: List of asset IDs to delete
db: Database session
Returns:
Dictionary with preview information
"""
preview = {
'total_shots': len(shot_ids),
'total_assets': len(asset_ids),
'total_items': len(shot_ids) + len(asset_ids),
'estimated_tasks': 0,
'estimated_submissions': 0,
'estimated_attachments': 0,
'estimated_notes': 0,
'estimated_reviews': 0,
'affected_users': set(),
'projects_affected': set()
}
try:
# Count tasks for shots
if shot_ids:
shot_task_count = db.query(Task).filter(
Task.shot_id.in_(shot_ids),
Task.deleted_at.is_(None)
).count()
preview['estimated_tasks'] += shot_task_count
# Get affected users and projects for shots
shot_users = db.query(Task.assigned_user_id).filter(
Task.shot_id.in_(shot_ids),
Task.deleted_at.is_(None),
Task.assigned_user_id.isnot(None)
).distinct().all()
preview['affected_users'].update([u[0] for u in shot_users])
shot_projects = db.query(Shot.episode_id).join(Shot.episode).filter(
Shot.id.in_(shot_ids)
).distinct().all()
preview['projects_affected'].update([p[0] for p in shot_projects])
# Count tasks for assets
if asset_ids:
asset_task_count = db.query(Task).filter(
Task.asset_id.in_(asset_ids),
Task.deleted_at.is_(None)
).count()
preview['estimated_tasks'] += asset_task_count
# Get affected users and projects for assets
asset_users = db.query(Task.assigned_user_id).filter(
Task.asset_id.in_(asset_ids),
Task.deleted_at.is_(None),
Task.assigned_user_id.isnot(None)
).distinct().all()
preview['affected_users'].update([u[0] for u in asset_users])
asset_projects = db.query(Asset.project_id).filter(
Asset.id.in_(asset_ids)
).distinct().all()
preview['projects_affected'].update([p[0] for p in asset_projects])
# Get all task IDs for counting related items
all_task_ids = []
if shot_ids:
shot_task_ids = [t.id for t in db.query(Task.id).filter(
Task.shot_id.in_(shot_ids),
Task.deleted_at.is_(None)
).all()]
all_task_ids.extend(shot_task_ids)
if asset_ids:
asset_task_ids = [t.id for t in db.query(Task.id).filter(
Task.asset_id.in_(asset_ids),
Task.deleted_at.is_(None)
).all()]
all_task_ids.extend(asset_task_ids)
if all_task_ids:
# Count submissions
preview['estimated_submissions'] = db.query(Submission).filter(
Submission.task_id.in_(all_task_ids),
Submission.deleted_at.is_(None)
).count()
# Count attachments
preview['estimated_attachments'] = db.query(TaskAttachment).filter(
TaskAttachment.task_id.in_(all_task_ids),
TaskAttachment.deleted_at.is_(None)
).count()
# Count notes
preview['estimated_notes'] = db.query(ProductionNote).filter(
ProductionNote.task_id.in_(all_task_ids),
ProductionNote.deleted_at.is_(None)
).count()
# Count reviews
submission_ids = [s.id for s in db.query(Submission.id).filter(
Submission.task_id.in_(all_task_ids),
Submission.deleted_at.is_(None)
).all()]
if submission_ids:
preview['estimated_reviews'] = db.query(Review).filter(
Review.submission_id.in_(submission_ids),
Review.deleted_at.is_(None)
).count()
# Convert sets to counts
preview['affected_users_count'] = len(preview['affected_users'])
preview['projects_affected_count'] = len(preview['projects_affected'])
# Remove the sets from the response (not JSON serializable)
del preview['affected_users']
del preview['projects_affected']
except SQLAlchemyError:
# Return basic preview on error
pass
return preview
def optimize_batch_size(self, total_items: int, estimated_related_items: int) -> int:
"""
Calculate optimal batch size based on the number of items and their complexity.
Args:
total_items: Total number of items to process
estimated_related_items: Estimated number of related items (tasks, submissions, etc.)
Returns:
Optimal batch size
"""
# Base batch size
base_batch_size = 50
# Adjust based on complexity
if estimated_related_items == 0:
# No related items, can use larger batches
return min(100, total_items)
# Calculate complexity factor
complexity_factor = estimated_related_items / total_items if total_items > 0 else 1
if complexity_factor > 100:
# Very complex items, use smaller batches
return min(10, total_items)
elif complexity_factor > 50:
# Moderately complex items
return min(25, total_items)
else:
# Simple items, use base batch size
return min(base_batch_size, total_items)
backend/services/data_consistency.py
+421
View File
@@ -0,0 +1,421 @@
"""
Data Consistency Service for Shot/Asset Task Status Optimization
This service ensures data consistency between individual task updates and aggregated views,
and provides real-time update propagation mechanisms.
"""
from typing import Dict, List, Optional, Any, Set
from sqlalchemy.orm import Session
from sqlalchemy import and_, or_, func
from datetime import datetime
import json
import logging
from models.task import Task
from models.shot import Shot
from models.asset import Asset
from models.project import Project
from models.episode import Episode
from schemas.shot import TaskStatusInfo
logger = logging.getLogger(__name__)
class DataConsistencyService:
"""Service for maintaining data consistency between individual task updates and aggregated views."""
def __init__(self, db: Session):
self.db = db
def validate_task_aggregation_consistency(self, entity_id: int, entity_type: str) -> Dict[str, Any]:
"""
Validate that aggregated task data matches individual task records.
Args:
entity_id: ID of the shot or asset
entity_type: 'shot' or 'asset'
Returns:
Dict containing validation results and any inconsistencies found
"""
logger.info(f"Validating task aggregation consistency for {entity_type} {entity_id}")
# Get individual task records
if entity_type == 'shot':
tasks = self.db.query(Task).filter(
and_(
Task.shot_id == entity_id,
Task.deleted_at.is_(None)
)
).all()
entity = self.db.query(Shot).filter(Shot.id == entity_id).first()
elif entity_type == 'asset':
tasks = self.db.query(Task).filter(
and_(
Task.asset_id == entity_id,
Task.deleted_at.is_(None)
)
).all()
entity = self.db.query(Asset).filter(Asset.id == entity_id).first()
else:
raise ValueError(f"Invalid entity_type: {entity_type}")
if not entity:
return {
'valid': False,
'error': f'{entity_type.title()} not found',
'inconsistencies': []
}
# Build expected aggregated data from individual tasks
expected_task_status = {}
expected_task_details = []
# Get all task types for the project
project = self.db.query(Project).filter(Project.id == entity.project_id).first()
if not project:
return {
'valid': False,
'error': 'Project not found',
'inconsistencies': []
}
# Get standard and custom task types
if entity_type == 'shot':
standard_types = ["layout", "animation", "simulation", "lighting", "compositing"]
custom_types = project.custom_shot_task_types or []
else: # asset
standard_types = ["modeling", "surfacing", "rigging"]
custom_types = project.custom_asset_task_types or []
all_task_types = standard_types + custom_types
# Initialize all task types as not_started
for task_type in all_task_types:
expected_task_status[task_type] = "not_started"
# Update with actual task data
for task in tasks:
expected_task_status[task.task_type] = task.status
expected_task_details.append(TaskStatusInfo(
task_type=task.task_type,
status=task.status,
task_id=task.id,
assigned_user_id=task.assigned_user_id
))
# Get current aggregated data using the optimized query
if entity_type == 'shot':
aggregated_data = self._get_shot_aggregated_data(entity_id)
else:
aggregated_data = self._get_asset_aggregated_data(entity_id)
# Compare expected vs actual aggregated data
inconsistencies = []
# Check task_status consistency
for task_type, expected_status in expected_task_status.items():
actual_status = aggregated_data.get('task_status', {}).get(task_type)
if actual_status != expected_status:
inconsistencies.append({
'type': 'task_status_mismatch',
'task_type': task_type,
'expected': expected_status,
'actual': actual_status
})
# Check task_details consistency
actual_task_details = aggregated_data.get('task_details', [])
expected_task_ids = {detail.task_id for detail in expected_task_details}
actual_task_ids = {detail.get('task_id') for detail in actual_task_details if detail.get('task_id')}
if expected_task_ids != actual_task_ids:
inconsistencies.append({
'type': 'task_details_mismatch',
'expected_task_ids': list(expected_task_ids),
'actual_task_ids': list(actual_task_ids),
'missing_tasks': list(expected_task_ids - actual_task_ids),
'extra_tasks': list(actual_task_ids - expected_task_ids)
})
return {
'valid': len(inconsistencies) == 0,
'entity_id': entity_id,
'entity_type': entity_type,
'inconsistencies': inconsistencies,
'expected_task_status': expected_task_status,
'actual_task_status': aggregated_data.get('task_status', {}),
'validation_timestamp': datetime.utcnow().isoformat()
}
def _get_shot_aggregated_data(self, shot_id: int) -> Dict[str, Any]:
"""Get aggregated task data for a shot using the optimized query."""
from sqlalchemy.orm import joinedload, selectinload
shot_with_tasks = (
self.db.query(Shot)
.filter(and_(Shot.id == shot_id, Shot.deleted_at.is_(None)))
.outerjoin(Task, (Task.shot_id == Shot.id) & (Task.deleted_at.is_(None)))
.options(
joinedload(Shot.episode).joinedload(Episode.project),
selectinload(Shot.tasks).options(selectinload(Task.assigned_user))
)
.add_columns(
Task.id.label('task_id'),
Task.task_type,
Task.status.label('task_status'),
Task.assigned_user_id,
Task.updated_at.label('task_updated_at')
)
.all()
)
if not shot_with_tasks:
return {'task_status': {}, 'task_details': []}
# Process the results similar to the optimized list_shots implementation
shot = shot_with_tasks[0][0]
project = shot.episode.project if shot.episode else None
if not project:
return {'task_status': {}, 'task_details': []}
# Get task types
standard_types = ["layout", "animation", "simulation", "lighting", "compositing"]
custom_types = project.custom_shot_task_types or []
all_task_types = standard_types + custom_types
# Initialize task status
task_status = {}
for task_type in all_task_types:
task_status[task_type] = "not_started"
# Build task details
task_details = []
for row in shot_with_tasks:
task_id = row[1]
task_type = row[2]
task_status_val = row[3]
assigned_user_id = row[4]
if task_id is not None:
task_status[task_type] = task_status_val
task_details.append({
'task_type': task_type,
'status': task_status_val,
'task_id': task_id,
'assigned_user_id': assigned_user_id
})
return {
'task_status': task_status,
'task_details': task_details
}
def _get_asset_aggregated_data(self, asset_id: int) -> Dict[str, Any]:
"""Get aggregated task data for an asset using the optimized query."""
from sqlalchemy.orm import joinedload, selectinload
asset_with_tasks = (
self.db.query(Asset)
.filter(and_(Asset.id == asset_id, Asset.deleted_at.is_(None)))
.outerjoin(Task, (Task.asset_id == Asset.id) & (Task.deleted_at.is_(None)))
.options(
joinedload(Asset.project),
selectinload(Asset.tasks).options(selectinload(Task.assigned_user))
)
.add_columns(
Task.id.label('task_id'),
Task.task_type,
Task.status.label('task_status'),
Task.assigned_user_id,
Task.updated_at.label('task_updated_at')
)
.all()
)
if not asset_with_tasks:
return {'task_status': {}, 'task_details': []}
# Process the results similar to the optimized list_assets implementation
asset = asset_with_tasks[0][0]
project = asset.project
if not project:
return {'task_status': {}, 'task_details': []}
# Get task types
standard_types = ["modeling", "surfacing", "rigging"]
custom_types = project.custom_asset_task_types or []
all_task_types = standard_types + custom_types
# Initialize task status
task_status = {}
for task_type in all_task_types:
task_status[task_type] = "not_started"
# Build task details
task_details = []
for row in asset_with_tasks:
task_id = row[1]
task_type = row[2]
task_status_val = row[3]
assigned_user_id = row[4]
if task_id is not None:
task_status[task_type] = task_status_val
task_details.append({
'task_type': task_type,
'status': task_status_val,
'task_id': task_id,
'assigned_user_id': assigned_user_id
})
return {
'task_status': task_status,
'task_details': task_details
}
def validate_bulk_consistency(self, entity_ids: List[int], entity_type: str) -> Dict[str, Any]:
"""
Validate consistency for multiple entities at once.
Args:
entity_ids: List of shot or asset IDs
entity_type: 'shot' or 'asset'
Returns:
Dict containing bulk validation results
"""
logger.info(f"Validating bulk consistency for {len(entity_ids)} {entity_type}s")
results = []
total_inconsistencies = 0
for entity_id in entity_ids:
validation_result = self.validate_task_aggregation_consistency(entity_id, entity_type)
results.append(validation_result)
if not validation_result['valid']:
total_inconsistencies += len(validation_result.get('inconsistencies', []))
return {
'total_entities': len(entity_ids),
'valid_entities': sum(1 for r in results if r['valid']),
'invalid_entities': sum(1 for r in results if not r['valid']),
'total_inconsistencies': total_inconsistencies,
'results': results,
'validation_timestamp': datetime.utcnow().isoformat()
}
def propagate_task_update(self, task_id: int, old_status: Optional[str] = None, new_status: Optional[str] = None) -> Dict[str, Any]:
"""
Propagate task status changes to ensure real-time updates in aggregated data.
Args:
task_id: ID of the updated task
old_status: Previous task status (optional)
new_status: New task status (optional)
Returns:
Dict containing propagation results
"""
logger.info(f"Propagating task update for task {task_id}")
# Get the task and its parent entity
task = self.db.query(Task).filter(Task.id == task_id).first()
if not task:
return {
'success': False,
'error': 'Task not found',
'task_id': task_id
}
# Determine entity type and ID
if task.shot_id:
entity_type = 'shot'
entity_id = task.shot_id
elif task.asset_id:
entity_type = 'asset'
entity_id = task.asset_id
else:
return {
'success': False,
'error': 'Task is not associated with a shot or asset',
'task_id': task_id
}
# Validate consistency after the update
validation_result = self.validate_task_aggregation_consistency(entity_id, entity_type)
# Log the propagation
propagation_log = {
'task_id': task_id,
'entity_type': entity_type,
'entity_id': entity_id,
'old_status': old_status,
'new_status': new_status or task.status,
'consistency_valid': validation_result['valid'],
'inconsistencies': validation_result.get('inconsistencies', []),
'timestamp': datetime.utcnow().isoformat()
}
logger.info(f"Task update propagated: {json.dumps(propagation_log)}")
return {
'success': True,
'task_id': task_id,
'entity_type': entity_type,
'entity_id': entity_id,
'validation_result': validation_result,
'propagation_log': propagation_log
}
def get_consistency_report(self, project_id: Optional[int] = None) -> Dict[str, Any]:
"""
Generate a comprehensive consistency report for shots and assets.
Args:
project_id: Optional project ID to filter by
Returns:
Dict containing comprehensive consistency report
"""
logger.info(f"Generating consistency report for project {project_id}")
# Get all shots and assets
shot_query = self.db.query(Shot.id).filter(Shot.deleted_at.is_(None))
asset_query = self.db.query(Asset.id).filter(Asset.deleted_at.is_(None))
if project_id:
shot_query = shot_query.filter(Shot.project_id == project_id)
asset_query = asset_query.filter(Asset.project_id == project_id)
shot_ids = [row[0] for row in shot_query.all()]
asset_ids = [row[0] for row in asset_query.all()]
# Validate consistency for all entities
shot_results = self.validate_bulk_consistency(shot_ids, 'shot') if shot_ids else {'total_entities': 0, 'valid_entities': 0, 'invalid_entities': 0, 'total_inconsistencies': 0, 'results': []}
asset_results = self.validate_bulk_consistency(asset_ids, 'asset') if asset_ids else {'total_entities': 0, 'valid_entities': 0, 'invalid_entities': 0, 'total_inconsistencies': 0, 'results': []}
return {
'project_id': project_id,
'shots': shot_results,
'assets': asset_results,
'summary': {
'total_entities': shot_results['total_entities'] + asset_results['total_entities'],
'valid_entities': shot_results['valid_entities'] + asset_results['valid_entities'],
'invalid_entities': shot_results['invalid_entities'] + asset_results['invalid_entities'],
'total_inconsistencies': shot_results['total_inconsistencies'] + asset_results['total_inconsistencies'],
'consistency_percentage': (
(shot_results['valid_entities'] + asset_results['valid_entities']) /
max(1, shot_results['total_entities'] + asset_results['total_entities'])
) * 100
},
'report_timestamp': datetime.utcnow().isoformat()
}
def create_data_consistency_service(db: Session) -> DataConsistencyService:
"""Factory function to create a DataConsistencyService instance."""
return DataConsistencyService(db)
backend/services/recovery_service.py
+1444
View File
File diff suppressed because it is too large Load Diff
backend/services/shot_soft_deletion.py
+486
View File
@@ -0,0 +1,486 @@
"""
Shot Soft Deletion Service
This service handles the soft deletion of shots and all related data including:
- Tasks associated with the shot
- Submissions for those tasks
- Production notes for those tasks
- Task attachments for those tasks
- Reviews for those submissions
All operations are performed within database transactions to ensure atomicity.
"""
from datetime import datetime
from typing import List, Dict, Any, Optional
from sqlalchemy.orm import Session
from sqlalchemy.exc import SQLAlchemyError
from sqlalchemy import func
import sys
import os
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from models.shot import Shot
from models.task import Task, Submission, Review, ProductionNote, TaskAttachment
from models.user import User
from models.activity import Activity, ActivityType
from utils.activity import ActivityService
class DeletionInfo:
"""Information about what will be deleted when a shot is soft deleted."""
def __init__(self):
self.shot_id: int = 0
self.shot_name: str = ""
self.episode_name: str = ""
self.project_id: int = 0
self.project_name: str = ""
# Counts of items that will be marked as deleted
self.task_count: int = 0
self.submission_count: int = 0
self.attachment_count: int = 0
self.note_count: int = 0
self.review_count: int = 0
# File information (preserved, not deleted)
self.total_file_size: int = 0
self.file_count: int = 0
# Affected users
self.affected_users: List[Dict[str, Any]] = []
# Timestamps
self.last_activity_date: Optional[str] = None
self.created_at: str = ""
class DeletionResult:
"""Result of a shot soft deletion operation."""
def __init__(self):
self.success: bool = False
self.shot_id: int = 0
self.shot_name: str = ""
# Database update results
self.marked_deleted_tasks: int = 0
self.marked_deleted_submissions: int = 0
self.marked_deleted_attachments: int = 0
self.marked_deleted_notes: int = 0
self.marked_deleted_reviews: int = 0
# Timing
self.operation_duration: float = 0.0
self.deleted_at: str = ""
self.deleted_by: int = 0
# Errors
self.errors: List[str] = []
self.warnings: List[str] = []
class ShotSoftDeletionService:
"""Service for handling shot soft deletion operations."""
def __init__(self):
self.activity_service = ActivityService()
def get_deletion_info(self, shot_id: int, db: Session) -> Optional[DeletionInfo]:
"""
Get information about what will be deleted when a shot is soft deleted.
Args:
shot_id: ID of the shot to analyze
db: Database session
Returns:
DeletionInfo object with counts and affected users, or None if shot not found
"""
try:
# Get the shot (only if not already deleted)
shot = db.query(Shot).filter(
Shot.id == shot_id,
Shot.deleted_at.is_(None)
).first()
if not shot:
return None
info = DeletionInfo()
info.shot_id = shot.id
info.shot_name = shot.name
info.episode_name = shot.episode.name if shot.episode else ""
info.project_id = shot.project_id
info.project_name = shot.project.name if shot.project else ""
info.created_at = shot.created_at.isoformat() if shot.created_at else ""
# Get all active tasks for this shot
tasks = db.query(Task).filter(
Task.shot_id == shot_id,
Task.deleted_at.is_(None)
).all()
info.task_count = len(tasks)
if not tasks:
return info
task_ids = [task.id for task in tasks]
# Count submissions
submissions = db.query(Submission).filter(
Submission.task_id.in_(task_ids),
Submission.deleted_at.is_(None)
).all()
info.submission_count = len(submissions)
# Count attachments and calculate file sizes
attachments = db.query(TaskAttachment).filter(
TaskAttachment.task_id.in_(task_ids),
TaskAttachment.deleted_at.is_(None)
).all()
info.attachment_count = len(attachments)
info.file_count = len(attachments) + len(submissions)
info.total_file_size = sum(att.file_size for att in attachments if att.file_size)
# Count production notes
notes = db.query(ProductionNote).filter(
ProductionNote.task_id.in_(task_ids),
ProductionNote.deleted_at.is_(None)
).all()
info.note_count = len(notes)
# Count reviews
if submissions:
submission_ids = [sub.id for sub in submissions]
reviews = db.query(Review).filter(
Review.submission_id.in_(submission_ids),
Review.deleted_at.is_(None)
).all()
info.review_count = len(reviews)
# Get affected users
info.affected_users = self._get_affected_users(tasks, submissions, notes, db)
# Get last activity date
info.last_activity_date = self._get_last_activity_date(shot_id, task_ids, db)
return info
except SQLAlchemyError as e:
raise Exception(f"Database error while getting deletion info: {str(e)}")
def soft_delete_shot_cascade(self, shot_id: int, db: Session, current_user: User) -> DeletionResult:
"""
Perform cascading soft deletion of a shot and all related data.
Args:
shot_id: ID of the shot to soft delete
db: Database session
current_user: User performing the deletion
Returns:
DeletionResult with operation details
"""
start_time = datetime.utcnow()
result = DeletionResult()
result.shot_id = shot_id
result.deleted_by = current_user.id
try:
# Get the shot (only if not already deleted)
shot = db.query(Shot).filter(
Shot.id == shot_id,
Shot.deleted_at.is_(None)
).first()
if not shot:
result.errors.append("Shot not found or already deleted")
return result
result.shot_name = shot.name
deleted_at = datetime.utcnow()
result.deleted_at = deleted_at.isoformat()
# Get deletion info for logging
deletion_info = self.get_deletion_info(shot_id, db)
# Mark related data as deleted
self._mark_related_data_deleted(shot_id, db, current_user, deleted_at, result)
# Mark the shot as deleted
shot.deleted_at = deleted_at
shot.deleted_by = current_user.id
db.flush()
# Log the deletion activity
self._log_shot_deletion(shot, current_user, deletion_info, db)
result.success = True
except SQLAlchemyError as e:
result.errors.append(f"Database error during deletion: {str(e)}")
except Exception as e:
result.errors.append(f"Unexpected error during deletion: {str(e)}")
# Calculate operation duration
end_time = datetime.utcnow()
result.operation_duration = (end_time - start_time).total_seconds()
return result
def _mark_related_data_deleted(self, shot_id: int, db: Session, current_user: User,
deleted_at: datetime, result: DeletionResult) -> None:
"""
Mark all data related to a shot as deleted.
Args:
shot_id: ID of the shot
db: Database session
current_user: User performing the deletion
deleted_at: Timestamp for deletion
result: DeletionResult to update with counts
"""
# Get all active tasks for this shot
tasks = db.query(Task).filter(
Task.shot_id == shot_id,
Task.deleted_at.is_(None)
).all()
if not tasks:
return
task_ids = [task.id for task in tasks]
# Mark tasks as deleted
task_update_count = db.query(Task).filter(
Task.shot_id == shot_id,
Task.deleted_at.is_(None)
).update({
Task.deleted_at: deleted_at,
Task.deleted_by: current_user.id
}, synchronize_session=False)
result.marked_deleted_tasks = task_update_count
# Mark submissions as deleted
submission_update_count = db.query(Submission).filter(
Submission.task_id.in_(task_ids),
Submission.deleted_at.is_(None)
).update({
Submission.deleted_at: deleted_at,
Submission.deleted_by: current_user.id
}, synchronize_session=False)
result.marked_deleted_submissions = submission_update_count
# Mark attachments as deleted
attachment_update_count = db.query(TaskAttachment).filter(
TaskAttachment.task_id.in_(task_ids),
TaskAttachment.deleted_at.is_(None)
).update({
TaskAttachment.deleted_at: deleted_at,
TaskAttachment.deleted_by: current_user.id
}, synchronize_session=False)
result.marked_deleted_attachments = attachment_update_count
# Mark production notes as deleted
note_update_count = db.query(ProductionNote).filter(
ProductionNote.task_id.in_(task_ids),
ProductionNote.deleted_at.is_(None)
).update({
ProductionNote.deleted_at: deleted_at,
ProductionNote.deleted_by: current_user.id
}, synchronize_session=False)
result.marked_deleted_notes = note_update_count
# Mark reviews as deleted (for submissions that belong to these tasks)
# First get submission IDs
submission_ids = [sub.id for sub in db.query(Submission.id).filter(
Submission.task_id.in_(task_ids)
).all()]
if submission_ids:
review_update_count = db.query(Review).filter(
Review.submission_id.in_(submission_ids),
Review.deleted_at.is_(None)
).update({
Review.deleted_at: deleted_at,
Review.deleted_by: current_user.id
}, synchronize_session=False)
result.marked_deleted_reviews = review_update_count
db.flush()
def _get_affected_users(self, tasks: List[Task], submissions: List[Submission],
notes: List[ProductionNote], db: Session) -> List[Dict[str, Any]]:
"""
Get list of users affected by the shot deletion.
Args:
tasks: List of tasks that will be deleted
submissions: List of submissions that will be deleted
notes: List of production notes that will be deleted
db: Database session
Returns:
List of affected user information
"""
user_data = {}
# Collect user IDs and their involvement
for task in tasks:
if task.assigned_user_id:
if task.assigned_user_id not in user_data:
user_data[task.assigned_user_id] = {
'task_count': 0,
'submission_count': 0,
'note_count': 0,
'last_activity_date': None
}
user_data[task.assigned_user_id]['task_count'] += 1
if task.updated_at:
current_date = user_data[task.assigned_user_id]['last_activity_date']
if not current_date or task.updated_at > current_date:
user_data[task.assigned_user_id]['last_activity_date'] = task.updated_at
for submission in submissions:
if submission.user_id not in user_data:
user_data[submission.user_id] = {
'task_count': 0,
'submission_count': 0,
'note_count': 0,
'last_activity_date': None
}
user_data[submission.user_id]['submission_count'] += 1
if submission.submitted_at:
current_date = user_data[submission.user_id]['last_activity_date']
if not current_date or submission.submitted_at > current_date:
user_data[submission.user_id]['last_activity_date'] = submission.submitted_at
for note in notes:
if note.user_id not in user_data:
user_data[note.user_id] = {
'task_count': 0,
'submission_count': 0,
'note_count': 0,
'last_activity_date': None
}
user_data[note.user_id]['note_count'] += 1
if note.updated_at:
current_date = user_data[note.user_id]['last_activity_date']
if not current_date or note.updated_at > current_date:
user_data[note.user_id]['last_activity_date'] = note.updated_at
# Get user details
affected_users = []
if user_data:
users = db.query(User).filter(User.id.in_(user_data.keys())).all()
for user in users:
data = user_data[user.id]
affected_users.append({
'id': user.id,
'name': f"{user.first_name} {user.last_name}",
'email': user.email,
'role': user.role,
'task_count': data['task_count'],
'submission_count': data['submission_count'],
'note_count': data['note_count'],
'last_activity_date': data['last_activity_date'].isoformat() if data['last_activity_date'] else None
})
return affected_users
def _get_last_activity_date(self, shot_id: int, task_ids: List[int], db: Session) -> Optional[str]:
"""
Get the most recent activity date for the shot and its tasks.
Args:
shot_id: ID of the shot
task_ids: List of task IDs
db: Database session
Returns:
ISO formatted date string of last activity, or None
"""
try:
# Get the most recent activity from various sources
dates = []
# Shot updated_at
shot = db.query(Shot.updated_at).filter(Shot.id == shot_id).first()
if shot and shot.updated_at:
dates.append(shot.updated_at)
if task_ids:
# Task updated_at
task_dates = db.query(func.max(Task.updated_at)).filter(
Task.id.in_(task_ids)
).scalar()
if task_dates:
dates.append(task_dates)
# Submission submitted_at
submission_dates = db.query(func.max(Submission.submitted_at)).filter(
Submission.task_id.in_(task_ids)
).scalar()
if submission_dates:
dates.append(submission_dates)
# Production note updated_at
note_dates = db.query(func.max(ProductionNote.updated_at)).filter(
ProductionNote.task_id.in_(task_ids)
).scalar()
if note_dates:
dates.append(note_dates)
if dates:
return max(dates).isoformat()
return None
except SQLAlchemyError:
return None
def _log_shot_deletion(self, shot: Shot, current_user: User, deletion_info: Optional[DeletionInfo], db: Session) -> None:
"""
Log the shot deletion activity.
Args:
shot: The shot that was deleted
current_user: User who performed the deletion
deletion_info: Information about what was deleted
db: Database session
"""
try:
# Create activity record
activity = Activity(
type=ActivityType.SHOT_UPDATED, # We'll use SHOT_UPDATED for now, could add SHOT_DELETED later
user_id=current_user.id,
project_id=shot.project_id,
shot_id=shot.id,
description=f"Shot '{shot.name}' was deleted by {current_user.first_name} {current_user.last_name}",
activity_metadata={
'action': 'soft_delete',
'shot_name': shot.name,
'episode_name': shot.episode.name if shot.episode else None,
'project_id': shot.project_id,
'project_name': shot.project.name if shot.project else None,
'deleted_counts': {
'tasks': deletion_info.task_count if deletion_info else 0,
'submissions': deletion_info.submission_count if deletion_info else 0,
'attachments': deletion_info.attachment_count if deletion_info else 0,
'notes': deletion_info.note_count if deletion_info else 0,
'reviews': deletion_info.review_count if deletion_info else 0
},
'affected_users_count': len(deletion_info.affected_users) if deletion_info else 0
}
)
db.add(activity)
db.flush()
except SQLAlchemyError as e:
# Don't fail the deletion if logging fails, just add a warning
pass