Understanding migration dependencies and establishing proper workflows is crucial for managing complex Django projects with multiple apps and developers. This section covers dependency management, conflict resolution, and best practices for team collaboration.
# Migration dependencies define the order in which migrations must be applied
# There are several types of dependencies:
class Migration(migrations.Migration):
"""Example migration showing different dependency types"""
dependencies = [
# 1. Sequential dependency within same app
('blog', '0001_initial'),
# 2. Cross-app dependency
('auth', '0012_alter_user_first_name_max_length'),
# 3. Dependency on Django's built-in migrations
('contenttypes', '0002_remove_content_type_name'),
# 4. Dependency on third-party app migrations
('taggit', '0003_taggeditem_add_unique_index'),
]
operations = [
migrations.CreateModel(
name='Post',
fields=[
('id', models.AutoField(primary_key=True)),
('title', models.CharField(max_length=200)),
('author', models.ForeignKey('auth.User', on_delete=models.CASCADE)),
('content_type', models.ForeignKey('contenttypes.ContentType', on_delete=models.CASCADE)),
],
),
]
# Dependency analysis tools
class DependencyAnalyzer:
"""Analyze migration dependencies"""
def __init__(self):
from django.db.migrations.loader import MigrationLoader
self.loader = MigrationLoader(connection)
self.graph = self.loader.graph
def get_dependency_chain(self, app_label, migration_name):
"""Get complete dependency chain for a migration"""
migration_key = (app_label, migration_name)
if migration_key not in self.graph.nodes:
return None
# Get all dependencies recursively
dependencies = []
visited = set()
def collect_dependencies(key):
if key in visited:
return
visited.add(key)
migration = self.graph.nodes[key]
for dep_key in migration.dependencies:
if dep_key in self.graph.nodes:
dependencies.append(dep_key)
collect_dependencies(dep_key)
collect_dependencies(migration_key)
return dependencies
def find_circular_dependencies(self):
"""Find circular dependencies in migration graph"""
# Use topological sort to detect cycles
visited = set()
rec_stack = set()
cycles = []
def has_cycle(node):
visited.add(node)
rec_stack.add(node)
migration = self.graph.nodes[node]
for dep_node in migration.dependencies:
if dep_node not in self.graph.nodes:
continue
if dep_node not in visited:
if has_cycle(dep_node):
return True
elif dep_node in rec_stack:
cycles.append((node, dep_node))
return True
rec_stack.remove(node)
return False
for node in self.graph.nodes:
if node not in visited:
has_cycle(node)
return cycles
def get_migration_order(self, app_label=None):
"""Get correct order for applying migrations"""
# Get topological sort of migrations
from django.db.migrations.executor import MigrationExecutor
executor = MigrationExecutor(connection)
if app_label:
# Get leaf nodes for specific app
targets = [(app_label, None)]
else:
# Get all leaf nodes
targets = executor.loader.graph.leaf_nodes()
plan = executor.migration_plan(targets)
return [
{
'app': migration.app_label,
'name': migration.name,
'backwards': backwards,
'dependencies': migration.dependencies,
}
for migration, backwards in plan
]
def validate_dependencies(self):
"""Validate all migration dependencies"""
validation_errors = []
for migration_key in self.graph.nodes:
app_label, migration_name = migration_key
migration = self.graph.nodes[migration_key]
for dep_key in migration.dependencies:
dep_app, dep_migration = dep_key
# Check if dependency exists
if dep_key not in self.graph.nodes:
validation_errors.append({
'migration': f"{app_label}.{migration_name}",
'error': f"Missing dependency: {dep_app}.{dep_migration}",
'type': 'missing_dependency'
})
# Check for self-dependency
if dep_key == migration_key:
validation_errors.append({
'migration': f"{app_label}.{migration_name}",
'error': "Self-dependency detected",
'type': 'self_dependency'
})
return validation_errors
# Cross-app dependency management
class CrossAppDependencyManager:
"""Manage dependencies between different apps"""
@staticmethod
def create_cross_app_migration(source_app, target_app, target_migration):
"""Create migration with cross-app dependency"""
from django.core.management import call_command
from django.db.migrations.writer import MigrationWriter
import os
# Create empty migration
call_command('makemigrations', source_app, '--empty',
name=f'depend_on_{target_app}_{target_migration}')
# Find the created migration file
migrations_dir = os.path.join(
apps.get_app_config(source_app).path, 'migrations'
)
migration_files = [
f for f in os.listdir(migrations_dir)
if f.startswith('0') and f.endswith('.py')
]
latest_migration = max(migration_files)
migration_path = os.path.join(migrations_dir, latest_migration)
# Read and modify the migration
with open(migration_path, 'r') as f:
content = f.read()
# Add the cross-app dependency
dependency_line = f" ('{target_app}', '{target_migration}'),"
# Insert dependency
content = content.replace(
'dependencies = [',
f'dependencies = [\n{dependency_line}'
)
with open(migration_path, 'w') as f:
f.write(content)
return migration_path
@staticmethod
def analyze_cross_app_dependencies():
"""Analyze cross-app dependencies in the project"""
from django.db.migrations.loader import MigrationLoader
loader = MigrationLoader(connection)
cross_app_deps = {}
for migration_key in loader.graph.nodes:
app_label, migration_name = migration_key
migration = loader.graph.nodes[migration_key]
for dep_app, dep_migration in migration.dependencies:
if dep_app != app_label: # Cross-app dependency
if app_label not in cross_app_deps:
cross_app_deps[app_label] = {}
if dep_app not in cross_app_deps[app_label]:
cross_app_deps[app_label][dep_app] = []
cross_app_deps[app_label][dep_app].append({
'migration': migration_name,
'depends_on': dep_migration
})
return cross_app_deps
@staticmethod
def suggest_dependency_optimization():
"""Suggest optimizations for cross-app dependencies"""
cross_deps = CrossAppDependencyManager.analyze_cross_app_dependencies()
suggestions = []
for app_label, dependencies in cross_deps.items():
for dep_app, dep_list in dependencies.items():
if len(dep_list) > 3:
suggestions.append({
'type': 'consolidate_dependencies',
'app': app_label,
'target_app': dep_app,
'count': len(dep_list),
'suggestion': f"Consider consolidating {len(dep_list)} dependencies from {app_label} to {dep_app}"
})
return suggestions
class DependencyResolver:
"""Resolve complex migration dependencies"""
def __init__(self):
from django.db.migrations.loader import MigrationLoader
self.loader = MigrationLoader(connection)
def resolve_conflicts(self, app_label=None):
"""Resolve migration conflicts"""
conflicts = self.loader.detect_conflicts()
if not conflicts:
return {"status": "no_conflicts"}
resolution_plan = {}
for conflicted_app, conflict_migrations in conflicts.items():
if app_label and conflicted_app != app_label:
continue
resolution_plan[conflicted_app] = {
'conflicts': conflict_migrations,
'resolution_strategy': self._determine_resolution_strategy(
conflicted_app, conflict_migrations
)
}
return resolution_plan
def _determine_resolution_strategy(self, app_label, conflict_migrations):
"""Determine best strategy for resolving conflicts"""
strategies = []
# Strategy 1: Automatic merge
if len(conflict_migrations) == 2:
strategies.append({
'type': 'automatic_merge',
'command': f'python manage.py makemigrations {app_label} --merge',
'description': 'Automatically merge conflicting migrations'
})
# Strategy 2: Manual resolution
strategies.append({
'type': 'manual_resolution',
'steps': [
'Review conflicting migrations manually',
'Determine correct merge order',
'Create custom merge migration if needed'
],
'description': 'Manually resolve conflicts'
})
# Strategy 3: Rollback and reapply
strategies.append({
'type': 'rollback_reapply',
'steps': [
f'python manage.py migrate {app_label} <common_ancestor>',
'Resolve conflicts in code',
f'python manage.py makemigrations {app_label}',
f'python manage.py migrate {app_label}'
],
'description': 'Rollback to common ancestor and reapply'
})
return strategies
def create_merge_migration(self, app_label, migration1, migration2):
"""Create a merge migration for conflicting migrations"""
from django.core.management import call_command
from django.db.migrations.writer import MigrationWriter
import os
# Create merge migration
call_command('makemigrations', app_label, '--merge')
# Find the created merge migration
migrations_dir = os.path.join(
apps.get_app_config(app_label).path, 'migrations'
)
migration_files = [
f for f in os.listdir(migrations_dir)
if 'merge' in f and f.endswith('.py')
]
if migration_files:
latest_merge = max(migration_files)
return os.path.join(migrations_dir, latest_merge)
return None
def validate_merge_safety(self, app_label, migration1, migration2):
"""Validate that merging two migrations is safe"""
# Load migrations
migration1_obj = self.loader.get_migration(app_label, migration1)
migration2_obj = self.loader.get_migration(app_label, migration2)
conflicts = []
# Check for conflicting operations
for op1 in migration1_obj.operations:
for op2 in migration2_obj.operations:
conflict = self._check_operation_conflict(op1, op2)
if conflict:
conflicts.append(conflict)
return {
'safe_to_merge': len(conflicts) == 0,
'conflicts': conflicts,
'recommendations': self._get_merge_recommendations(conflicts)
}
def _check_operation_conflict(self, op1, op2):
"""Check if two operations conflict"""
op1_name = op1.__class__.__name__
op2_name = op2.__class__.__name__
# Check for field conflicts
if (op1_name in ['AddField', 'AlterField', 'RemoveField'] and
op2_name in ['AddField', 'AlterField', 'RemoveField']):
if (hasattr(op1, 'model_name') and hasattr(op2, 'model_name') and
hasattr(op1, 'name') and hasattr(op2, 'name')):
if (op1.model_name == op2.model_name and
op1.name == op2.name):
return {
'type': 'field_conflict',
'model': op1.model_name,
'field': op1.name,
'operation1': op1_name,
'operation2': op2_name
}
# Check for model conflicts
if (op1_name in ['CreateModel', 'DeleteModel', 'AlterModelOptions'] and
op2_name in ['CreateModel', 'DeleteModel', 'AlterModelOptions']):
if (hasattr(op1, 'name') and hasattr(op2, 'name') and
op1.name == op2.name):
return {
'type': 'model_conflict',
'model': op1.name,
'operation1': op1_name,
'operation2': op2_name
}
return None
def _get_merge_recommendations(self, conflicts):
"""Get recommendations for resolving merge conflicts"""
recommendations = []
for conflict in conflicts:
if conflict['type'] == 'field_conflict':
recommendations.append(
f"Manually resolve field '{conflict['field']}' "
f"conflict in model '{conflict['model']}'"
)
elif conflict['type'] == 'model_conflict':
recommendations.append(
f"Manually resolve model '{conflict['model']}' conflict"
)
if not conflicts:
recommendations.append("Migrations can be safely merged automatically")
return recommendations
# Workflow automation
class MigrationWorkflow:
"""Automate common migration workflows"""
@staticmethod
def pre_migration_checks():
"""Run pre-migration safety checks"""
checks = {
'backup_recommended': False,
'conflicts_detected': False,
'destructive_operations': False,
'large_table_operations': False,
'recommendations': []
}
# Check for conflicts
from django.db.migrations.loader import MigrationLoader
loader = MigrationLoader(connection)
conflicts = loader.detect_conflicts()
if conflicts:
checks['conflicts_detected'] = True
checks['recommendations'].append(
"Resolve migration conflicts before proceeding"
)
# Check for destructive operations
from django.db.migrations.executor import MigrationExecutor
executor = MigrationExecutor(connection)
plan = executor.migration_plan(executor.loader.graph.leaf_nodes())
destructive_ops = ['RemoveField', 'DeleteModel', 'AlterField']
for migration, backwards in plan:
if backwards:
checks['destructive_operations'] = True
checks['backup_recommended'] = True
break
for operation in migration.operations:
if operation.__class__.__name__ in destructive_ops:
checks['destructive_operations'] = True
checks['backup_recommended'] = True
break
# Add recommendations
if checks['backup_recommended']:
checks['recommendations'].append(
"Create database backup before applying migrations"
)
if checks['destructive_operations']:
checks['recommendations'].append(
"Review destructive operations carefully"
)
return checks
@staticmethod
def post_migration_validation():
"""Validate database state after migrations"""
validation_results = {
'schema_valid': True,
'data_integrity_ok': True,
'issues': []
}
try:
# Run Django's system checks
from django.core.management import call_command
from io import StringIO
output = StringIO()
call_command('check', stdout=output, stderr=output)
check_output = output.getvalue()
if 'ERROR' in check_output or 'CRITICAL' in check_output:
validation_results['schema_valid'] = False
validation_results['issues'].append(
f"System check errors: {check_output}"
)
except Exception as e:
validation_results['schema_valid'] = False
validation_results['issues'].append(f"System check failed: {e}")
# Check for orphaned records (simplified example)
try:
from django.db import connection
with connection.cursor() as cursor:
# Check for foreign key constraint violations
# This is database-specific and simplified
if connection.vendor == 'postgresql':
cursor.execute("""
SELECT conname, conrelid::regclass
FROM pg_constraint
WHERE contype = 'f'
AND NOT EXISTS (
SELECT 1 FROM information_schema.table_constraints
WHERE constraint_name = conname
)
""")
violations = cursor.fetchall()
if violations:
validation_results['data_integrity_ok'] = False
validation_results['issues'].extend([
f"Foreign key constraint violation: {v[0]} on {v[1]}"
for v in violations
])
except Exception as e:
validation_results['issues'].append(
f"Data integrity check failed: {e}"
)
return validation_results
@staticmethod
def create_migration_plan(target_apps=None):
"""Create comprehensive migration plan"""
from django.db.migrations.executor import MigrationExecutor
executor = MigrationExecutor(connection)
if target_apps:
targets = [(app, None) for app in target_apps]
else:
targets = executor.loader.graph.leaf_nodes()
plan = executor.migration_plan(targets)
migration_plan = {
'total_migrations': len(plan),
'estimated_time': 'Unknown',
'phases': [],
'risks': [],
'recommendations': []
}
# Group migrations by phase
current_phase = []
phase_number = 1
for migration, backwards in plan:
migration_info = {
'app': migration.app_label,
'name': migration.name,
'backwards': backwards,
'operations': [
op.__class__.__name__ for op in migration.operations
],
'estimated_duration': 'Unknown'
}
current_phase.append(migration_info)
# Start new phase for cross-app dependencies
if len(current_phase) >= 5: # Arbitrary phase size
migration_plan['phases'].append({
'phase': phase_number,
'migrations': current_phase.copy()
})
current_phase = []
phase_number += 1
# Add remaining migrations
if current_phase:
migration_plan['phases'].append({
'phase': phase_number,
'migrations': current_phase
})
# Analyze risks
for migration, backwards in plan:
if backwards:
migration_plan['risks'].append(
f"Rollback operation in {migration.app_label}.{migration.name}"
)
for operation in migration.operations:
op_name = operation.__class__.__name__
if op_name in ['RemoveField', 'DeleteModel']:
migration_plan['risks'].append(
f"Data loss risk in {migration.app_label}.{migration.name}: {op_name}"
)
elif op_name == 'RunSQL':
migration_plan['risks'].append(
f"Custom SQL in {migration.app_label}.{migration.name}"
)
# Add recommendations
if migration_plan['risks']:
migration_plan['recommendations'].append(
"Create database backup before proceeding"
)
if migration_plan['total_migrations'] > 20:
migration_plan['recommendations'].append(
"Consider running migrations in batches"
)
return migration_plan
class BranchMigrationWorkflow:
"""Manage migrations across different Git branches"""
@staticmethod
def check_branch_migration_conflicts():
"""Check for migration conflicts between branches"""
import subprocess
import json
try:
# Get current branch
current_branch = subprocess.check_output(
['git', 'rev-parse', '--abbrev-ref', 'HEAD'],
text=True
).strip()
# Get main branch (usually 'main' or 'master')
try:
main_branch = 'main'
subprocess.check_output(['git', 'rev-parse', main_branch])
except subprocess.CalledProcessError:
main_branch = 'master'
# Get migration files in current branch
current_migrations = BranchMigrationWorkflow._get_migration_files()
# Get migration files in main branch
subprocess.run(['git', 'checkout', main_branch], check=True)
main_migrations = BranchMigrationWorkflow._get_migration_files()
# Return to current branch
subprocess.run(['git', 'checkout', current_branch], check=True)
# Compare migrations
conflicts = BranchMigrationWorkflow._compare_migrations(
current_migrations, main_migrations
)
return {
'current_branch': current_branch,
'main_branch': main_branch,
'conflicts': conflicts,
'resolution_needed': len(conflicts) > 0
}
except subprocess.CalledProcessError as e:
return {'error': f"Git operation failed: {e}"}
@staticmethod
def _get_migration_files():
"""Get all migration files in current state"""
import os
from django.apps import apps
migrations = {}
for app_config in apps.get_app_configs():
migrations_dir = os.path.join(app_config.path, 'migrations')
if os.path.exists(migrations_dir):
migration_files = [
f for f in os.listdir(migrations_dir)
if f.endswith('.py') and not f.startswith('__')
]
migrations[app_config.label] = sorted(migration_files)
return migrations
@staticmethod
def _compare_migrations(current_migrations, main_migrations):
"""Compare migration files between branches"""
conflicts = []
for app_label in current_migrations:
current_files = set(current_migrations[app_label])
main_files = set(main_migrations.get(app_label, []))
# Find conflicting migration numbers
current_numbers = {
f.split('_')[0] for f in current_files
if f.split('_')[0].isdigit()
}
main_numbers = {
f.split('_')[0] for f in main_files
if f.split('_')[0].isdigit()
}
# Check for number conflicts
conflicting_numbers = current_numbers & main_numbers
for number in conflicting_numbers:
current_file = next(
f for f in current_files
if f.startswith(number + '_')
)
main_file = next(
f for f in main_files
if f.startswith(number + '_')
)
if current_file != main_file:
conflicts.append({
'app': app_label,
'number': number,
'current_branch_file': current_file,
'main_branch_file': main_file,
'type': 'migration_number_conflict'
})
return conflicts
@staticmethod
def resolve_branch_conflicts(conflicts):
"""Suggest resolutions for branch conflicts"""
resolutions = []
for conflict in conflicts:
if conflict['type'] == 'migration_number_conflict':
resolutions.append({
'conflict': conflict,
'strategies': [
{
'name': 'Renumber migration',
'description': 'Renumber the conflicting migration to next available number',
'steps': [
f"Rename {conflict['current_branch_file']} to use next available number",
"Update any references to the old migration name",
"Test the migration"
]
},
{
'name': 'Merge migrations',
'description': 'Combine operations from both migrations',
'steps': [
"Review operations in both migrations",
"Create new migration combining operations",
"Remove conflicting migrations",
"Test the merged migration"
]
},
{
'name': 'Rebase branch',
'description': 'Rebase feature branch on latest main',
'steps': [
"git rebase main",
"Resolve migration conflicts during rebase",
"python manage.py makemigrations --merge if needed"
]
}
]
})
return resolutions
# Continuous Integration workflow
class CIMigrationWorkflow:
"""Migration workflow for CI/CD pipelines"""
@staticmethod
def validate_migrations_for_ci():
"""Validate migrations for CI environment"""
validation_results = {
'valid': True,
'errors': [],
'warnings': [],
'recommendations': []
}
# Check for unapplied migrations
from django.db.migrations.executor import MigrationExecutor
executor = MigrationExecutor(connection)
plan = executor.migration_plan(executor.loader.graph.leaf_nodes())
if plan:
validation_results['warnings'].append(
f"{len(plan)} unapplied migrations detected"
)
# Check for migration conflicts
conflicts = executor.loader.detect_conflicts()
if conflicts:
validation_results['valid'] = False
validation_results['errors'].append(
f"Migration conflicts detected: {conflicts}"
)
# Check for dangerous operations in CI
dangerous_operations = ['RunSQL', 'RunPython']
for migration, backwards in plan:
for operation in migration.operations:
if operation.__class__.__name__ in dangerous_operations:
validation_results['warnings'].append(
f"Potentially dangerous operation in "
f"{migration.app_label}.{migration.name}: "
f"{operation.__class__.__name__}"
)
# Add recommendations
if validation_results['warnings']:
validation_results['recommendations'].append(
"Review warnings before deploying to production"
)
return validation_results
@staticmethod
def generate_migration_report():
"""Generate migration report for CI/CD"""
from django.db.migrations.executor import MigrationExecutor
from django.db.migrations.loader import MigrationLoader
loader = MigrationLoader(connection)
executor = MigrationExecutor(connection)
# Get migration plan
plan = executor.migration_plan(executor.loader.graph.leaf_nodes())
report = {
'timestamp': timezone.now().isoformat(),
'database': connection.settings_dict['NAME'],
'total_migrations_to_apply': len(plan),
'migrations_by_app': {},
'potential_issues': [],
'estimated_impact': 'low'
}
# Group migrations by app
for migration, backwards in plan:
app_label = migration.app_label
if app_label not in report['migrations_by_app']:
report['migrations_by_app'][app_label] = {
'count': 0,
'migrations': []
}
report['migrations_by_app'][app_label]['count'] += 1
report['migrations_by_app'][app_label]['migrations'].append({
'name': migration.name,
'backwards': backwards,
'operations': [op.__class__.__name__ for op in migration.operations]
})
# Check for potential issues
if backwards:
report['potential_issues'].append(
f"Rollback operation: {app_label}.{migration.name}"
)
report['estimated_impact'] = 'high'
for operation in migration.operations:
op_name = operation.__class__.__name__
if op_name in ['RemoveField', 'DeleteModel']:
report['potential_issues'].append(
f"Destructive operation: {app_label}.{migration.name} - {op_name}"
)
report['estimated_impact'] = 'high'
elif op_name in ['AddField', 'AlterField'] and report['estimated_impact'] == 'low':
report['estimated_impact'] = 'medium'
return report
@staticmethod
def create_ci_migration_script():
"""Create script for running migrations in CI"""
script_content = '''#!/bin/bash
set -e
echo "Starting migration process..."
# Pre-migration checks
echo "Running pre-migration checks..."
python manage.py check --deploy
# Check for migration conflicts
echo "Checking for migration conflicts..."
python manage.py showmigrations --plan | grep -q "\\[ \\]" || {
echo "No pending migrations found"
exit 0
}
# Create backup (if in production-like environment)
if [ "$ENVIRONMENT" = "production" ] || [ "$ENVIRONMENT" = "staging" ]; then
echo "Creating database backup..."
# Add backup command here
fi
# Apply migrations
echo "Applying migrations..."
python manage.py migrate --verbosity=2
# Post-migration validation
echo "Running post-migration validation..."
python manage.py check
echo "Migration process completed successfully!"
'''
return script_content
Understanding migration dependencies and establishing proper workflows ensures smooth database evolution in team environments. Proper dependency management, conflict resolution, and automated workflows are essential for maintaining data integrity and preventing deployment issues.
Management Commands
Django provides a comprehensive set of management commands for working with migrations. Understanding these commands and their options enables effective migration management across development, testing, and production environments.
Transaction Handling
Django migrations run within database transactions by default, providing atomicity and consistency during schema changes. Understanding transaction behavior in migrations is crucial for maintaining data integrity and handling complex migration scenarios safely.