first commit
This commit is contained in:
brusnitsyn
138
app/utils/email_sender.py
Normal file
138
app/utils/email_sender.py
Normal file
@@ -0,0 +1,138 @@
|
||||
import smtplib
|
||||
from email.mime.text import MIMEText
|
||||
from email.mime.multipart import MIMEMultipart
|
||||
from email.mime.application import MIMEApplication
|
||||
from typing import List, Optional
|
||||
from datetime import datetime
|
||||
from app.core.config import settings
|
||||
from app.core.logging import migration_logger
|
||||
|
||||
|
||||
class EmailSender:
|
||||
"""Класс для отправки email уведомлений"""
|
||||
|
||||
def __init__(self):
|
||||
self.smtp_server = settings.EMAIL_HOST
|
||||
self.smtp_port = settings.EMAIL_PORT
|
||||
self.username = settings.EMAIL_USER
|
||||
self.password = settings.EMAIL_PASSWORD
|
||||
self.from_addr = settings.EMAIL_FROM
|
||||
self.to_addrs = settings.EMAIL_TO
|
||||
|
||||
def send_email(self, subject: str, body: str, attachments: Optional[List[str]] = None) -> bool:
|
||||
"""Отправка email с вложениями"""
|
||||
if not all([self.smtp_server, self.username, self.password, self.from_addr]):
|
||||
migration_logger.warning("Настройки email не заполнены. Отправка email пропущена.")
|
||||
return False
|
||||
|
||||
try:
|
||||
# Создаем сообщение
|
||||
msg = MIMEMultipart()
|
||||
msg['From'] = self.from_addr
|
||||
msg['To'] = ', '.join(self.to_addrs)
|
||||
msg['Subject'] = subject
|
||||
msg['Date'] = datetime.now().strftime('%a, %d %b %Y %H:%M:%S %z')
|
||||
|
||||
# Добавляем текст
|
||||
msg.attach(MIMEText(body, 'plain', 'utf-8'))
|
||||
|
||||
# Добавляем вложения
|
||||
if attachments:
|
||||
for file_path in attachments:
|
||||
try:
|
||||
with open(file_path, 'rb') as f:
|
||||
part = MIMEApplication(f.read(), Name=file_path.split('/')[-1])
|
||||
part['Content-Disposition'] = f'attachment; filename="{file_path.split("/")[-1]}"'
|
||||
msg.attach(part)
|
||||
except Exception as e:
|
||||
migration_logger.error(f"Ошибка при добавлении вложения {file_path}: {e}")
|
||||
|
||||
# Отправляем
|
||||
with smtplib.SMTP_SSL(self.smtp_server, self.smtp_port) as server:
|
||||
server.login(self.username, self.password)
|
||||
server.send_message(msg)
|
||||
|
||||
migration_logger.info(f"Email успешно отправлен на {', '.join(self.to_addrs)}")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
migration_logger.error(f"Ошибка при отправке email: {e}")
|
||||
return False
|
||||
|
||||
def send_error_notification(self, error_message: str, traceback_str: str = None, table_name: str = None):
|
||||
"""Отправить уведомление об ошибке"""
|
||||
subject = f"ОШИБКА МИГРАЦИИ - {datetime.now().strftime('%Y-%m-%d %H:%M')}"
|
||||
|
||||
body = f"""
|
||||
🚨 ОШИБКА В ПРОЦЕССЕ МИГРАЦИИ ДАННЫХ
|
||||
{'='*60}
|
||||
|
||||
Время: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}
|
||||
Сервис: Migration Service
|
||||
|
||||
"""
|
||||
if table_name:
|
||||
body += f"Таблица: {table_name}\n"
|
||||
|
||||
body += f"""
|
||||
Ошибка:
|
||||
{error_message}
|
||||
|
||||
"""
|
||||
if traceback_str:
|
||||
body += f"""
|
||||
📎 Детали:
|
||||
{traceback_str}
|
||||
|
||||
"""
|
||||
|
||||
body += """
|
||||
🔧 Необходимо проверить логи и устранить проблему.
|
||||
"""
|
||||
|
||||
return self.send_email(subject, body)
|
||||
|
||||
def send_success_notification(self, stats: dict, duration: float):
|
||||
"""Отправить уведомление об успешной миграции"""
|
||||
subject = f"УСПЕШНАЯ МИГРАЦИЯ - {datetime.now().strftime('%Y-%m-%d %H:%M')}"
|
||||
|
||||
body = f"""
|
||||
МИГРАЦИЯ ДАННЫХ УСПЕШНО ЗАВЕРШЕНА
|
||||
{'='*60}
|
||||
|
||||
Время: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}
|
||||
Длительность: {duration:.1f} сек
|
||||
|
||||
СТАТИСТИКА:
|
||||
{'='*40}
|
||||
Всего строк в БД: {stats.get('total_rows', 0)}
|
||||
Таблиц обработано: {stats.get('total_tables', 0)}
|
||||
Последняя репликация: {stats.get('last_replication', 'Нет данных')}
|
||||
|
||||
Миграция выполнена успешно!
|
||||
"""
|
||||
|
||||
return self.send_email(subject, body)
|
||||
|
||||
def send_start_notification(self, tables: List[str], full_reload: bool):
|
||||
"""Отправить уведомление о начале миграции"""
|
||||
subject = f"НАЧАЛО МИГРАЦИИ - {datetime.now().strftime('%Y-%m-%d %H:%M')}"
|
||||
|
||||
body = f"""
|
||||
НАЧАЛО МИГРАЦИИ ДАННЫХ
|
||||
{'='*60}
|
||||
|
||||
Время старта: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}
|
||||
Режим: {'ПОЛНАЯ ПЕРЕЗАГРУЗКА' if full_reload else 'ИНКРЕМЕНТАЛЬНАЯ'}
|
||||
Таблиц для обработки: {len(tables)}
|
||||
|
||||
СПИСОК ТАБЛИЦ:
|
||||
{chr(10).join([f' • {t}' for t in tables])}
|
||||
|
||||
Миграция запущена...
|
||||
"""
|
||||
|
||||
return self.send_email(subject, body)
|
||||
|
||||
|
||||
email_sender = EmailSender()
|
||||
153
app/utils/index_helpers.py
Normal file
153
app/utils/index_helpers.py
Normal file
@@ -0,0 +1,153 @@
|
||||
from typing import Optional, List, Dict, Any
|
||||
import pandas as pd
|
||||
from app.core.database import db_connector
|
||||
from app.core.logging import migration_logger
|
||||
|
||||
|
||||
def get_primary_key(table_name: str) -> Optional[str]:
|
||||
"""Получить колонку первичного ключа"""
|
||||
try:
|
||||
query = f"""
|
||||
SELECT TOP 1
|
||||
c.name as column_name
|
||||
FROM sys.indexes i
|
||||
INNER JOIN sys.index_columns ic ON i.object_id = ic.object_id AND i.index_id = ic.index_id
|
||||
INNER JOIN sys.columns c ON ic.object_id = c.object_id AND ic.column_id = c.column_id
|
||||
WHERE i.object_id = OBJECT_ID('{table_name}')
|
||||
AND i.is_primary_key = 1
|
||||
"""
|
||||
|
||||
pk_df = pd.read_sql_query(query, db_connector.src_engine)
|
||||
|
||||
if not pk_df.empty:
|
||||
pk_column = pk_df.iloc[0]['column_name']
|
||||
migration_logger.info(f"PRIMARY KEY для {table_name}: {pk_column}")
|
||||
return pk_column
|
||||
|
||||
migration_logger.warning(f"В {table_name} нет PRIMARY KEY, ищем ID колонку...")
|
||||
|
||||
columns_query = f"""
|
||||
SELECT COLUMN_NAME
|
||||
FROM INFORMATION_SCHEMA.COLUMNS
|
||||
WHERE TABLE_NAME = '{table_name}'
|
||||
"""
|
||||
|
||||
columns_df = pd.read_sql_query(columns_query, db_connector.src_engine)
|
||||
|
||||
id_keywords = ['ID', 'Id', 'id', 'Code', 'KEY']
|
||||
for col in columns_df['COLUMN_NAME']:
|
||||
for keyword in id_keywords:
|
||||
if keyword in col:
|
||||
migration_logger.info(f"ID колонка для {table_name}: {col}")
|
||||
return col
|
||||
|
||||
if not columns_df.empty:
|
||||
first_col = columns_df.iloc[0]['COLUMN_NAME']
|
||||
migration_logger.warning(f"Используем первую колонку для {table_name}: {first_col}")
|
||||
return first_col
|
||||
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
migration_logger.error(f"Ошибка поиска PK для {table_name}: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def get_foreign_keys(table_name: str) -> List[Dict[str, str]]:
|
||||
"""Получить внешние ключи без проверки связанных таблиц"""
|
||||
try:
|
||||
query = f"""
|
||||
SELECT
|
||||
fk.name as fk_name,
|
||||
pc.name as parent_column,
|
||||
rc.name as referenced_column,
|
||||
rt.name as referenced_table
|
||||
FROM sys.foreign_keys fk
|
||||
INNER JOIN sys.foreign_key_columns fkc ON fk.object_id = fkc.constraint_object_id
|
||||
INNER JOIN sys.columns pc ON fkc.parent_object_id = pc.object_id AND fkc.parent_column_id = pc.column_id
|
||||
INNER JOIN sys.columns rc ON fkc.referenced_object_id = rc.object_id AND fkc.referenced_column_id = rc.column_id
|
||||
INNER JOIN sys.tables rt ON fkc.referenced_object_id = rt.object_id
|
||||
WHERE fk.parent_object_id = OBJECT_ID('{table_name}')
|
||||
ORDER BY fk.name, fkc.constraint_column_id
|
||||
"""
|
||||
|
||||
fk_df = pd.read_sql_query(query, db_connector.src_engine)
|
||||
|
||||
if fk_df.empty:
|
||||
return []
|
||||
|
||||
foreign_keys = {}
|
||||
for _, row in fk_df.iterrows():
|
||||
fk_name = row['fk_name']
|
||||
if fk_name not in foreign_keys:
|
||||
foreign_keys[fk_name] = {
|
||||
'name': fk_name,
|
||||
'parent_column': row['parent_column'],
|
||||
'referenced_table': row['referenced_table'],
|
||||
'referenced_column': row['referenced_column']
|
||||
}
|
||||
|
||||
result = list(foreign_keys.values())
|
||||
migration_logger.info(f"Найдено {len(result)} внешних ключей для {table_name}")
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
migration_logger.error(f"Ошибка получения внешних ключей для {table_name}: {e}")
|
||||
return []
|
||||
|
||||
|
||||
def get_indexes(table_name: str) -> List[Dict[str, Any]]:
|
||||
"""Получить индексы таблицы"""
|
||||
try:
|
||||
query = f"""
|
||||
SELECT
|
||||
i.name as index_name,
|
||||
i.is_unique,
|
||||
c.name as column_name
|
||||
FROM sys.indexes i
|
||||
INNER JOIN sys.index_columns ic ON i.object_id = ic.object_id AND i.index_id = ic.index_id
|
||||
INNER JOIN sys.columns c ON ic.object_id = c.object_id AND ic.column_id = c.column_id
|
||||
WHERE i.object_id = OBJECT_ID('{table_name}')
|
||||
AND i.is_primary_key = 0
|
||||
ORDER BY i.name, ic.key_ordinal
|
||||
"""
|
||||
|
||||
idx_df = pd.read_sql_query(query, db_connector.src_engine)
|
||||
|
||||
if idx_df.empty:
|
||||
return []
|
||||
|
||||
indexes = {}
|
||||
for _, row in idx_df.iterrows():
|
||||
idx_name = row['index_name']
|
||||
if idx_name not in indexes:
|
||||
indexes[idx_name] = {
|
||||
'name': idx_name,
|
||||
'unique': bool(row['is_unique']),
|
||||
'columns': []
|
||||
}
|
||||
indexes[idx_name]['columns'].append(row['column_name'])
|
||||
|
||||
result = list(indexes.values())
|
||||
migration_logger.info(f"Найдено {len(result)} индексов для {table_name}")
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
migration_logger.error(f"Ошибка получения индексов для {table_name}: {e}")
|
||||
return []
|
||||
|
||||
|
||||
def get_max_id_from_postgres(table_name: str, id_column: str) -> Optional[int]:
|
||||
"""Получить максимальный ID из PostgreSQL"""
|
||||
try:
|
||||
query = f'SELECT MAX("{id_column}") as max_id FROM "{table_name.lower()}"'
|
||||
df = pd.read_sql_query(query, db_connector.dst_engine)
|
||||
|
||||
if not df.empty and df.iloc[0]['max_id'] is not None:
|
||||
return int(df.iloc[0]['max_id'])
|
||||
|
||||
return 0
|
||||
|
||||
except Exception as e:
|
||||
migration_logger.error(f"Ошибка получения max ID из PG для {table_name}: {e}")
|
||||
return 0
|
||||
45
app/utils/type_mapper.py
Normal file
45
app/utils/type_mapper.py
Normal file
@@ -0,0 +1,45 @@
|
||||
from typing import Optional
|
||||
|
||||
|
||||
def mssql_to_postgres_type(mssql_type: str, max_length: Optional[int] = None) -> str:
|
||||
"""Преобразование типа MSSQL в PostgreSQL"""
|
||||
mssql_type = mssql_type.lower()
|
||||
|
||||
type_map = {
|
||||
'int': 'INTEGER',
|
||||
'bigint': 'BIGINT',
|
||||
'smallint': 'SMALLINT',
|
||||
'tinyint': 'SMALLINT',
|
||||
'bit': 'BOOLEAN',
|
||||
'float': 'DOUBLE PRECISION',
|
||||
'real': 'REAL',
|
||||
'decimal': 'NUMERIC',
|
||||
'numeric': 'NUMERIC',
|
||||
'money': 'NUMERIC(19,4)',
|
||||
'smallmoney': 'NUMERIC(10,4)',
|
||||
'datetime': 'TIMESTAMP',
|
||||
'datetime2': 'TIMESTAMP',
|
||||
'smalldatetime': 'TIMESTAMP',
|
||||
'date': 'DATE',
|
||||
'time': 'TIME',
|
||||
'char': f'CHAR({max_length})' if max_length else 'CHAR(1)',
|
||||
'nchar': f'CHAR({max_length})' if max_length else 'CHAR(1)',
|
||||
'varchar': f'VARCHAR({max_length})' if max_length and max_length < 8000 else 'TEXT',
|
||||
'nvarchar': f'VARCHAR({max_length})' if max_length and max_length < 8000 else 'TEXT',
|
||||
'text': 'TEXT',
|
||||
'ntext': 'TEXT',
|
||||
'uniqueidentifier': 'UUID',
|
||||
'timestamp': 'BYTEA',
|
||||
'rowversion': 'BYTEA',
|
||||
'binary': 'BYTEA',
|
||||
'varbinary': 'BYTEA',
|
||||
'image': 'BYTEA',
|
||||
'xml': 'XML',
|
||||
'json': 'JSONB',
|
||||
'sql_variant': 'TEXT',
|
||||
'geometry': 'TEXT',
|
||||
'geography': 'TEXT',
|
||||
'hierarchyid': 'VARCHAR(255)'
|
||||
}
|
||||
|
||||
return type_map.get(mssql_type, 'TEXT')
|
||||
Reference in New Issue
Block a user