first commit
This commit is contained in:
brusnitsyn
85
app/core/config.py
Normal file
85
app/core/config.py
Normal file
@@ -0,0 +1,85 @@
|
||||
import logging
|
||||
import os
|
||||
from typing import List, Literal, Optional, Dict
|
||||
from urllib.parse import quote_plus
|
||||
from pydantic_settings import BaseSettings
|
||||
from pydantic import Field
|
||||
from dotenv import load_dotenv
|
||||
|
||||
load_dotenv()
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
"""Конфигурация приложения из переменных окружения"""
|
||||
|
||||
# Настройки MSSQL
|
||||
MSSQL_SERVER: str = Field(..., env='MSSQL_SERVER')
|
||||
MSSQL_DATABASE: str = Field(..., env='MSSQL_DATABASE')
|
||||
MSSQL_USERNAME: str = Field(..., env='MSSQL_USERNAME')
|
||||
MSSQL_PASSWORD: str = Field(..., env='MSSQL_PASSWORD')
|
||||
MSSQL_DRIVER: str = 'pymssql'
|
||||
|
||||
# Настройки PostgreSQL
|
||||
POSTGRES_HOST: str = Field(..., env='POSTGRES_HOST')
|
||||
POSTGRES_PORT: int = Field(5432, env='POSTGRES_PORT')
|
||||
POSTGRES_DATABASE: str = Field(..., env='POSTGRES_DATABASE')
|
||||
POSTGRES_USERNAME: str = Field(..., env='POSTGRES_USERNAME')
|
||||
POSTGRES_PASSWORD: str = Field(..., env='POSTGRES_PASSWORD')
|
||||
|
||||
# Настройки email
|
||||
EMAIL_HOST: str = Field(..., env='EMAIL_HOST')
|
||||
EMAIL_PORT: int = Field(465, env='EMAIL_PORT')
|
||||
EMAIL_USER: str = Field(..., env='EMAIL_USER')
|
||||
EMAIL_PASSWORD: str = Field(..., env='EMAIL_PASSWORD')
|
||||
EMAIL_FROM: str = Field(..., env='EMAIL_FROM')
|
||||
EMAIL_TO: List[str] = Field(default_factory=lambda: ['andrew.brusnitsyn@gmail.com'])
|
||||
EMAIL_SUBJECT: str = 'Результат миграции данных MSSQL → PostgreSQL'
|
||||
|
||||
# Настройки миграции
|
||||
CHUNK_SIZE: int = Field(1000, env='CHUNK_SIZE')
|
||||
BATCH_SIZE: int = Field(10, env='BATCH_SIZE')
|
||||
TABLES_TO_COPY: List[str] = Field(
|
||||
default_factory=lambda: ['oms_Department']
|
||||
)
|
||||
LIFE_TABLES: List[str] = Field(
|
||||
default_factory=lambda: ['oms_Department']
|
||||
)
|
||||
|
||||
# Колонки дат для инкрементальной загрузки
|
||||
DEFAULT_DATE_COLUMNS: List[str] = Field(
|
||||
default_factory=lambda: [
|
||||
'DateExtract', 'DateDirection', 'DateRecipient', 'CreateDate',
|
||||
'UpdateDate', 'ModifiedDate', 'ChangeDate', 'LastModified'
|
||||
]
|
||||
)
|
||||
|
||||
# Файлы состояния
|
||||
LAST_REPLICATION_FILE: str = 'last_replication.json'
|
||||
LOG_DIR: str = 'logs'
|
||||
|
||||
# Настройки API
|
||||
API_V1_PREFIX: str = '/api/v1'
|
||||
DEBUG: bool = Field(False, env='DEBUG')
|
||||
|
||||
@property
|
||||
def MSSQL_CONNECTION_STRING(self) -> str:
|
||||
"""Формирование строки подключения к MSSQL"""
|
||||
# Для pymssql с Windows аутентификацией используем формат:
|
||||
# mssql+pymssql://domain\\username:password@server:port/database
|
||||
# ИЛИ для trusted connection:
|
||||
# mssql+pymssql://username:password@server:port/database?charset=utf8
|
||||
|
||||
return rf'mssql+{self.MSSQL_DRIVER}://{self.MSSQL_USERNAME}:{self.MSSQL_PASSWORD}@{self.MSSQL_SERVER}/{self.MSSQL_DATABASE}'
|
||||
|
||||
@property
|
||||
def POSTGRES_CONNECTION_STRING(self) -> str:
|
||||
"""Формирование строки подключения к PostgreSQL"""
|
||||
return rf'postgresql://{self.POSTGRES_USERNAME}:{self.POSTGRES_PASSWORD}@{self.POSTGRES_HOST}:{self.POSTGRES_PORT}/{self.POSTGRES_DATABASE}'
|
||||
|
||||
class Config:
|
||||
env_file = '.env'
|
||||
case_sensitive = False
|
||||
|
||||
|
||||
# Глобальный экземпляр настроек
|
||||
settings = Settings()
|
||||
68
app/core/database.py
Normal file
68
app/core/database.py
Normal file
@@ -0,0 +1,68 @@
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.engine import Engine
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from contextlib import contextmanager
|
||||
from typing import Optional
|
||||
from app.core.config import settings
|
||||
|
||||
|
||||
class DatabaseConnector:
|
||||
"""Управление подключениями к базам данных"""
|
||||
|
||||
def __init__(self):
|
||||
self._src_engine: Optional[Engine] = None
|
||||
self._dst_engine: Optional[Engine] = None
|
||||
self.dst_session = None
|
||||
self.schedule_session = None
|
||||
|
||||
@property
|
||||
def src_engine(self) -> Engine:
|
||||
"""Подключение к MSSQL"""
|
||||
if not self._src_engine:
|
||||
self._src_engine = create_engine(
|
||||
settings.MSSQL_CONNECTION_STRING,
|
||||
pool_pre_ping=True,
|
||||
echo=settings.DEBUG
|
||||
)
|
||||
return self._src_engine
|
||||
|
||||
@property
|
||||
def dst_engine(self) -> Engine:
|
||||
"""Подключение к PostgreSQL (основная БД)"""
|
||||
if not self._dst_engine:
|
||||
self._dst_engine = create_engine(
|
||||
settings.POSTGRES_CONNECTION_STRING,
|
||||
pool_pre_ping=True,
|
||||
echo=settings.DEBUG
|
||||
)
|
||||
self.dst_session = sessionmaker(bind=self._dst_engine)
|
||||
return self._dst_engine
|
||||
|
||||
@contextmanager
|
||||
def src_connection(self):
|
||||
"""Контекстный менеджер для MSSQL соединения"""
|
||||
conn = self.src_engine.connect()
|
||||
try:
|
||||
yield conn
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
@contextmanager
|
||||
def dst_connection(self):
|
||||
"""Контекстный менеджер для PostgreSQL соединения"""
|
||||
conn = self.dst_engine.connect()
|
||||
try:
|
||||
yield conn
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def dispose_engines(self):
|
||||
"""Закрытие всех соединений"""
|
||||
if self._src_engine:
|
||||
self._src_engine.dispose()
|
||||
if self._dst_engine:
|
||||
self._dst_engine.dispose()
|
||||
|
||||
|
||||
# Глобальный экземпляр подключений
|
||||
db_connector = DatabaseConnector()
|
||||
159
app/core/logging.py
Normal file
159
app/core/logging.py
Normal file
@@ -0,0 +1,159 @@
|
||||
import logging
|
||||
import os
|
||||
import json
|
||||
import traceback
|
||||
from datetime import datetime
|
||||
from typing import Dict, Any, Optional, List
|
||||
from pathlib import Path
|
||||
from app.core.config import settings
|
||||
|
||||
|
||||
class MigrationLogger:
|
||||
"""Класс для логирования процесса миграции"""
|
||||
|
||||
def __init__(self):
|
||||
self.start_time = datetime.now()
|
||||
self.timestamp = self.start_time.strftime("%Y%m%d_%H%M%S")
|
||||
|
||||
# Создаем директорию для логов
|
||||
log_dir = Path(settings.LOG_DIR)
|
||||
log_dir.mkdir(exist_ok=True)
|
||||
|
||||
self.log_file = log_dir / f"migration_log_{self.timestamp}.log"
|
||||
self.stats = {
|
||||
'total_tables': len(settings.TABLES_TO_COPY),
|
||||
'copied_tables': [],
|
||||
'failed_tables': [],
|
||||
'schema_changes': [],
|
||||
'total_rows': 0,
|
||||
'start_time': self.start_time,
|
||||
'end_time': None,
|
||||
'errors': []
|
||||
}
|
||||
|
||||
# Настраиваем logging
|
||||
self._setup_logging()
|
||||
|
||||
def _setup_logging(self):
|
||||
"""Настройка системы логирования"""
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
|
||||
handlers=[
|
||||
logging.FileHandler(self.log_file, encoding='utf-8'),
|
||||
logging.StreamHandler()
|
||||
]
|
||||
)
|
||||
self.logger = logging.getLogger(__name__)
|
||||
|
||||
def info(self, message: str):
|
||||
"""Логирование информационного сообщения"""
|
||||
self.logger.info(message)
|
||||
|
||||
def error(self, message: str, exception: Optional[Exception] = None):
|
||||
"""Логирование ошибки"""
|
||||
error_details = {
|
||||
'message': message,
|
||||
'exception': str(exception) if exception else None,
|
||||
'traceback': traceback.format_exc() if exception else None,
|
||||
'timestamp': datetime.now().isoformat()
|
||||
}
|
||||
self.stats['errors'].append(error_details)
|
||||
self.logger.error(f"{message}. Ошибка: {exception}")
|
||||
|
||||
def warning(self, message: str):
|
||||
"""Логирование предупреждения"""
|
||||
self.logger.warning(message)
|
||||
|
||||
def debug(self, message: str):
|
||||
"""Логирование отладочной информации"""
|
||||
self.logger.debug(message)
|
||||
|
||||
def table_start(self, table_name: str):
|
||||
"""Логирование начала обработки таблицы"""
|
||||
self.logger.info(f"{'='*60}")
|
||||
self.logger.info(f"Начало обработки таблицы: {table_name}")
|
||||
self.logger.info(f"{'='*60}")
|
||||
|
||||
def table_success(self, table_name: str, row_count: int):
|
||||
"""Логирование успешного копирования таблицы"""
|
||||
self.stats['copied_tables'].append({
|
||||
'name': table_name,
|
||||
'row_count': row_count,
|
||||
'timestamp': datetime.now().isoformat()
|
||||
})
|
||||
self.stats['total_rows'] += row_count
|
||||
self.logger.info(f"Таблица {table_name} успешно скопирована ({row_count} строк)")
|
||||
|
||||
def table_failure(self, table_name: str, error: str):
|
||||
"""Логирование ошибки при копировании таблицы"""
|
||||
self.stats['failed_tables'].append({
|
||||
'name': table_name,
|
||||
'error': error,
|
||||
'timestamp': datetime.now().isoformat()
|
||||
})
|
||||
self.logger.error(f"Ошибка при копировании таблицы {table_name}: {error}")
|
||||
|
||||
def progress(self, table_name: str, chunk_num: int, total_rows: int):
|
||||
"""Логирование прогресса загрузки"""
|
||||
self.logger.info(f"Таблица {table_name}: загружено чанков {chunk_num}, строк {total_rows}")
|
||||
|
||||
def schema_change(self, table_name: str, new_columns: List[str]):
|
||||
"""Логирование изменения схемы"""
|
||||
self.stats['schema_changes'].append({
|
||||
'table': table_name,
|
||||
'new_columns': new_columns,
|
||||
'timestamp': datetime.now().isoformat()
|
||||
})
|
||||
self.logger.info(f"В таблице {table_name} обнаружены новые колонки: {new_columns}")
|
||||
|
||||
def generate_report(self) -> Dict[str, Any]:
|
||||
"""Генерация итогового отчета"""
|
||||
self.stats['end_time'] = datetime.now()
|
||||
duration = self.stats['end_time'] - self.stats['start_time']
|
||||
self.stats['duration_seconds'] = duration.total_seconds()
|
||||
self.stats['duration_human'] = str(duration)
|
||||
|
||||
report = {
|
||||
'summary': {
|
||||
'total_tables': self.stats['total_tables'],
|
||||
'successful_tables': len(self.stats['copied_tables']),
|
||||
'failed_tables': len(self.stats['failed_tables']),
|
||||
'schema_changes': len(self.stats['schema_changes']),
|
||||
'success_rate': (len(self.stats['copied_tables']) / self.stats['total_tables'] * 100)
|
||||
if self.stats['total_tables'] > 0 else 0,
|
||||
'total_rows': self.stats['total_rows'],
|
||||
'start_time': self.stats['start_time'].isoformat(),
|
||||
'end_time': self.stats['end_time'].isoformat(),
|
||||
'duration': self.stats['duration_human']
|
||||
},
|
||||
'successful_tables': [
|
||||
{'name': t['name'], 'rows': t['row_count']}
|
||||
for t in self.stats['copied_tables']
|
||||
],
|
||||
'failed_tables': [
|
||||
{'name': t['name'], 'error': t['error']}
|
||||
for t in self.stats['failed_tables']
|
||||
],
|
||||
'schema_changes': self.stats['schema_changes'],
|
||||
'errors': self.stats['errors']
|
||||
}
|
||||
|
||||
# Сохраняем отчет в JSON
|
||||
report_file = Path(settings.LOG_DIR) / f"migration_report_{self.timestamp}.json"
|
||||
with open(report_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(report, f, ensure_ascii=False, indent=2, default=str)
|
||||
|
||||
return report
|
||||
|
||||
def get_log_content(self) -> str:
|
||||
"""Получение содержимого лог-файла"""
|
||||
try:
|
||||
with open(self.log_file, 'r', encoding='utf-8') as f:
|
||||
return f.read()
|
||||
except Exception as e:
|
||||
return f"Ошибка при чтении лог-файла: {e}"
|
||||
|
||||
|
||||
# Глобальный экземпляр логгера
|
||||
migration_logger = MigrationLogger()
|
||||
Reference in New Issue
Block a user