Добавил поддержку webhook

This commit is contained in:
brusnitsyn
2026-06-16 16:03:49 +09:00
parent 17f4d4307f
commit 6ef045ca88
2 changed files with 157 additions and 2 deletions

View File

@@ -10,6 +10,8 @@ except ImportError:
HTTPException = None HTTPException = None
BaseModel = object BaseModel = object
from sqlalchemy import text
from .config import Config from .config import Config
from .queue import migration_queue from .queue import migration_queue
from .table_config_repository import TableConfigRepository from .table_config_repository import TableConfigRepository
@@ -45,6 +47,16 @@ class ScheduleRequest(BaseModel):
initial_force_full: bool = False initial_force_full: bool = False
class WebhookCreate(BaseModel):
url: str
secret: Optional[str] = None
label: Optional[str] = None
class WebhookToggle(BaseModel):
active: bool
def create_app(): def create_app():
"""Создание FastAPI приложения.""" """Создание FastAPI приложения."""
if FastAPI is None: if FastAPI is None:
@@ -57,12 +69,29 @@ def create_app():
from sqlalchemy import create_engine from sqlalchemy import create_engine
config = Config() config = Config()
_shared['engine'] = create_engine( engine = create_engine(
config.POSTGRES_CONNECTION_STRING, config.POSTGRES_CONNECTION_STRING,
pool_pre_ping=True, pool_pre_ping=True,
pool_recycle=1800, pool_recycle=1800,
) )
_shared['engine'] = engine
_shared['config'] = config _shared['config'] = config
schema = f'"{config.REPLICATOR_SCHEMA}"'
with engine.connect() as conn:
conn.execute(text(f'CREATE SCHEMA IF NOT EXISTS {schema}'))
conn.execute(text(f"""
CREATE TABLE IF NOT EXISTS {schema}.webhook_subscriptions (
id SERIAL PRIMARY KEY,
url TEXT NOT NULL,
secret TEXT,
label TEXT,
active BOOLEAN NOT NULL DEFAULT true,
created_at TIMESTAMP NOT NULL DEFAULT now()
)
"""))
conn.commit()
if Config.START_API_WORKER: if Config.START_API_WORKER:
migration_queue.start() migration_queue.start()
@@ -169,6 +198,62 @@ def create_app():
return {"status": "scheduled", "schedule": schedule} return {"status": "scheduled", "schedule": schedule}
# -----------------------------------------------------------------------
# Webhooks
# -----------------------------------------------------------------------
_WEBHOOKS_TABLE = f'"{Config.REPLICATOR_SCHEMA}".webhook_subscriptions'
@api.get("/webhooks")
def list_webhooks():
with _shared['engine'].connect() as conn:
rows = conn.execute(
text(f"SELECT id, url, secret, label, active, created_at FROM {_WEBHOOKS_TABLE} ORDER BY id")
).mappings().all()
return [dict(r) for r in rows]
@api.post("/webhooks", status_code=201)
def create_webhook(body: WebhookCreate):
with _shared['engine'].connect() as conn:
row = conn.execute(
text(f"""
INSERT INTO {_WEBHOOKS_TABLE} (url, secret, label)
VALUES (:url, :secret, :label)
RETURNING id, url, secret, label, active, created_at
"""),
{'url': body.url, 'secret': body.secret, 'label': body.label},
).mappings().first()
conn.commit()
return dict(row)
@api.patch("/webhooks/{webhook_id}")
def toggle_webhook(webhook_id: int, body: WebhookToggle):
with _shared['engine'].connect() as conn:
row = conn.execute(
text(f"""
UPDATE {_WEBHOOKS_TABLE}
SET active = :active
WHERE id = :id
RETURNING id, url, secret, label, active, created_at
"""),
{'active': body.active, 'id': webhook_id},
).mappings().first()
conn.commit()
if not row:
raise HTTPException(status_code=404, detail="Webhook not found")
return dict(row)
@api.delete("/webhooks/{webhook_id}", status_code=204)
def delete_webhook(webhook_id: int):
with _shared['engine'].connect() as conn:
result = conn.execute(
text(f"DELETE FROM {_WEBHOOKS_TABLE} WHERE id = :id"),
{'id': webhook_id},
)
conn.commit()
if result.rowcount == 0:
raise HTTPException(status_code=404, detail="Webhook not found")
return api return api

View File

@@ -4,6 +4,8 @@ import threading
import time import time
import csv import csv
import io import io
import json
import urllib.request
from datetime import datetime from datetime import datetime
from typing import Any, Dict, List, Optional, Tuple from typing import Any, Dict, List, Optional, Tuple
@@ -415,6 +417,70 @@ class DatabaseMigrator:
self._state_table_ready = True self._state_table_ready = True
def ensure_webhooks_table(self):
"""Создание таблицы подписок webhook если не существует."""
schema = self.quote_identifier(self.config.REPLICATOR_SCHEMA)
table = f"{schema}.webhook_subscriptions"
with self.dst_engine.connect() as conn:
conn.execute(text(f'CREATE SCHEMA IF NOT EXISTS {schema}'))
conn.execute(text(f"""
CREATE TABLE IF NOT EXISTS {table} (
id SERIAL PRIMARY KEY,
url TEXT NOT NULL,
secret TEXT,
label TEXT,
active BOOLEAN NOT NULL DEFAULT true,
created_at TIMESTAMP NOT NULL DEFAULT now()
)
"""))
conn.commit()
def _send_webhooks(self, report: dict) -> None:
"""POST итогов миграции на все активные webhook-подписки."""
schema = self.quote_identifier(self.config.REPLICATOR_SCHEMA)
table = f"{schema}.webhook_subscriptions"
try:
with self.dst_engine.connect() as conn:
rows = conn.execute(
text(f"SELECT url, secret FROM {table} WHERE active = true")
).mappings().all()
except Exception as e:
self.logger.log_warning(f"Не удалось прочитать webhook-подписки: {e}")
return
if not rows:
return
summary = report.get('summary', {})
failed = report.get('failed_tables', [])
payload = json.dumps({
'status': 'success' if not failed else ('partial_success' if summary.get('successful_tables') else 'failed'),
'started_at': str(self.logger.start_time),
'finished_at': datetime.now().isoformat(),
'tables': {
'total': summary.get('total_tables', 0),
'success': summary.get('successful_tables', 0),
'failed': summary.get('failed_tables', 0),
},
'errors': [{'table': t.get('name'), 'error': t.get('error')} for t in failed],
}).encode()
timeout = int(os.getenv('WEBHOOK_TIMEOUT', '10'))
for row in rows:
url, secret = row['url'], row['secret'] or ''
for attempt in range(3):
try:
req = urllib.request.Request(url, data=payload, headers={
'Content-Type': 'application/json',
'X-Syncio-Secret': secret,
})
urllib.request.urlopen(req, timeout=timeout)
self.logger.log_info(f"Webhook отправлен: {url}")
break
except Exception as e:
if attempt == 2:
self.logger.log_warning(f"Webhook {url} не отвечает (попытка 3/3): {e}")
def get_last_watermark(self, table_name: str) -> Dict[str, Any]: def get_last_watermark(self, table_name: str) -> Dict[str, Any]:
"""Чтение последнего успешно обработанного x_DateTime.""" """Чтение последнего успешно обработанного x_DateTime."""
empty_watermark = {'last_x_datetime': None, 'last_sequence_value': None} empty_watermark = {'last_x_datetime': None, 'last_sequence_value': None}
@@ -1705,6 +1771,8 @@ class DatabaseMigrator:
] ]
self.logger.stats['total_tables'] = len(table_configs) self.logger.stats['total_tables'] = len(table_configs)
self.ensure_webhooks_table()
self.logger.log_info("="*60) self.logger.log_info("="*60)
self.logger.log_info("НАЧАЛО МИГРАЦИИ ДАННЫХ") self.logger.log_info("НАЧАЛО МИГРАЦИИ ДАННЫХ")
self.logger.log_info(f"Время начала: {self.logger.start_time}") self.logger.log_info(f"Время начала: {self.logger.start_time}")
@@ -1741,7 +1809,9 @@ class DatabaseMigrator:
self.send_notification(report) self.send_notification(report)
else: else:
self.logger.log_info("Email отключен для этого запуска") self.logger.log_info("Email отключен для этого запуска")
self._send_webhooks(report)
return report return report
def _get_pg_row_count(self, table_name: str) -> int: def _get_pg_row_count(self, table_name: str) -> int: