From 372671204d22e0f16f5af885581ce59d711f3d9f Mon Sep 17 00:00:00 2001 From: root Date: Sat, 15 Aug 2026 00:07:14 +0000 Subject: [PATCH] Fiabilise la synchronisation des grands mails Outlook --- app_new/outlook/models.py | 4 +- app_new/outlook/sync.py | 131 ++++++++++++------ app_new/outlook/templates/outlook/folder.html | 3 +- .../e2f6a7b8c9d0_expand_outlook_mail_body.py | 28 ++++ .../test_outlook_sync_resilience.py | 70 ++++++++++ 5 files changed, 189 insertions(+), 47 deletions(-) create mode 100644 migrations/versions/e2f6a7b8c9d0_expand_outlook_mail_body.py create mode 100644 tests/integration/test_outlook_sync_resilience.py diff --git a/app_new/outlook/models.py b/app_new/outlook/models.py index 1dd5e79..65132b7 100644 --- a/app_new/outlook/models.py +++ b/app_new/outlook/models.py @@ -4,6 +4,7 @@ Intégration avec Outlook/Microsoft """ from datetime import datetime, timezone from app_new.extensions import db +from sqlalchemy.dialects.mysql import LONGTEXT class OutlookAccount(db.Model): @@ -67,7 +68,8 @@ class OutlookMail(db.Model): sender = db.Column(db.String(255), nullable=True) sender_email = db.Column(db.String(255), nullable=True) body_preview = db.Column(db.Text, nullable=True) - body_content = db.Column(db.Text, nullable=True) + # Un corps HTML Outlook dépasse fréquemment la limite de 64 Kio de TEXT. + body_content = db.Column(LONGTEXT, nullable=True) body_type = db.Column(db.String(20), nullable=True) # html ou text received_at = db.Column(db.DateTime, nullable=True) is_read = db.Column(db.Boolean, default=False) diff --git a/app_new/outlook/sync.py b/app_new/outlook/sync.py index 6f13817..215ba58 100644 --- a/app_new/outlook/sync.py +++ b/app_new/outlook/sync.py @@ -1,9 +1,10 @@ """Outlook Sync Routes - GMAO Collège""" -from flask import Blueprint, render_template, redirect, url_for, request, flash, jsonify, send_file +from flask import Blueprint, current_app, render_template, redirect, url_for, request, flash, jsonify, send_file from flask_login import login_required, current_user from app_new.extensions import db from ..outlook.models import OutlookAccount, OutlookMail, OutlookFolder, OutlookAttachment, OutlookMailInterpretation, GmaoContext from ..outlook.auth import get_access_token +from ..core.models.maintenance import WatchdogLog import os import subprocess import json @@ -11,6 +12,28 @@ import tempfile sync_bp = Blueprint('outlook_sync', __name__, template_folder='templates') + +def _sync_error_message(exc, *, account_id, folder_id=None, mail_data=None): + """Produit un journal utile sans inclure le corps ni les destinataires du mail.""" + original = getattr(exc, 'orig', None) + error_code = getattr(original, 'args', [None])[0] if original else None + subject = ((mail_data or {}).get('Subject') or '(sans objet)')[:160] + mail_id = ((mail_data or {}).get('Id') or '(inconnu)')[:180] + content = ((mail_data or {}).get('Body') or {}).get('Content') or '' + detail = str(original or exc).split('[SQL:', 1)[0].strip()[:500] + return ( + f"Synchronisation Outlook impossible pour le message {mail_id} " + f"(compte={account_id}, dossier={folder_id or 'inbox'}, objet={subject!r}, " + f"corps={len(content.encode('utf-8'))} octets, code={error_code}): {detail}" + ) + + +def _store_sync_error(message): + """Ajoute une erreur à la page /logs/ dans la transaction courante.""" + db.session.add(WatchdogLog( + watchdog_name='outlook_sync', level='error', message=message[:10000], + )) + @sync_bp.route('/api/sync-folders/', methods=['POST']) @login_required def api_sync_folders(account_id): @@ -139,7 +162,7 @@ def api_sync_mails(account_id, folder_id='inbox'): return jsonify({'error': error}), 401 # Paramètres de pagination - max_messages = int(request.args.get('max', 200)) # Max messages à récupérer + max_messages = max(1, min(request.args.get('max', 200, type=int) or 200, 5000)) try: # Trouver le dossier @@ -185,49 +208,56 @@ def api_sync_mails(account_id, folder_id='inbox'): saved_count = 0 updated_count = 0 + skipped_count = 0 for mail_data in all_mails: - mail_id = mail_data.get('Id') - existing = OutlookMail.query.filter_by(id=mail_id).first() - - from_addr = mail_data.get('From', {}).get('EmailAddress', {}) - body = mail_data.get('Body', {}) - received_str = mail_data.get('ReceivedDateTime', '') try: - received_at = datetime.fromisoformat(received_str.replace('Z', '+00:00')) - except: - received_at = datetime.now(timezone.utc) - - # Destinataires - import json - to_recipients = json.dumps([r.get('EmailAddress', {}).get('Address', '') for r in mail_data.get('ToRecipients', [])]) - cc_recipients = json.dumps([r.get('EmailAddress', {}).get('Address', '') for r in mail_data.get('CcRecipients', [])]) - - if not existing: - mail = OutlookMail( - id=mail_id, - account_id=account.id, - folder_id=folder.id, - subject=mail_data.get('Subject', ''), - from_name=from_addr.get('Name', ''), - from_email=from_addr.get('Address', ''), - body_preview=mail_data.get('BodyPreview', ''), - body_content=body.get('Content', ''), - body_type=body.get('ContentType', 'text'), - received_at=received_at, - is_read=mail_data.get('IsRead', False), - importance=mail_data.get('Importance', 'normal'), - has_attachments=mail_data.get('HasAttachments', False), - to_recipients=to_recipients, - cc_recipients=cc_recipients + # Le SAVEPOINT et le flush immédiat empêchent un message + # invalide de faire échouer les suivants par autoflush. + with db.session.begin_nested(): + mail_id = mail_data.get('Id') + existing = OutlookMail.query.filter_by(id=mail_id).first() + from_addr = mail_data.get('From', {}).get('EmailAddress', {}) + body = mail_data.get('Body', {}) + received_str = mail_data.get('ReceivedDateTime', '') + try: + received_at = datetime.fromisoformat(received_str.replace('Z', '+00:00')) + except (TypeError, ValueError): + received_at = datetime.now(timezone.utc) + + to_recipients = json.dumps([r.get('EmailAddress', {}).get('Address', '') for r in mail_data.get('ToRecipients', [])]) + cc_recipients = json.dumps([r.get('EmailAddress', {}).get('Address', '') for r in mail_data.get('CcRecipients', [])]) + + if not existing: + db.session.add(OutlookMail( + id=mail_id, account_id=account.id, folder_id=folder.id, + subject=mail_data.get('Subject', ''), + from_name=from_addr.get('Name', ''), + from_email=from_addr.get('Address', ''), + body_preview=mail_data.get('BodyPreview', ''), + body_content=body.get('Content', ''), + body_type=body.get('ContentType', 'text'), + received_at=received_at, + is_read=mail_data.get('IsRead', False), + importance=mail_data.get('Importance', 'normal'), + has_attachments=mail_data.get('HasAttachments', False), + to_recipients=to_recipients, + cc_recipients=cc_recipients, + )) + db.session.flush() + saved_count += 1 + elif existing.is_read != mail_data.get('IsRead', False): + existing.is_read = mail_data.get('IsRead', False) + db.session.flush() + updated_count += 1 + except Exception as mail_error: + skipped_count += 1 + message = _sync_error_message( + mail_error, account_id=account.id, folder_id=folder.id, + mail_data=mail_data, ) - db.session.add(mail) - saved_count += 1 - else: - # Mettre à jour le statut de lecture - if existing.is_read != mail_data.get('IsRead', False): - existing.is_read = mail_data.get('IsRead', False) - updated_count += 1 + current_app.logger.warning(message, exc_info=True) + _store_sync_error(message) # Mettre à jour les infos du dossier folder.total_items = len(all_mails) @@ -241,13 +271,24 @@ def api_sync_mails(account_id, folder_id='inbox'): 'message': f'{saved_count} nouveaux mails, {updated_count} mis à jour sur {len(all_mails)} récupérés', 'total': len(all_mails), 'new': saved_count, - 'updated': updated_count + 'updated': updated_count, + 'skipped': skipped_count, + 'warning': f'{skipped_count} message(s) ignoré(s), détails dans /logs/.' if skipped_count else None, }) except Exception as e: - import traceback - traceback.print_exc() - return jsonify({'error': f'Erreur: {str(e)}'}), 500 + current_app.logger.exception('Échec global de la synchronisation Outlook') + db.session.rollback() + message = _sync_error_message( + e, account_id=account_id, folder_id=folder_id, + ) + try: + _store_sync_error(message) + db.session.commit() + except Exception: + db.session.rollback() + current_app.logger.exception("Impossible d'enregistrer l'erreur Outlook dans /logs/") + return jsonify({'error': 'La synchronisation Outlook a échoué. Consultez /logs/ pour le détail.'}), 500 # ─── Pièces jointes ───────────────────────────────────────────────────────── diff --git a/app_new/outlook/templates/outlook/folder.html b/app_new/outlook/templates/outlook/folder.html index 969b230..8e23e22 100644 --- a/app_new/outlook/templates/outlook/folder.html +++ b/app_new/outlook/templates/outlook/folder.html @@ -104,11 +104,12 @@ function syncMails(maxMessages) { .then(response => response.json()) .then(data => { if (data.success) { - status.className = 'alert alert-success mb-3'; + status.className = data.skipped ? 'alert alert-warning mb-3' : 'alert alert-success mb-3'; let msg = data.message || 'Emails synchronisés'; if (data.total) { msg += ' (' + data.total + ' messages)'; } + if (data.warning) msg += '
' + data.warning; status.innerHTML = ' ' + msg; setTimeout(() => window.location.reload(), 1500); } else { diff --git a/migrations/versions/e2f6a7b8c9d0_expand_outlook_mail_body.py b/migrations/versions/e2f6a7b8c9d0_expand_outlook_mail_body.py new file mode 100644 index 0000000..bbdf36e --- /dev/null +++ b/migrations/versions/e2f6a7b8c9d0_expand_outlook_mail_body.py @@ -0,0 +1,28 @@ +"""Agrandit le stockage du corps des messages Outlook. + +Revision ID: e2f6a7b8c9d0 +Revises: d1e5f6a7b8c9 +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import mysql + +revision = "e2f6a7b8c9d0" +down_revision = "d1e5f6a7b8c9" +branch_labels = None +depends_on = None + + +def upgrade(): + op.alter_column( + "outlook_mails", + "body_content", + existing_type=sa.Text(), + type_=mysql.LONGTEXT(), + existing_nullable=True, + ) + + +def downgrade(): + # La réduction pourrait détruire les corps supérieurs à 64 Kio. + raise RuntimeError("Downgrade refusé : risque de perte des grands corps Outlook") diff --git a/tests/integration/test_outlook_sync_resilience.py b/tests/integration/test_outlook_sync_resilience.py new file mode 100644 index 0000000..8e47738 --- /dev/null +++ b/tests/integration/test_outlook_sync_resilience.py @@ -0,0 +1,70 @@ +from uuid import uuid4 + +from app_new.extensions import db +from app_new.core.models.maintenance import WatchdogLog +from app_new.outlook.models import OutlookAccount, OutlookFolder, OutlookMail + + +class _OutlookResponse: + status_code = 200 + + def __init__(self, messages): + self._messages = messages + + def json(self): + return {'value': self._messages} + + +def _message(message_id, subject, body): + return { + 'Id': message_id, + 'Subject': subject, + 'From': {'EmailAddress': {'Name': 'Expéditeur', 'Address': 'sender@example.test'}}, + 'ReceivedDateTime': '2026-08-14T10:00:00Z', + 'IsRead': False, + 'HasAttachments': False, + 'BodyPreview': body[:200], + 'Body': {'Content': body, 'ContentType': 'HTML'}, + 'ToRecipients': [], 'CcRecipients': [], 'Importance': 'normal', + } + + +def test_outlook_sync_stores_large_body_and_logs_only_bad_message( + authenticated_client, app, admin_user, monkeypatch): + with app.app_context(): + account = OutlookAccount(user_id=admin_user['id'], email='sync@example.test', tenant_id='test') + folder = OutlookFolder( + id=f'folder-{uuid4().hex}', account=account, name='Inbox', + display_name='Boîte de réception', folder_type='inbox', + ) + db.session.add_all([account, folder]); db.session.commit() + account_id, folder_id = account.id, folder.id + + large_body = '

' + ('Contenu volumineux ' * 7000) + '

' + confidential_bad_body = 'SECRET_BODY_' * 100 + messages = [ + _message('X' * 600, 'Message invalide', confidential_bad_body), + _message(f'mail-{uuid4().hex}', 'Grand message valide', large_body), + ] + monkeypatch.setattr('app_new.outlook.sync.get_access_token', lambda unused: ('token', None)) + monkeypatch.setattr('requests.get', lambda *args, **kwargs: _OutlookResponse(messages)) + + response = authenticated_client.post( + f'/outlook/api/sync-mails/{account_id}/{folder_id}?max=20' + ) + assert response.status_code == 200 + payload = response.get_json() + assert payload['new'] == 1 + assert payload['skipped'] == 1 + + stored = OutlookMail.query.filter_by(account_id=account_id, subject='Grand message valide').one() + assert len(stored.body_content.encode('utf-8')) > 65535 + log = WatchdogLog.query.filter_by(watchdog_name='outlook_sync', level='error').one() + assert 'Message invalide' in log.message + assert 'SECRET_BODY' not in log.message + + OutlookMail.query.filter_by(account_id=account_id).delete(synchronize_session=False) + OutlookFolder.query.filter_by(account_id=account_id).delete(synchronize_session=False) + OutlookAccount.query.filter_by(id=account_id).delete(synchronize_session=False) + WatchdogLog.query.filter_by(id=log.id).delete(synchronize_session=False) + db.session.commit()