emailcleaner.class.php

Go to the documentation of this file.

<?php
/* Copyright (C) 2026  Braito                  <braito4@hotmail.com>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 */
 
require_once DOL_DOCUMENT_ROOT.'/core/lib/files.lib.php';
 
class EmailCleaner
{
  private const TABLE_AI_REQUEST_LOG = 'ai_request_log';
 
  public $db;
 
  public function __construct($db)
  {
    $this->db = $db;
  }
 
  public static function isRuntimeAvailable()
  {
    if (!isModEnabled('ai')) return false;
    if (!isModEnabled('emailcollector')) return false;
    return true;
  }
 
  public static function isEmailCollectorHookAction($action)
  {
    $actionCode = dol_strtolower(trim((string) $action));
    if (strpos($actionCode, 'hook') !== 0) return false;
    return in_array($actionCode, array('hook_ai_emailcleaner', 'hookaiemailcleaner'), true);
  }
 
  public function processEmailCollectorMessage($parameters, &$object)
  {
    global $conf, $dolibarr_main_data_root;
 
 
    $rawBody = self::normalizeText((string) ($parameters['messagetext'] ?? ''));
    $subject = (string) ($parameters['subject'] ?? '');
    $header = (string) ($parameters['header'] ?? '');
    // Email sender metadata; the SQL plugin treats variables named 'from' as SQL clauses.
    // @phan-suppress-next-line SqlInjection
    $from = (string) ($parameters['from'] ?? '');
    $to = self::extractHeaderFieldValue($header, 'To');
    $cc = self::extractHeaderFieldValue($header, 'Cc');
    $replyTo = self::extractHeaderFieldValue($header, 'Reply-To');
    $headerDate = self::extractHeaderFieldValue($header, 'Date');
    $inReplyTo = self::extractHeaderFieldValue($header, 'In-Reply-To');
    $referencesRaw = self::extractHeaderFieldValue($header, 'References');
    $references = self::extractHeaderMessageIds($referencesRaw);
    $collectorId = (!empty($object->id) ? (int) $object->id : 0);
    $entity = (!empty($conf->entity) ? (int) $conf->entity : 1);
    $isolatedMode = (int) getDolGlobalInt('AI_EMAILCLEANER_ISOLATED_MODE', 1);
    $thirdpartyId = ($isolatedMode ? 0 : (!empty($parameters['thirdpartyid']) ? (int) $parameters['thirdpartyid'] : 0));
    $objectId = ($isolatedMode ? 0 : (!empty($parameters['objectid']) ? (int) $parameters['objectid'] : 0));
    $hasAttachments = (!empty($parameters['attachments']) && is_array($parameters['attachments']));
    $actionParamRaw = (string) ($parameters['actionparam'] ?? '');
    $actionParamCfg = self::parseActionParamConfig($actionParamRaw);
    $contextProfileCode = trim((string) (!empty($actionParamCfg['context_profile_code']) ? $actionParamCfg['context_profile_code'] : ''));
    $contextProfileVersion = trim((string) (!empty($actionParamCfg['context_profile_version']) ? $actionParamCfg['context_profile_version'] : ''));
 
    if (trim($rawBody) === '') return array();
 
    $msgid = self::extractMessageIdFromHeader($header);
    if ($msgid === '') {
      $msgid = sha1($subject."\n".$from."\n".dol_substr($rawBody, 0, 4000, '8bit'));
    }
    $messageScopes = self::buildMessageScopesBasic($rawBody, $subject, $from, $headerDate, $msgid, $inReplyTo, $references);
    $replyOnly = (string) ($messageScopes['reply_only'] ?? '');
    $quotedContext = (string) ($messageScopes['quoted_context'] ?? '');
    $normalizedMessageId = self::normalizeMessageId($msgid);
    $normalizedInReplyTo = self::normalizeMessageId($inReplyTo);
    $normalizedReferences = self::normalizeMessageIdList($references);
    $ancestorMessageIds = self::normalizeMessageIdList(array_merge($normalizedReferences, ($normalizedInReplyTo !== '' ? array($normalizedInReplyTo) : array())));
    $emailContext = array(
      'threading' => array(
        'message_id' => ($normalizedMessageId !== '' ? $normalizedMessageId : $msgid),
        'in_reply_to' => ($normalizedInReplyTo !== '' ? $normalizedInReplyTo : null),
        'references' => $normalizedReferences,
        'ancestor_message_ids' => $ancestorMessageIds,
        'thread_depth' => count($ancestorMessageIds),
        'thread_key' => self::buildThreadKey($normalizedMessageId, $normalizedInReplyTo, $normalizedReferences),
        'link_type' => ($normalizedInReplyTo !== '' ? 'direct_reply' : (!empty($normalizedReferences) ? 'referenced_thread' : 'standalone')),
      ),
      'addresses' => array(
        'from' => $from,
        'to' => ($to !== '' ? $to : null),
        'cc' => ($cc !== '' ? $cc : null),
        'reply_to' => ($replyTo !== '' ? $replyTo : null),
      ),
      'meta' => array(
        'date_header' => ($headerDate !== '' ? $headerDate : null),
        'subject' => $subject,
        'subject_normalized' => self::normalizeText($subject),
        'collector_id' => $collectorId,
        'thirdparty_id' => ($thirdpartyId > 0 ? $thirdpartyId : null),
        'object_id' => ($objectId > 0 ? $objectId : null),
        'has_attachments' => ($hasAttachments ? 1 : 0),
        'isolated_mode' => ($isolatedMode ? 1 : 0),
      ),
      'body_context' => array(
        'reply_only' => ($replyOnly !== '' ? $replyOnly : null),
        'quoted_context' => ($quotedContext !== '' ? $quotedContext : null),
        'has_quoted_context' => ($quotedContext !== '' ? 1 : 0),
        'quoted_block_count' => count((array) ($messageScopes['quoted_blocks'] ?? array())),
        'quoted_blocks' => (array) ($messageScopes['quoted_blocks'] ?? array()),
        'parsing_warnings' => (array) ($messageScopes['parsing_warnings'] ?? array()),
      ),
      'timeline' => array(
        'order' => 'current_then_quoted_newest_to_oldest',
        'entries' => (array) ($messageScopes['timeline'] ?? array()),
      ),
    );
    $attachmentRows = self::collectAttachmentRows(
      (isset($parameters['attachments']) && is_array($parameters['attachments']) ? $parameters['attachments'] : array()),
      (isset($parameters['savedattachments']) && is_array($parameters['savedattachments']) ? $parameters['savedattachments'] : array()),
      (string) ($parameters['savedattachmentsdir'] ?? '')
    );
    $hasPdfAttachments = false;
    foreach ($attachmentRows as $attRow) {
      if (!empty($attRow['is_pdf'])) {
        $hasPdfAttachments = true;
        break;
      }
    }
 
    $cleanResult = $this->runEmailCleaner($subject, $from, $rawBody);
    $cleanedText = (string) ($cleanResult['clean_body'] ?? '');
    $confidence = (float) ($cleanResult['confidence'] ?? 0.0);
    $segments = (!empty($cleanResult['segments']) && is_array($cleanResult['segments'])) ? $cleanResult['segments'] : array();
    $emailUnderstanding = (!empty($cleanResult['email_understanding']) && is_array($cleanResult['email_understanding'])) ? $cleanResult['email_understanding'] : self::buildFallbackEmailUnderstanding($cleanedText);
    $engine = (string) ($cleanResult['engine'] ?? 'fallback');
    $fallbackUsed = !empty($cleanResult['fallback_used']);
    $hCleanup = self::buildNoiseSummaryFromSegments($segments);
    $emailContext['body_context']['excluded_context_summary'] = $hCleanup;
    $complianceMeta = self::buildComplianceMetadata($engine, $fallbackUsed, $confidence);
    $pdfAttachmentItems = array();
    if ($hasPdfAttachments) {
      foreach ($attachmentRows as $attRow) {
        if (empty($attRow['is_pdf'])) continue;
        $pdfAttachmentItems[] = array(
          'name' => (string) ($attRow['name'] ?? ''),
          'relative_path' => (string) ($attRow['relative_path'] ?? ''),
          'sha256' => (string) ($attRow['sha256'] ?? ''),
          'detected_doc_type' => self::detectPdfDocTypeFromAttachmentName((string) ($attRow['name'] ?? '')),
          'linking_target' => 'actioncomm',
          'final_object_attachment_pending' => 1,
        );
      }
    }
    $handoffPayload = array(
      'handoff_version' => 'emailcleaner_v2',
      'email' => array(
        'subject' => $subject,
        'from' => $from,
        'date' => ($headerDate !== '' ? $headerDate : null),
        'raw_hash' => hash('sha256', $rawBody),
        'message_id' => $msgid,
      ),
      'relevant_text' => array(
        array(
          'type' => 'main_content',
          'text' => $cleanedText,
          'confidence' => $confidence,
          'source' => 'clean_body',
        ),
      ),
      'email_understanding' => $emailUnderstanding,
      'conversation_context' => $emailContext,
      'supporting_evidence' => array(
        'segment_count' => count($segments),
        'fallback_used' => ($fallbackUsed ? 1 : 0),
        'understanding_source' => ($engine === 'ai' ? 'ai_structured_preprocess' : 'fallback_clean_text'),
      ),
      'excluded_noise_summary' => $hCleanup,
      'compliance' => $complianceMeta,
      'needs_reprocessing' => ($fallbackUsed || $confidence < (float) getDolGlobalString('AI_EMAILCLEANER_MIN_CONFIDENCE', '0.60') ? 1 : 0),
      'needs_pdf_linking' => ($hasPdfAttachments ? 1 : 0),
      'pdf_attachments' => $pdfAttachmentItems,
    );
 
    $baseDir = (!empty($conf->ai->dir_output) ? $conf->ai->dir_output : $dolibarr_main_data_root.'/ai');
    $storeDir = rtrim($baseDir, '/').'/emailcleaner/entity_'.$entity;
    dol_mkdir($storeDir);
 
    $safeKey = preg_replace('/[^a-zA-Z0-9_\-\.]/', '_', $msgid);
    if ($safeKey === '' || $safeKey === null) $safeKey = sha1($msgid);
    if (dol_strlen($safeKey) > 180) $safeKey = dol_substr($safeKey, 0, 180);
 
    $payload = array(
      'entity' => $entity,
      'collector_id' => $collectorId,
      'isolated_mode' => ($isolatedMode ? 1 : 0),
      'message_id' => $msgid,
      'email_from' => $from,
      'email_subject' => $subject,
      'raw_hash' => hash('sha256', $rawBody),
      'clean_hash' => hash('sha256', $cleanedText),
      'raw_body' => $rawBody,
      'clean_body' => $cleanedText,
      'segments' => $segments,
      'cleaning_confidence' => $confidence,
      'cleaning_model' => ($engine === 'ai' ? 'auto' : null),
      'prompt_code' => 'email_cleaner_v2',
      'prompt_version' => '2',
      'context_profile_code' => ($contextProfileCode !== '' ? $contextProfileCode : null),
      'context_profile_version' => ($contextProfileVersion !== '' ? $contextProfileVersion : null),
      'engine' => $engine,
      'fallback_used' => ($fallbackUsed ? 1 : 0),
      'email_context' => $emailContext,
      'email_understanding' => $emailUnderstanding,
      'compliance' => $complianceMeta,
      'handoff_payload_json' => $handoffPayload,
      'date_cleaning_gmt' => dol_print_date(dol_now('gmt'), '%Y-%m-%d %H:%M:%S'),
    );
 
    $actioncommId = $this->findActionCommIdByMessageId($entity, $msgid);
    $payload['fk_actioncomm'] = ($actioncommId > 0 ? $actioncommId : null);
 
    $aiRequestLogId = $this->insertAiRequestLogRow(
      $entity,
      $actioncommId,
      $collectorId,
      $msgid,
      $rawBody,
      $cleanedText,
      $segments,
      $confidence,
      $engine,
      ($engine === 'ai' ? 'auto' : null),
      'email_cleaner_v2',
      '2',
      $contextProfileCode,
      $contextProfileVersion,
      $emailContext,
      $emailUnderstanding,
      $handoffPayload,
      $payload
    );
    if ($aiRequestLogId > 0) {
      $payload['ai_request_log_id'] = $aiRequestLogId;
      $handoffPayload['ai_request_log_id'] = $aiRequestLogId;
      $payload['handoff_payload_json'] = $handoffPayload;
    } elseif ($actioncommId <= 0) {
      $payload['ai_request_log_status'] = 'not_persisted_no_event';
    } elseif (!getDolGlobalInt('AI_LOG_REQUESTS')) {
      $payload['ai_request_log_status'] = 'not_persisted_logging_disabled';
    } else {
      $payload['ai_request_log_status'] = 'not_persisted_storage_unavailable';
    }
 
    $json = json_encode($payload, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES);
    if ($json !== false) {
      $file = $storeDir.'/'.dol_sanitizeFileName($safeKey).'.json';
      dol_syslog(__METHOD__.": save email cleaner output into ".$file, LOG_DEBUG);
      @file_put_contents($file, $json);
      $payload['file'] = $file;
    }
 
    return $payload;
  }
 
  private function findActionCommIdByMessageId($entity, $msgid)
  {
    $msgid = trim((string) $msgid);
    if ($msgid === '') return 0;
 
    $sql = "SELECT id";
    $sql .= " FROM ".MAIN_DB_PREFIX."actioncomm";
    $sql .= " WHERE entity = ".((int) $entity);
    $sql .= " AND email_msgid = '".$this->db->escape($msgid)."'";
    $sql .= " ORDER BY id DESC";
    $sql .= $this->db->plimit(1);
 
    $resql = $this->db->query($sql);
    if (!$resql) return 0;
    $obj = $this->db->fetch_object($resql);
    $this->db->free($resql);
 
    return ($obj && !empty($obj->id) ? (int) $obj->id : 0);
  }
 
  private function insertAiRequestLogRow($entity, $actioncommId, $collectorId, $msgid, $rawBody, $cleanedText, $segments, $confidence, $engine, $model, $promptCode, $promptVersion, $contextProfileCode, $contextProfileVersion, $emailContext, $emailUnderstanding, $handoffPayload, $payload)
  {
    global $user;
 
    if ($actioncommId <= 0) return 0;
    if (!getDolGlobalInt('AI_LOG_REQUESTS')) return 0;
    if (!$this->isAiRequestLogTableAvailable()) return 0;
 
    require_once DOL_DOCUMENT_ROOT.'/ai/lib/ai.lib.php';
 
    $inputMetadata = array(
      'source' => 'emailcollector',
      'collector_id' => (int) $collectorId,
      'message_id' => (string) $msgid,
      'raw_hash' => hash('sha256', (string) $rawBody),
      'clean_hash' => hash('sha256', (string) $cleanedText),
      'prompt_code' => (string) $promptCode,
      'prompt_version' => (string) $promptVersion,
      'context_profile_code' => ($contextProfileCode !== '' ? (string) $contextProfileCode : null),
      'context_profile_version' => ($contextProfileVersion !== '' ? (string) $contextProfileVersion : null),
    );
    $outputJson = array(
      'clean_body' => $cleanedText,
      'segments' => (is_array($segments) ? $segments : array()),
      'cleaning_confidence' => (float) $confidence,
      'cleaning_model' => ($model !== null && $model !== '' ? (string) $model : null),
      'engine' => (string) $engine,
      'collector_id' => (int) $collectorId,
      'message_id' => (string) $msgid,
      'context_profile_code' => ($contextProfileCode !== '' ? (string) $contextProfileCode : null),
      'context_profile_version' => ($contextProfileVersion !== '' ? (string) $contextProfileVersion : null),
      'email_context' => (is_array($emailContext) ? $emailContext : array()),
      'email_understanding' => (is_array($emailUnderstanding) ? $emailUnderstanding : array()),
      'handoff_payload_json' => (is_array($handoffPayload) ? $handoffPayload : array()),
      'cleaner_payload' => (is_array($payload) ? $payload : array()),
    );
    $inputMetadataRaw = json_encode($inputMetadata, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES);
    $outputRaw = json_encode($outputJson, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES);
    if ($inputMetadataRaw === false || $outputRaw === false) return 0;
 
    $inputHash = hash('sha256', (string) $msgid);
    $outputHash = hash('sha256', $outputRaw);
    $securityHash = hash('sha256', ((int) $entity).'|'.((int) $actioncommId).'|email_cleaner|'.$inputHash.'|'.$outputHash);
    $minConfidence = (float) getDolGlobalString('AI_EMAILCLEANER_MIN_CONFIDENCE', '0.60');
    if ($minConfidence <= 0 || $minConfidence > 1) $minConfidence = 0.60;
    $status = ((float) $confidence >= $minConfidence ? 'Success' : 'LowConfidence');
    if ((string) $engine === 'fallback') $status = 'Fallback';
    $provider = ((string) $engine === 'ai' ? 'dolibarr_ai' : 'fallback');
    // Plain log text, escaped by ai_log_request() when building the INSERT.
    // @phan-suppress-next-line SqlInjection
    $queryText = 'EmailCleaner message '.$msgid;
 
    // Use the shared logger so the module logging policy also applies to collected emails.
    $context = array(
      'fk_actioncomm' => (int) $actioncommId,
      'input_hash' => $inputHash,
      'output_hash' => $outputHash,
      'security_hash' => $securityHash,
      'preserve_payloads' => true,
    );
    $logId = 0;
    ai_log_request(
      $this->db,
      $user,
      $queryText,
      array('tool' => 'email_cleaner'),
      $provider,
      0.0,
      (float) $confidence,
      $status,
      '',
      $inputMetadataRaw,
      $outputRaw,
      $context,
      $logId
    );
 
    return $logId;
  }
 
  private function isAiRequestLogTableAvailable()
  {
    static $available = null;
    if ($available !== null) return $available;
    $available = (bool) count($this->db->DDLInfoTable(MAIN_DB_PREFIX.self::TABLE_AI_REQUEST_LOG));
    return $available;
  }
 
  public static function parseActionParamConfig($raw)
  {
    $raw = trim((string) $raw);
    if ($raw === '') return array();
    if ($raw[0] !== '{') return array('context_profile_code' => $raw);
    $dec = json_decode($raw, true);
    if (!is_array($dec)) return array();
    return $dec;
  }
 
  public static function buildNoiseSummaryFromSegments($segments)
  {
    $out = array('counts' => array(), 'samples' => array());
    if (empty($segments) || !is_array($segments)) return $out;
    foreach ($segments as $seg) {
      if (!is_array($seg)) continue;
      $type = dol_strtolower(trim((string) ($seg['type'] ?? 'unknown')));
      $text = trim((string) ($seg['text'] ?? ''));
      if ($type === '') $type = 'unknown';
      if (empty($out['counts'][$type])) $out['counts'][$type] = 0;
      $out['counts'][$type]++;
      if ($text !== '' && (empty($out['samples'][$type]) || !is_array($out['samples'][$type]))) $out['samples'][$type] = array();
      if ($text !== '' && count($out['samples'][$type]) < 2) {
        $out['samples'][$type][] = (dol_strlen($text) > 180 ? dol_substr($text, 0, 180) : $text);
      }
    }
    return $out;
  }
 
  public static function buildComplianceMetadata($engine, $fallbackUsed, $confidence)
  {
    $minConfidence = (float) getDolGlobalString('AI_EMAILCLEANER_MIN_CONFIDENCE', '0.60');
    if ($minConfidence <= 0 || $minConfidence > 1) $minConfidence = 0.60;
 
    return array(
      'ai_transparency_label' => 'AI-generated technical preprocessing output',
      'human_review_required' => 1,
      'autonomous_business_action_allowed' => 0,
      'pii_redaction_enabled' => (getDolGlobalInt('AI_PRIVACY_REDACTION', 0) ? 1 : 0),
      'ai_system_used' => ((string) $engine === 'ai' ? 1 : 0),
      'fallback_used' => ($fallbackUsed ? 1 : 0),
      'cleaning_confidence' => (float) $confidence,
      'confidence_threshold' => $minConfidence,
      'policy_scope' => 'diagnostic_only_no_business_decision',
    );
  }
 
  public static function collectAttachmentRows($attachments, $savedAttachments, $savedDir = '')
  {
    $out = array();
    $savedByName = array();
    if (is_array($savedAttachments)) {
      foreach ($savedAttachments as $sa) {
        if (!is_array($sa)) continue;
        $n = trim((string) ($sa['name'] ?? ''));
        if ($n === '') continue;
        $savedByName[dol_strtolower($n)] = $sa;
      }
    }
 
    if (is_array($attachments)) {
      foreach ($attachments as $att) {
        $name = '';
        $relative = '';
        $sha = '';
        if (is_object($att)) {
          if (is_callable(array($att, 'getName'))) $name = (string) call_user_func(array($att, 'getName'));
          if (is_callable(array($att, 'getPath'))) $relative = (string) call_user_func(array($att, 'getPath'));
          if (is_callable(array($att, 'getContent'))) {
            $content = (string) call_user_func(array($att, 'getContent'));
            if ($content !== '') $sha = hash('sha256', $content);
          }
        } elseif (is_array($att)) {
          $name = (string) (!empty($att['name']) ? $att['name'] : '');
          $relative = (string) (!empty($att['path']) ? $att['path'] : '');
        }
        if ($name === '') continue;
        $lookup = dol_strtolower($name);
        if (isset($savedByName[$lookup]) && is_array($savedByName[$lookup])) {
          $sa = $savedByName[$lookup];
          if (!empty($sa['relative_path'])) $relative = (string) $sa['relative_path'];
          if (!empty($sa['sha256'])) $sha = (string) $sa['sha256'];
        }
        if ($sha === '' && $relative !== '') {
          $candidate = $relative;
          if ($candidate[0] !== '/' && $savedDir !== '') $candidate = rtrim($savedDir, '/').'/'.$candidate;
          if (@is_readable($candidate)) {
            $h = @hash_file('sha256', $candidate);
            if (is_string($h)) $sha = $h;
          }
        }
        $ext = dol_strtolower(pathinfo($name, PATHINFO_EXTENSION));
        $out[] = array(
          'name' => $name,
          'relative_path' => $relative,
          'sha256' => $sha,
          'is_pdf' => ($ext === 'pdf'),
        );
      }
    }
 
    if (empty($out) && is_array($savedAttachments)) {
      foreach ($savedAttachments as $sa) {
        if (!is_array($sa)) continue;
        $name = trim((string) ($sa['name'] ?? ''));
        if ($name === '') continue;
        $ext = dol_strtolower(pathinfo($name, PATHINFO_EXTENSION));
        $out[] = array(
          'name' => $name,
          'relative_path' => (string) ($sa['relative_path'] ?? ''),
          'sha256' => (string) ($sa['sha256'] ?? ''),
          'is_pdf' => ($ext === 'pdf'),
        );
      }
    }
 
    return $out;
  }
 
  public static function detectPdfDocTypeFromAttachmentName($filename)
  {
    $ext = dol_strtolower(pathinfo((string) $filename, PATHINFO_EXTENSION));
    return ($ext === 'pdf' ? 'pdf_document' : 'unknown');
  }
 
  public function runEmailCleaner($subject, $from, $rawBody)
  {
    $fallbackCleanBody = self::fallbackCleanBody($rawBody);
    $out = array(
      'clean_body' => $fallbackCleanBody,
      'confidence' => 0.40,
      'segments' => array(),
      'email_understanding' => self::buildFallbackEmailUnderstanding($fallbackCleanBody),
      'engine' => 'fallback',
      'fallback_used' => true,
    );
 
    require_once DOL_DOCUMENT_ROOT.'/ai/class/ai.class.php';
    $ai = new Ai($this->db);
    if (empty($ai->getApiService())) return $out;
 
    $maxInput = (int) getDolGlobalInt('AI_EMAILCLEANER_MAX_INPUT', 16000);
    if ($maxInput <= 1000) $maxInput = 16000;
    $redactionEnabled = (int) getDolGlobalInt('AI_PRIVACY_REDACTION', 0);
    $guard = null;
    $fromForPrompt = $from;
    $subjectForPrompt = $subject;
    $bodyForPrompt = $rawBody;
    if (dol_strlen($bodyForPrompt) > $maxInput) {
      $bodyForPrompt = dol_substr($bodyForPrompt, 0, $maxInput);
    }
    if ($redactionEnabled) {
      require_once DOL_DOCUMENT_ROOT.'/ai/class/privacy_guard.class.php';
      if (class_exists('PrivacyGuard')) {
        $guard = new PrivacyGuard();
        $fromForPrompt = $guard->mask((string) $fromForPrompt);
        $subjectForPrompt = $guard->mask((string) $subjectForPrompt);
        $bodyForPrompt = $guard->mask((string) $bodyForPrompt);
      }
    }
 
    $prompt = "You are an EmailCleaner and structured email preprocessor for ERP ingestion.\n";
    $prompt .= "Task: separate current message content from quoted thread/noise and extract neutral understanding of the email.\n";
    $prompt .= "Important constraints:\n";
    $prompt .= "- Do NOT classify business intent and do NOT decide any action.\n";
    $prompt .= "- Do NOT create, update, approve, reject, match, or route business objects.\n";
    $prompt .= "- Extract only explicit information present in the email. Do not guess missing dates, amounts, references, contacts, or meaning.\n";
    $prompt .= "- Be conservative: when unsure, keep text in clean_body.\n";
    $prompt .= "- Put quoted/forwarded old thread content in segments, but keep only current-message facts in email_understanding.\n";
    $prompt .= "- Return valid JSON only.\n";
    $prompt .= "JSON schema:\n";
    $prompt .= "{\n";
    $prompt .= "  \"clean_body\": string,\n";
    $prompt .= "  \"segments\": [\n";
    $prompt .= "    {\"type\":\"main_content|quoted_thread|signature|legal_disclaimer|environmental_notice|security_notice|social_footer|system_noise|unknown\",\"text\":string}\n";
    $prompt .= "  ],\n";
    $prompt .= "  \"email_understanding\": {\n";
    $prompt .= "    \"key_points\": string[],\n";
    $prompt .= "    \"standardized\": {\n";
    $prompt .= "      \"dates\": string[],\n";
    $prompt .= "      \"amounts\": string[],\n";
    $prompt .= "      \"references\": string[],\n";
    $prompt .= "      \"contacts\": string[],\n";
    $prompt .= "      \"languages\": string[]\n";
    $prompt .= "    },\n";
    $prompt .= "    \"categories\": {\n";
    $prompt .= "      \"topics\": string[],\n";
    $prompt .= "      \"referenced_items\": string[],\n";
    $prompt .= "      \"requested_operations\": string[],\n";
    $prompt .= "      \"risks_or_warnings\": string[],\n";
    $prompt .= "      \"noise_types\": string[]\n";
    $prompt .= "    },\n";
    $prompt .= "    \"structured\": {\n";
    $prompt .= "      \"primary_elements\": string[],\n";
    $prompt .= "      \"secondary_elements\": string[],\n";
    $prompt .= "      \"other_elements\": string[]\n";
    $prompt .= "    }\n";
    $prompt .= "  },\n";
    $prompt .= "  \"confidence\": number\n";
    $prompt .= "}\n";
    $prompt .= "Constraints:\n";
    $prompt .= "- clean_body max 2000 chars.\n";
    $prompt .= "- key_points max 8 items, each max 160 chars.\n";
    $prompt .= "- every standardized/categories/structured list max 10 items, each max 180 chars.\n";
    $prompt .= "Email metadata:\n";
    $prompt .= "FROM: ".$fromForPrompt."\n";
    $prompt .= "SUBJECT: ".$subjectForPrompt."\n";
    $prompt .= "BODY:\n".$bodyForPrompt;
 
    $generated = $ai->generateContent($prompt, 'auto', 'textgeneration', '');
    if (is_array($generated) && !empty($generated['error'])) return $out;
    if (!is_string($generated) || trim($generated) === '') return $out;
    if ($guard) {
      $generated = $guard->unmaskAiResponse($generated);
    }
 
    $decoded = self::decodeJsonFromAiString($generated);
    if (empty($decoded) || !is_array($decoded)) return $out;
 
    $cleanBody = self::normalizeText((string) ($decoded['clean_body'] ?? ''));
    if ($cleanBody === '') return $out;
 
    $confidence = (float) ($decoded['confidence'] ?? 0.0);
    if ($confidence < 0) $confidence = 0;
    if ($confidence > 1) $confidence = 1;
 
    $minConfidence = (float) getDolGlobalString('AI_EMAILCLEANER_MIN_CONFIDENCE', '0.60');
    if ($minConfidence <= 0 || $minConfidence > 1) $minConfidence = 0.60;
 
    $segments = array();
    if (!empty($decoded['segments']) && is_array($decoded['segments'])) {
      foreach ($decoded['segments'] as $seg) {
        if (!is_array($seg)) continue;
        $type = trim((string) ($seg['type'] ?? 'unknown'));
        $text = self::normalizeText((string) ($seg['text'] ?? ''));
        if ($text === '') continue;
        if ($type === '') $type = 'unknown';
        $segments[] = array('type' => $type, 'text' => $text);
        if (count($segments) >= 80) break;
      }
    }
 
    if ($confidence < $minConfidence) {
      // Conservative fallback: keep more context if confidence is low.
      $out['clean_body'] = self::fallbackCleanBody($rawBody);
      $out['confidence'] = $confidence;
      $out['segments'] = $segments;
      $out['email_understanding'] = self::sanitizeEmailUnderstanding($decoded['email_understanding'] ?? array(), $out['clean_body']);
      $out['engine'] = 'ai_low_confidence_fallback';
      $out['fallback_used'] = true;
      return $out;
    }
 
    $out['clean_body'] = $cleanBody;
    $out['confidence'] = $confidence;
    $out['segments'] = $segments;
    $out['email_understanding'] = self::sanitizeEmailUnderstanding($decoded['email_understanding'] ?? array(), $cleanBody);
    $out['engine'] = 'ai';
    $out['fallback_used'] = false;
 
    return $out;
  }
 
  private static function buildFallbackEmailUnderstanding($cleanBody)
  {
    $cleanBody = self::normalizeText((string) $cleanBody);
    $keyPoints = array();
    if ($cleanBody !== '') {
      $lines = preg_split("/\n/", $cleanBody);
      if (is_array($lines)) {
        foreach ($lines as $line) {
          $line = trim((string) $line);
          if ($line === '') continue;
          $line = preg_replace('/\s+/', ' ', $line);
          if (dol_strlen($line) > 160) $line = dol_substr($line, 0, 160);
          $keyPoints[] = $line;
          if (count($keyPoints) >= 5) break;
        }
      }
    }
 
    return array(
      'key_points' => $keyPoints,
      'standardized' => array(
        'dates' => array(),
        'amounts' => array(),
        'references' => array(),
        'contacts' => array(),
        'languages' => array(),
      ),
      'categories' => array(
        'topics' => array(),
        'referenced_items' => array(),
        'requested_operations' => array(),
        'risks_or_warnings' => array(),
        'noise_types' => array(),
      ),
      'structured' => array(
        'primary_elements' => $keyPoints,
        'secondary_elements' => array(),
        'other_elements' => array(),
      ),
    );
  }
 
  private static function sanitizeEmailUnderstanding($raw, $fallbackCleanBody = '')
  {
    if (!is_array($raw)) {
      return self::buildFallbackEmailUnderstanding($fallbackCleanBody);
    }
 
    $out = self::buildFallbackEmailUnderstanding('');
    $out['key_points'] = self::sanitizeStringList($raw['key_points'] ?? array(), 8, 160);
 
    $standardized = (isset($raw['standardized']) && is_array($raw['standardized'])) ? $raw['standardized'] : array();
    $out['standardized'] = array(
      'dates' => self::sanitizeStringList($standardized['dates'] ?? array(), 10, 80),
      'amounts' => self::sanitizeStringList($standardized['amounts'] ?? array(), 10, 80),
      'references' => self::sanitizeStringList($standardized['references'] ?? array(), 10, 120),
      'contacts' => self::sanitizeStringList($standardized['contacts'] ?? array(), 10, 160),
      'languages' => self::sanitizeStringList($standardized['languages'] ?? array(), 5, 40),
    );
 
    $categories = (isset($raw['categories']) && is_array($raw['categories'])) ? $raw['categories'] : array();
    $out['categories'] = array(
      'topics' => self::sanitizeStringList($categories['topics'] ?? array(), 10, 160),
      'referenced_items' => self::sanitizeStringList($categories['referenced_items'] ?? array(), 10, 160),
      'requested_operations' => self::sanitizeStringList($categories['requested_operations'] ?? array(), 10, 160),
      'risks_or_warnings' => self::sanitizeStringList($categories['risks_or_warnings'] ?? array(), 10, 180),
      'noise_types' => self::sanitizeStringList($categories['noise_types'] ?? array(), 10, 80),
    );
 
    $structured = (isset($raw['structured']) && is_array($raw['structured'])) ? $raw['structured'] : array();
    $out['structured'] = array(
      'primary_elements' => self::sanitizeStringList($structured['primary_elements'] ?? array(), 12, 180),
      'secondary_elements' => self::sanitizeStringList($structured['secondary_elements'] ?? array(), 12, 180),
      'other_elements' => self::sanitizeStringList($structured['other_elements'] ?? array(), 12, 180),
    );
 
    if (empty($out['key_points']) && empty($out['structured']['primary_elements'])) {
      return self::buildFallbackEmailUnderstanding($fallbackCleanBody);
    }
 
    return $out;
  }
 
  private static function sanitizeStringList($items, $maxItems, $maxLen)
  {
    $items = is_array($items) ? $items : array();
    $maxItems = max(0, (int) $maxItems);
    $maxLen = max(1, (int) $maxLen);
    $out = array();
    $seen = array();
 
    foreach ($items as $item) {
      $item = trim((string) $item);
      if ($item === '' || dol_strtolower($item) === 'null') continue;
      $item = preg_replace('/\s+/', ' ', $item);
      $item = trim((string) $item);
      if ($item === '') continue;
      if (dol_strlen($item) > $maxLen) {
        $item = dol_substr($item, 0, $maxLen);
        $item = rtrim($item, " \t\r\n,;:");
      }
      $key = dol_strtolower($item);
      if (!empty($seen[$key])) continue;
      $seen[$key] = 1;
      $out[] = $item;
      if (count($out) >= $maxItems) break;
    }
 
    return $out;
  }
 
  public static function normalizeText($text)
  {
    $text = (string) $text;
    $text = str_replace("\r\n", "\n", $text);
    $text = str_replace("\r", "\n", $text);
    return trim($text);
  }
 
  public static function extractMessageIdFromHeader($header)
  {
    $header = (string) $header;
    if ($header === '') return '';
    if (preg_match('/^Message-ID:\s*<([^>]+)>/mi', $header, $m)) {
      return trim((string) $m[1]);
    }
    if (preg_match('/^Message-ID:\s*([^\s]+)/mi', $header, $m)) {
      return trim(str_replace(array('<', '>'), '', (string) $m[1]));
    }
    return '';
  }
 
  public static function extractHeaderFieldValue($header, $fieldName)
  {
    $header = str_replace("\r\n", "\n", (string) $header);
    $header = str_replace("\r", "\n", $header);
    $fieldName = trim((string) $fieldName);
    if ($header === '' || $fieldName === '') return '';
 
    $re = '/^'.preg_quote($fieldName, '/').'\s*:\s*(.+)$/mi';
    if (!preg_match($re, $header, $m, PREG_OFFSET_CAPTURE)) return '';
 
    $val = (string) ($m[1][0] ?? '');
    $offset = (int) ($m[1][1] ?? 0);
    $after = dol_substr($header, $offset + dol_strlen($val, '8bit'), null, '8bit');
    if ($after !== '') {
      $lines = preg_split("/\n/", $after);
      if (is_array($lines)) {
        foreach ($lines as $ln) {
          if ($ln === '') break;
          if (preg_match('/^[ \t]+/', $ln)) {
            $val .= ' '.trim($ln);
            continue;
          }
          break;
        }
      }
    }
 
    return trim(str_replace(array('<', '>'), '', $val));
  }
 
  public static function extractHeaderMessageIds($raw)
  {
    $raw = trim((string) $raw);
    if ($raw === '') return array();
 
    $out = array();
    if (preg_match_all('/<([^>]+)>/', $raw, $m)) {
      foreach ((array) $m[1] as $v) {
        $v = trim((string) $v);
        if ($v === '') continue;
        $out[] = $v;
      }
    }
    if (empty($out)) {
      $parts = preg_split('/\s+/', str_replace(array(',', ';'), ' ', $raw));
      if (is_array($parts)) {
        foreach ($parts as $p) {
          $p = trim(str_replace(array('<', '>'), '', (string) $p));
          if ($p === '' || strpos($p, '@') === false) continue;
          $out[] = $p;
        }
      }
    }
 
    $out = array_values(array_unique($out));
    if (count($out) > 20) $out = array_slice($out, -20);
    return $out;
  }
 
  public static function normalizeMessageId($messageId)
  {
    $messageId = trim((string) $messageId);
    $messageId = trim($messageId, "<> \t\n\r\0\x0B");
    return dol_strtolower($messageId);
  }
 
  public static function normalizeMessageIdList($messageIds)
  {
    $out = array();
    $seen = array();
    foreach ((array) $messageIds as $messageId) {
      $messageId = self::normalizeMessageId((string) $messageId);
      if ($messageId === '' || isset($seen[$messageId])) continue;
      $seen[$messageId] = true;
      $out[] = $messageId;
    }
    if (count($out) > 20) $out = array_slice($out, -20);
    return $out;
  }
 
  public static function buildThreadKey($messageId, $inReplyTo, $references)
  {
    $references = self::normalizeMessageIdList($references);
    $root = (!empty($references) ? (string) $references[0] : self::normalizeMessageId($inReplyTo));
    if ($root === '') $root = self::normalizeMessageId($messageId);
    return ($root !== '' ? 'rfc822:'.hash('sha256', $root) : '');
  }
 
  public static function buildMessageScopesBasic($body, $subject = '', $from = '', $date = '', $messageId = '', $inReplyTo = '', $references = array())
  {
    $body = self::normalizeText((string) $body);
    $replyOnly = self::extractReplyOnlyTextBasic($body);
    $quotedBlocks = self::extractQuotedMessageBlocksBasic($body, 16);
    $quotedParts = array();
    $timeline = array(array(
      'kind' => 'current_message',
      'index' => -1,
      'from' => (trim((string) $from) !== '' ? trim((string) $from) : null),
      'date_raw' => (trim((string) $date) !== '' ? trim((string) $date) : null),
      'subject' => (trim((string) $subject) !== '' ? trim((string) $subject) : null),
      'message_id' => (self::normalizeMessageId($messageId) !== '' ? self::normalizeMessageId($messageId) : null),
      'in_reply_to' => (self::normalizeMessageId($inReplyTo) !== '' ? self::normalizeMessageId($inReplyTo) : null),
      'references' => self::normalizeMessageIdList($references),
    ));
 
    foreach ($quotedBlocks as $index => $block) {
      if (!is_array($block)) continue;
      $text = trim((string) ($block['text'] ?? ''));
      if ($text !== '') $quotedParts[] = $text;
      $timeline[] = array(
        'kind' => 'quoted_message',
        'index' => (int) $index,
        'from' => ($block['from'] ?? null),
        'date_raw' => ($block['date_raw'] ?? null),
        'subject' => ($block['subject'] ?? null),
        'message_id' => ($block['message_id'] ?? null),
        'in_reply_to' => ($block['in_reply_to'] ?? null),
        'references' => (array) ($block['references'] ?? array()),
      );
    }
 
    $quotedContext = trim(implode("\n\n", $quotedParts));
    if ($quotedContext === '') $quotedContext = self::extractQuotedThreadSnippetBasic($body, 1200);
    if (dol_strlen($quotedContext) > 2400) $quotedContext = dol_substr($quotedContext, 0, 2400);
 
    $warnings = array();
    if ($quotedContext !== '' && empty($quotedBlocks)) $warnings[] = 'quoted_context_without_structured_blocks';
 
    return array(
      'schema_version' => 1,
      'parser' => 'emailcleaner_message_scopes_v1',
      'reply_only' => $replyOnly,
      'quoted_context' => $quotedContext,
      'quoted_blocks' => $quotedBlocks,
      'timeline' => $timeline,
      'parsing_warnings' => $warnings,
    );
  }
 
  public static function extractQuotedMessageBlocksBasic($body, $maxBlocks = 16)
  {
    $body = self::normalizeText((string) $body);
    $maxBlocks = max(1, min(32, (int) $maxBlocks));
    if ($body === '') return array();
 
    $lines = preg_split('/\n/', $body);
    if (!is_array($lines)) return array();
    $blocks = array();
    $current = array();
    $collecting = false;
    $flush = function (array &$blocks, array &$current, bool $collecting) use ($maxBlocks) {
      if (!$collecting || empty($current) || count($blocks) >= $maxBlocks) {
        $current = array();
        return;
      }
      $raw = trim(implode("\n", $current));
      $current = array();
      if ($raw === '') return;
      $header = self::extractLeadingQuotedHeaderBasic($raw);
      $text = trim((string) ($header['body'] ?? $raw));
      if ($text === '') return;
      $blocks[] = array(
        'from' => ($header['from'] ?? null),
        'date_raw' => ($header['date_raw'] ?? null),
        'subject' => ($header['subject'] ?? null),
        'message_id' => ($header['message_id'] ?? null),
        'in_reply_to' => ($header['in_reply_to'] ?? null),
        'references' => (array) ($header['references'] ?? array()),
        'text' => (dol_strlen($text) > 1200 ? dol_substr($text, 0, 1200) : $text),
      );
    };
 
    foreach ($lines as $line) {
      $trim = trim((string) $line);
      $isSeparator = (bool) preg_match('/^(?:-{2,}\s*)?(?:original message|mensaje original|mensagem original|forwarded message|message transfere|weitergeleitete nachricht)(?:\s*-{2,})?$/iu', trim($trim, '- '));
      $isWroteLine = (bool) preg_match('/^(?:on|el|em|le|am|op)\s+.{1,300}\s+(?:wrote|escribio|escreveu|a ecrit|schrieb|schreef)\s*:?$/iu', $trim);
      $isHeaderStart = (bool) preg_match('/^(?:from|de|da|von|van|sent|enviado|enviada|date|fecha|data)\s*:/iu', $trim);
      if ($isSeparator || $isWroteLine || ($isHeaderStart && !$collecting)) {
        $flush($blocks, $current, $collecting);
        $collecting = true;
        if (!$isSeparator) $current[] = preg_replace('/^>+\s*/', '', (string) $line);
        continue;
      }
      if ($collecting) {
        if ($isHeaderStart && !empty($current) && preg_match('/(?:^|\n)\s*$/', implode("\n", array_slice($current, -2)))) {
          $flush($blocks, $current, $collecting);
          $collecting = true;
        }
        $current[] = preg_replace('/^>+\s*/', '', (string) $line);
      }
    }
    $flush($blocks, $current, $collecting);
 
    return $blocks;
  }
 
  private static function extractLeadingQuotedHeaderBasic($raw)
  {
    $lines = preg_split('/\n/', self::normalizeText((string) $raw));
    if (!is_array($lines)) $lines = array((string) $raw);
    $headerLines = array();
    $bodyLines = array();
    $inHeader = true;
    foreach ($lines as $line) {
      $trim = trim((string) $line);
      if ($inHeader && ($trim === '' || preg_match('/^(?:from|de|da|von|van|sent|enviado|enviada|date|fecha|data|to|para|subject|asunto|assunto|objet|betreff|onderwerp|message-id|in-reply-to|references)\s*:/iu', $trim))) {
        $headerLines[] = $line;
        if ($trim === '' && count($headerLines) > 1) $inHeader = false;
        continue;
      }
      $inHeader = false;
      $bodyLines[] = $line;
    }
    $header = implode("\n", $headerLines);
    $get = function (array $names) use ($header) {
      foreach ((array) $names as $name) {
        if (preg_match('/^'.preg_quote((string) $name, '/').'\s*:\s*(.+)$/miu', $header, $m)) return trim((string) $m[1]);
      }
      return '';
    };
    $messageId = self::normalizeMessageId($get(array('Message-ID')));
    $inReplyTo = self::normalizeMessageId($get(array('In-Reply-To')));
    return array(
      'from' => ($get(array('From', 'De', 'Da', 'Von', 'Van')) ?: null),
      'date_raw' => ($get(array('Sent', 'Enviado', 'Enviada', 'Date', 'Fecha', 'Data')) ?: null),
      'subject' => ($get(array('Subject', 'Asunto', 'Assunto', 'Objet', 'Betreff', 'Onderwerp')) ?: null),
      'message_id' => ($messageId !== '' ? $messageId : null),
      'in_reply_to' => ($inReplyTo !== '' ? $inReplyTo : null),
      'references' => self::normalizeMessageIdList(self::extractHeaderMessageIds($get(array('References')))),
      'body' => trim(implode("\n", $bodyLines)),
    );
  }
 
  public static function extractReplyOnlyTextBasic($body)
  {
    $body = self::normalizeText((string) $body);
    if ($body === '') return '';
 
    $separators = array(
      '-----Original Message-----',
      '----- Original Message -----',
      '-----Mensaje original-----',
      '----- Mensaje original -----',
      '-----Mensagem original-----',
      '----- Mensagem original -----',
      '----- Forwarded message -----',
    );
    foreach ($separators as $sep) {
      $pos = stripos($body, $sep);
      if ($pos !== false && $pos > 0) {
        return trim(dol_substr($body, 0, $pos, '8bit'));
      }
    }
 
    $lines = preg_split("/\n/", $body);
    if (!is_array($lines)) return $body;
    $out = array();
    foreach ($lines as $line) {
      $trim = trim((string) $line);
      if ($trim === '') {
        if (!empty($out)) $out[] = '';
        continue;
      }
      if (preg_match('/^(from|de|sent|enviado|to|para|subject|asunto)\b[^:]{0,12}:/i', $trim)) break;
      if (preg_match('/^>+/', $trim)) break;
      $out[] = $line;
      if (dol_strlen(implode("\n", $out)) > 4000) break;
    }
 
    return trim(implode("\n", $out));
  }
 
  public static function extractQuotedThreadSnippetBasic($body, $maxLen = 1200)
  {
    $body = self::normalizeText((string) $body);
    $maxLen = (int) $maxLen;
    if ($body === '') return '';
    if ($maxLen <= 0) $maxLen = 1200;
 
    $startPos = false;
    foreach (array('original message', 'mensaje original', 'mensagem original', 'forwarded message') as $needle) {
      $p = stripos($body, $needle);
      if ($p !== false && ($startPos === false || $p < $startPos)) $startPos = $p;
    }
    if ($startPos === false) {
      $lines = preg_split("/\n/", $body);
      if (!is_array($lines)) return '';
      $quoteLines = array();
      foreach ($lines as $ln) {
        $t = trim((string) $ln);
        if ($t === '') continue;
        if (preg_match('/^>+/', $t)) {
          $quoteLines[] = preg_replace('/^>+\s*/', '', $t);
        }
      }
      $txt = trim(implode("\n", $quoteLines));
      return (dol_strlen($txt) > $maxLen ? dol_substr($txt, 0, $maxLen) : $txt);
    }
 
    $txt = trim(dol_substr($body, (int) $startPos, null, '8bit'));
    if (dol_strlen($txt) > $maxLen) $txt = dol_substr($txt, 0, $maxLen);
    return $txt;
  }
 
  private static function fallbackCleanBody($rawBody)
  {
    $text = self::normalizeText($rawBody);
    if ($text === '') return '';
 
    $lines = preg_split('/\n+/', $text);
    if (!is_array($lines)) return trim($text);
 
    $out = array();
    foreach ($lines as $line) {
      $line = trim((string) $line);
      if ($line === '') continue;
 
      $low = dol_string_nospecial($line);
      $low = dol_strtolower($low);
      if (preg_match('/^(from|de|sent|enviado|to|para|subject|asunto)\b[^:]{0,12}:/i', $line)) break;
      if (strpos($low, 'original message') !== false) break;
      if (strpos($low, 'mensaje original') !== false) break;
      if (strpos($low, 'forwarded message') !== false) break;
      if (preg_match('/^>+/', $line)) continue;
      if (preg_match('/\b(confidential|privacidad|confidencialidad)\b/i', $line)) continue;
 
      $out[] = $line;
      if (dol_strlen(implode("\n", $out)) > 6000) break;
    }
 
    $clean = trim(implode("\n", $out));
    if ($clean === '') {
      $clean = trim(dol_substr($text, 0, 2000));
    }
 
    return $clean;
  }
 
  private static function decodeJsonFromAiString($raw)
  {
    $raw = trim((string) $raw);
    if ($raw === '') return null;
 
    $decoded = json_decode($raw, true);
    if (is_array($decoded)) return $decoded;
 
    $first = strpos($raw, '{');
    $last = strrpos($raw, '}');
    if ($first === false || $last === false || $last <= $first) return null;
 
    $payload = dol_substr($raw, $first, $last - $first + 1, '8bit');
    $decoded = json_decode($payload, true);
    if (!is_array($decoded)) return null;
 
    return $decoded;
  }
}