dolibarr 25.0.0-alpha
ai.class.php
Go to the documentation of this file.
1<?php
2/* Copyright (C) 2024 Laurent Destailleur <eldy@users.sourceforge.net>
3 * Copyright (C) 2024 Frédéric France <frederic.france@free.fr>
4 * Copyright (C) 2024-2026 MDW <mdeweerd@users.noreply.github.com>
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 3 of the License, or
9 * (at your option) any later version.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program. If not, see <https://www.gnu.org/licenses/>.
18 * or see https://www.gnu.org/
19 */
20
27require_once DOL_DOCUMENT_ROOT."/core/lib/admin.lib.php";
28require_once DOL_DOCUMENT_ROOT.'/core/lib/geturl.lib.php';
29require_once DOL_DOCUMENT_ROOT.'/core/lib/date.lib.php';
30require_once DOL_DOCUMENT_ROOT."/ai/lib/ai.lib.php";
31
32
36class Ai
37{
41 protected $db;
42
46 private $apiService;
47
51 private $apiKey;
52
56 private $apiEndpoint;
57
61 public $tokenUsage;
62
63
64 const AI_DEFAULT_PROMPT_FOR_EMAIL = 'You are an email editor. Return only the content of the message. Do not add explanation.'; // Note: This instruction will also be completed by generateContent() to manage text versus HTML content.
65 const AI_DEFAULT_PROMPT_FOR_WEBPAGE = 'You are a website editor. Return all HTML content inside a section tag. Do not add explanation.';
66 const AI_DEFAULT_PROMPT_FOR_TEXT_TRANSLATION = 'You are a translator, answer with one and only one translation with no comment and explanation.';
67 const AI_DEFAULT_PROMPT_FOR_TEXT_SUMMARIZE = 'You are a writer, make the answer in the same language than the original text to summarize.';
68 const AI_DEFAULT_PROMPT_FOR_TEXT_SPELLCHECKER = 'You are a proofreader, write your response in the same language as the original text in order to correct spelling and grammar errors. If there is carriage return or line feed in original message, keep them. Keep also any HTML or markdown formatting without changing it or adding one, just fix spelling and grammar errors in text content. Answer with the corrected text and the original HTML formatting if there was, with no additional comment and explanation even to highlight the fixed errors.';
69 const AI_DEFAULT_PROMPT_FOR_TEXT_REPHRASER = 'You are a writer, write your response in the same language as the original text to rephrase. Give only one answer with no comment and explanation. If there is carriage return or line feed in original message, keep them. Keep also any HTML or markdown formatting without adding one.';
70 const AI_DEFAULT_PROMPT_FOR_EXTRAFIELD_FILLER = 'Give only one answer with no comment and explanation, I want the text to be ready to copy and paste.';
71 const AI_DEFAULT_PROMPT_FOR_DOC_PARSING = 'You are an Assistant to analyze documents. Return your answer with a JSON string and only a JSON string, do not add any other comment. The JSON must be valid: values must be literals, never expressions or calculations, and no comments.';
72
73
79 public function __construct($db)
80 {
81 $this->db = $db;
82
83 // Get API key according to enabled AI
84 $this->apiService = getDolGlobalString('AI_API_SERVICE', 'chatgpt');
85 $this->apiKey = getDolGlobalString('AI_API_'.strtoupper($this->apiService).'_KEY');
86 }
87
93 public function getApiService()
94 {
95 return $this->apiService;
96 }
97
109 public function generateContent($instructions, $model = 'auto', $function = 'textgeneration', $format = '', $moreheaders = array(), $moreendpoint = '')
110 {
111 global $dolibarr_main_data_root;
112
113 $arrayofai = getListOfAIServices();
114
115 // TODO Can store the need for a key into array returned by getListOfAIServices()
116 if (empty($this->apiKey) && in_array($this->apiService, array('chatgpt', 'groq', 'mistral'))) {
117 return array('error' => true, 'message' => 'API key is not defined for the AI enabled service ('.$this->apiService.')');
118 }
119
120 // $this->apiEndpoint is already set here only if it was previously forced.
121
122 if (empty($this->apiEndpoint) && $this->apiService == 'custom' && !getDolGlobalString('AI_API_CUSTOM_URL')) {
123 return array('error' => true, 'message' => 'API URL is not defined for the AI enabled service ('.$this->apiService.')');
124 }
125
126 // In most cases, it is empty and we must get it from $function and $this->apiService
127 if (empty($this->apiEndpoint)) {
128 // Return the endpoint from $this->apiService.
129 if ($function == 'imagegeneration') {
130 $this->apiEndpoint = getDolGlobalString('AI_API_'.strtoupper($this->apiService).'_URL', $arrayofai[$this->apiService]['url']);
131 $this->apiEndpoint .= (preg_match('/\/$/', $this->apiEndpoint) ? '' : '/').'images/generations';
132 } elseif ($function == 'audiogeneration') {
133 $this->apiEndpoint = getDolGlobalString('AI_API_'.strtoupper($this->apiService).'_URL', $arrayofai[$this->apiService]['url']);
134 $this->apiEndpoint .= (preg_match('/\/$/', $this->apiEndpoint) ? '' : '/').'audio/speech';
135 } elseif ($function == 'transcription') {
136 $this->apiEndpoint = getDolGlobalString('AI_API_'.strtoupper($this->apiService).'_URL', $arrayofai[$this->apiService]['url']);
137 $this->apiEndpoint .= (preg_match('/\/$/', $this->apiEndpoint) ? '' : '/').'transcriptions';
138 } elseif ($function == 'file') {
139 $this->apiEndpoint = getDolGlobalString('AI_API_'.strtoupper($this->apiService).'_URL', $arrayofai[$this->apiService]['url']);
140 $this->apiEndpoint .= (preg_match('/\/$/', $this->apiEndpoint) ? '' : '/').'files';
141 } elseif ($function == 'assistant') {
142 $this->apiEndpoint = getDolGlobalString('AI_API_'.strtoupper($this->apiService).'_URL', $arrayofai[$this->apiService]['url']);
143 $this->apiEndpoint .= (preg_match('/\/$/', $this->apiEndpoint) ? '' : '/').'assistants';
144 } elseif ($function == 'thread') {
145 $this->apiEndpoint = getDolGlobalString('AI_API_'.strtoupper($this->apiService).'_URL', $arrayofai[$this->apiService]['url']);
146 $this->apiEndpoint .= (preg_match('/\/$/', $this->apiEndpoint) ? '' : '/').'threads';
147 } elseif ($function == 'ocr') {
148 $this->apiEndpoint = getDolGlobalString('AI_API_'.strtoupper($this->apiService).'_URL', $arrayofai[$this->apiService]['url']);
149 $this->apiEndpoint .= (preg_match('/\/$/', $this->apiEndpoint) ? '' : '/').'ocr';
150 } else { // if $function == 'docparsing', 'text...', ...
151 $this->apiEndpoint = getDolGlobalString('AI_API_'.strtoupper($this->apiService).'_URL', $arrayofai[$this->apiService]['url']);
152 if ($this->apiService == 'google') {
153 // Google Gemini native API: the /models/<model>:generateContent suffix is
154 // appended later (once $model has been resolved). The OpenAI-style
155 // /chat/completions does not exist on the native Gemini endpoint.
156 $this->apiEndpoint = rtrim($this->apiEndpoint, '/');
157 } else {
158 $this->apiEndpoint .= (preg_match('/\/$/', $this->apiEndpoint) ? '' : '/').'chat/completions';
159 }
160 }
161 }
162 if ($moreendpoint) {
163 $this->apiEndpoint .= '/'.$moreendpoint;
164 }
165
166
167 // $model may be undefined or 'auto'.
168 // If this is the case, we must get it from $function and $this->apiService
169 if (empty($model) || $model == 'auto') {
170 // Return the model from $this->apiService.
171 if (in_array($function, array('file', 'assistant', 'thread'))) {
172 $model = '';
173 } elseif ($function == 'imagegeneration') {
174 $model = getDolGlobalString('AI_API_'.strtoupper($this->apiService).'_MODEL_IMAGE', $arrayofai[$this->apiService][$function]['default']);
175 } elseif ($function == 'audiogeneration') {
176 $model = getDolGlobalString('AI_API_'.strtoupper($this->apiService).'_MODEL_AUDIO', $arrayofai[$this->apiService][$function]['default']);
177 } elseif ($function == 'transcription') {
178 $model = getDolGlobalString('AI_API_'.strtoupper($this->apiService).'_MODEL_TRANSCRIPT', $arrayofai[$this->apiService][$function]['default']);
179 } elseif ($function == 'translation') {
180 $model = getDolGlobalString('AI_API_'.strtoupper($this->apiService).'_MODEL_TRANSLATE', $arrayofai[$this->apiService][$function]['default']);
181 } elseif (in_array($function, array('docparsing', 'ocr'))) {
182 $model = getDolGlobalString('AI_API_'.strtoupper($this->apiService).'_MODEL_DOCPARSING', $arrayofai[$this->apiService]['docparsing']['default']);
183 } else {
184 // else 'textgenerationemail', 'textgenerationwebpage', 'textgeneration', 'texttranslation', 'textsummarize', 'textrephraser', 'textspellchecker', ...
185 $model = getDolGlobalString('AI_API_'.strtoupper($this->apiService).'_MODEL_TEXT', $arrayofai[$this->apiService]['textgeneration']['default']);
186 }
187 }
188
189 // Google Gemini: append /models/<model>:generateContent now that $model is resolved.
190 if ($this->apiService == 'google' && !in_array($function, array('file', 'assistant', 'thread', 'ocr'))
191 && strpos($this->apiEndpoint, ':generateContent') === false) {
192 $this->apiEndpoint .= '/models/'.rawurlencode($model).':generateContent';
193 }
194
195 dol_syslog("Call API for apiKey=".substr($this->apiKey, 0, 5).'***********, apiEndpoint='.$this->apiEndpoint.", model=".$model.", format=".$format);
196 if (getDolGlobalString("AI_DEBUG")) {
197 if (@is_writable($dolibarr_main_data_root)) { // Avoid fatal error on fopen with open_basedir
198 $outputfile = $dolibarr_main_data_root."/dolibarr_ai.log";
199 $fp = fopen($outputfile, "w"); // overwrite
200
201 if ($fp) {
202 fwrite($fp, "Call API for apiKey=".substr($this->apiKey, 0, 5).'***********, apiEndpoint='.$this->apiEndpoint.", model=".$model.", format=".$format."\n");
203 fclose($fp);
204 dolChmod($outputfile);
205 }
206 }
207 }
208
209 $response = null;
210
211 try {
212 if (empty($this->apiEndpoint)) {
213 throw new Exception('The AI service '.$this->apiService.' is not yet supported for the type of request '.$function);
214 }
215
216 $configurationsJson = getDolGlobalString('AI_CONFIGURATIONS_PROMPT');
217 $configurations = json_decode($configurationsJson, true);
218
219 $prePrompt = '';
220 $postPrompt = '';
221
222 if (isset($configurations[$function])) {
223 if (isset($configurations[$function]['prePrompt'])) {
224 $prePrompt = $configurations[$function]['prePrompt'];
225 }
226
227 if (isset($configurations[$function]['postPrompt'])) {
228 $postPrompt = $configurations[$function]['postPrompt'];
229 }
230 }
231 //var_dump($prePrompt);
232
233 // Get the default value of prePrompt if not defined
234 if (empty($prePrompt) && $function == 'textgenerationemail') {
235 $prePrompt = self::AI_DEFAULT_PROMPT_FOR_EMAIL;
236 if ($format === 'html') {
237 $prePrompt .= ' Return all HTML content inside a section tag';
238 } else {
239 $prePrompt .= ' Return content in UTF8 text. Use Linux carriage return if you need to split a line. Do not include any HTML tag neither HTML entities.';
240 }
241 }
242 if (empty($prePrompt) && $function == 'textgenerationwebpage') {
243 $prePrompt = self::AI_DEFAULT_PROMPT_FOR_WEBPAGE;
244 }
245 if (empty($prePrompt) && $function == 'textgenerationextrafield') {
246 $prePrompt = self::AI_DEFAULT_PROMPT_FOR_EXTRAFIELD_FILLER;
247 }
248 if (empty($prePrompt) && $function == 'texttranslation') {
249 $prePrompt = self::AI_DEFAULT_PROMPT_FOR_TEXT_TRANSLATION;
250 }
251 if (empty($prePrompt) && $function == 'textsummarize') {
252 $prePrompt = self::AI_DEFAULT_PROMPT_FOR_TEXT_SUMMARIZE;
253 }
254 if (empty($prePrompt) && $function == 'textrephraser') {
255 $prePrompt = self::AI_DEFAULT_PROMPT_FOR_TEXT_REPHRASER;
256 }
257 if (empty($prePrompt) && $function == 'textspellchecker') {
258 $prePrompt = self::AI_DEFAULT_PROMPT_FOR_TEXT_SPELLCHECKER;
259 }
260 if (empty($prePrompt) && $function == 'docparsing') {
261 $prePrompt = self::AI_DEFAULT_PROMPT_FOR_DOC_PARSING;
262 }
263
264 if (is_array($instructions)) {
265 $arrayforpayload = $instructions;
266 $fullInstructions = '';
267 } else {
268 $fullInstructions = $instructions.($postPrompt ? (preg_match('/[\.\!\?]$/', $instructions) ? '' : '.').' '.$postPrompt : '');
269
270 // Set payload string
271 /*{
272 "messages": [
273 {
274 "content": "You are a helpful assistant.",
275 "role": "system"
276 },
277 {
278 "content": "Hello!",
279 "role": "user"
280 }
281 ],
282 "model": "tinyllama-1.1b",
283 "stream": true,
284 "max_tokens": 2048,
285 "stop": [
286 "hello"
287 ],
288 "frequency_penalty": 0,
289 "presence_penalty": 0,
290 "temperature": 0.7,
291 "top_p": 0.95
292 }*/
293
294 // Add a system message
295 $addDateTimeContext = false;
296 if ($addDateTimeContext) { // @phpstan-ignore-line
297 $prePrompt = ($prePrompt ? $prePrompt.(preg_match('/[\.\!\?]$/', $prePrompt) ? '' : '.').' ' : '').'Today we are '.dol_print_date(dol_now(), 'dayhourtext');
298 }
299
300 if ($this->apiService == 'google') {
301 // Google Gemini native payload format (different from OpenAI's "messages").
302 $arrayforpayload = array(
303 'contents' => array(
304 array('role' => 'user', 'parts' => array(array('text' => $fullInstructions)))
305 )
306 );
307 if ($prePrompt) {
308 $arrayforpayload['system_instruction'] = array(
309 'parts' => array(array('text' => $prePrompt))
310 );
311 }
312 } else {
313 // OpenAI-compatible payload format (chatgpt, mistral, groq, anthropic-compat, custom, ...)
314 $arrayforpayload = array(
315 'messages' => array(array('role' => 'user', 'content' => $fullInstructions)),
316 'model' => $model,
317 );
318 if ($prePrompt) {
319 $arrayforpayload['messages'][] = array('role' => 'system', 'content' => $prePrompt);
320 }
321 }
322 }
323
324 // For document parsing, ask the services that support it to return only a valid JSON
325 if ($function == 'docparsing' && !is_array($instructions)) {
326 if ($this->apiService == 'google') {
327 $arrayforpayload['generationConfig'] = array('responseMimeType' => 'application/json');
328 } elseif (in_array($this->apiService, array('chatgpt', 'groq', 'mistral', 'deepseek'))) {
329 $arrayforpayload['response_format'] = array('type' => 'json_object');
330 }
331 }
332
333 /*
334 $arrayforpayload['temperature'] = 0.7;
335 $arrayforpayload['max_tokens'] = -1;
336 $arrayforpayload['stream'] = false;
337 */
338
339 if ($function == 'thread') {
340 $payload = $instructions;
341 } else {
342 $payload = json_encode($arrayforpayload);
343 }
344
345 if ($this->apiService == 'google') {
346 // Google Gemini uses the x-goog-api-key header (Bearer is not accepted by the native API).
347 $headers = array(
348 'x-goog-api-key: ' . $this->apiKey,
349 );
350 } else {
351 $headers = array(
352 'Authorization: Bearer ' . $this->apiKey,
353 );
354 }
355 if ($function != 'file') {
356 $headers[] = 'Content-Type: application/json';
357 }
358 if (!empty($moreheaders)) {
359 foreach ($moreheaders as $morekey => $moreval) {
360 $headers[] = $morekey.': '.$moreval;
361 }
362 }
363
364 if (getDolGlobalString("AI_DEBUG")) {
365 if (@is_writable($dolibarr_main_data_root)) { // Avoid fatal error on fopen with open_basedir
366 $outputfile = $dolibarr_main_data_root."/dolibarr_ai.log";
367 $fp = fopen($outputfile, "a");
368
369 if ($fp) {
370 if ($function == 'docparsing' || $function == 'ocr') {
371 fwrite($fp, "Call endpoint ".$this->apiEndpoint." with POST and the following file to upload:\n");
372 fwrite($fp, (is_string($instructions) ? $instructions : var_export($instructions, true))."\n");
373 } else {
374 fwrite($fp, "Call endpoint ".$this->apiEndpoint." with POST and the following message:\n");
375 fwrite($fp, $fullInstructions."\n");
376 fwrite($fp, "And prepompt:\n");
377 fwrite($fp, $prePrompt."\n");
378 }
379 fwrite($fp, "HTTP Header\n");
380 fwrite($fp, var_export($headers, true)."\n");
381 fwrite($fp, "Payload\n");
382 fwrite($fp, var_export($payload, true)."\n");
383
384 fclose($fp);
385 dolChmod($outputfile);
386 }
387 }
388 }
389
390 // By default, we accept only external endpoints ($dolibarr_ai_allow_local_endpoints is not set).
391 // To allow local endpoints, we must set $dolibarr_ai_allow_local_endpoints to 1 or 2 in conf.php.
392 global $dolibarr_ai_allow_local_endpoints;
393 $localurl = $dolibarr_ai_allow_local_endpoints ?? 0;
394
395 // Parsing a document is long, so we use a response timeout of at least 90 seconds for this case
396 $timeoutresponse = 0;
397 if (in_array($function, array('docparsing', 'ocr'))) {
398 $timeoutresponse = max(90, getDolGlobalInt('MAIN_USE_RESPONSE_TIMEOUT', 30));
399 }
400
401 $response = getURLContent($this->apiEndpoint, 'POST', $payload, 1, $headers, array('http', 'https'), $localurl, -1, 0, $timeoutresponse);
402
403 if (empty($response['http_code'])) {
404 throw new Exception('API request failed. No http received'.(empty($response['curl_error_no']) ? '' : ' (curl error '.$response['curl_error_no'].': '.$response['curl_error_msg'].')'));
405 }
406 if (!empty($response['http_code']) && $response['http_code'] != 200) {
407 if (in_array($response['http_code'], array(400, 401, 403, 429)) && !empty($response['content'])) {
408 $tmp = json_decode($response['content'], true);
409 if (!empty($tmp['message'])) {
410 return array(
411 'error' => true,
412 'message' => $tmp['message'],
413 'code' => (empty($response['http_code']) ? 0 : $response['http_code']),
414 'curl_error_no' => (empty($response['curl_error_no']) ? 0 : $response['curl_error_no']),
415 'format' => $format,
416 'service' => $this->apiService,
417 'model' => $model,
418 'function' => $function
419 );
420 }
421 }
422 throw new Exception('API request on AI endpoint '.$this->apiEndpoint.' failed with status code '.$response['http_code']);
423 }
424
425 if (getDolGlobalString("AI_DEBUG")) {
426 if (@is_writable($dolibarr_main_data_root)) { // Avoid fatal error on fopen with open_basedir
427 $outputfile = $dolibarr_main_data_root."/dolibarr_ai.log";
428 $fp = fopen($outputfile, "a");
429
430 if ($fp) {
431 fwrite($fp, "Answer\n");
432 fwrite($fp, var_export((empty($response['content']) ? 'No content result' : $response['content']), true)."\n");
433
434 fclose($fp);
435 dolChmod($outputfile);
436 }
437 }
438 }
439
440
441 // Decode JSON response
442 $decodedResponse = json_decode($response['content'], true);
443
444 // Extraction content
445 if (!empty($decodedResponse['error'])) {
446 if (is_scalar($decodedResponse['error'])) {
447 // Casts needed because is_scalar() also accepts bool, int and float, and
448 // var_export() is typed as returning null when its second argument is not
449 // true, while the method is declared to return a string or the error array.
450 $generatedContent = (string) $decodedResponse['error'];
451 } else {
452 $generatedContent = (string) var_export($decodedResponse['error'], true);
453 }
454 } elseif ($function == 'ocr' && !empty($decodedResponse['pages']) && is_array($decodedResponse['pages'])) {
455 // Response of the OCR endpoint (Mistral /v1/ocr): the text extracted from each page is into pages[].markdown
456 $generatedContent = '';
457 foreach ($decodedResponse['pages'] as $page) {
458 if (isset($page['markdown']) && is_scalar($page['markdown'])) {
459 $generatedContent .= ($generatedContent ? "\n" : '').(string) $page['markdown'];
460 }
461 }
462 } elseif ($this->apiService == 'google') {
463 // Google Gemini response shape: candidates[0].content.parts[*].text
464 // (parts is an array because Gemini can return mixed-modality output;
465 // we concatenate the textual parts.)
466 $generatedContent = '';
467 if (!empty($decodedResponse['candidates'][0]['content']['parts'])) {
468 foreach ($decodedResponse['candidates'][0]['content']['parts'] as $part) {
469 if (isset($part['text']) && is_scalar($part['text'])) {
470 $generatedContent .= (string) $part['text'];
471 }
472 }
473 }
474 } else {
475 // A provider answering 200 with a body that does not carry
476 // choices[0].message.content must not reach the code below: reading the missing
477 // keys emits PHP warnings, and a 'choices' entry that is a string instead of an
478 // array raises a TypeError, which the catch below does not take (TypeError
479 // implements Throwable but does not extend Exception) and so ends as a fatal
480 // error on the ajax endpoints calling this method.
481 // Reasoning models (Mistral, ...) may return content as an array of chunks
482 // ({"type":"thinking",...}, {"type":"text","text":"..."}): we keep only the text chunks.
483 if (isset($decodedResponse['choices'][0]['message']['content'])
484 && is_array($decodedResponse['choices'][0]['message']['content'])) {
485 $textofchunks = '';
486 $foundtextchunk = false;
487 foreach ($decodedResponse['choices'][0]['message']['content'] as $chunk) {
488 if (is_array($chunk) && ($chunk['type'] ?? '') == 'text' && isset($chunk['text']) && is_string($chunk['text'])) {
489 $textofchunks .= $chunk['text'];
490 $foundtextchunk = true;
491 }
492 }
493 if ($foundtextchunk) {
494 $decodedResponse['choices'][0]['message']['content'] = $textofchunks;
495 }
496 }
497 if (!isset($decodedResponse['choices'][0]['message']['content'])
498 || !is_string($decodedResponse['choices'][0]['message']['content'])) {
499 throw new Exception('API response of AI endpoint '.$this->apiEndpoint.' does not contain the expected choices[0].message.content string');
500 }
501 $generatedContent = $decodedResponse['choices'][0]['message']['content'];
502 }
503 // All the branches above assign a string, but the inference of the analyser does not
504 // follow them through the enclosing try block, so the type is pinned here before the
505 // value is passed to functions declared to take a string.
506 $generatedContent = (string) $generatedContent;
507
508 dol_syslog("ai->generatedContent returned: ".dol_trunc($generatedContent, 50));
509
510 $this->tokenUsage = $decodedResponse['usage'] ?? null;
511
512 // If content is not HTML, we convert it into HTML
513 if ($format == 'html') {
514 if (!dol_textishtml($generatedContent)) {
515 dol_syslog("Result was detected as not HTML so we convert it into HTML.");
516 $generatedContent = dol_nl2br($generatedContent);
517 } else {
518 dol_syslog("Result was detected as already HTML. Do nothing.");
519 }
520
521 // TODO If content is for website module, we must
522 // - clan html header, keep body only and remove ``` ticks added by AI
523 // - add tags <section contenEditable="true"> </section>
524 }
525
526 return $generatedContent;
527 } catch (Exception $e) {
528 $errormessage = $e->getMessage();
529 $errormessagelog = $e->getMessage();
530 if (!empty($response['content'])) {
531 $decodedResponse = json_decode($response['content'], true);
532 $errormessagelog .= ' - '.$response['content'];
533
534 if (!empty($decodedResponse['error']['message'])) {
535 // With OpenAI, error is into an object error into the content
536 $errormessage .= ' - '.$decodedResponse['error']['message'];
537 } else {
538 $errormessage .= ' - '.$response['content'];
539 }
540 }
541
542 if (getDolGlobalString("AI_DEBUG")) {
543 if (@is_writable($dolibarr_main_data_root)) { // Avoid fatal error on fopen with open_basedir
544 $outputfile = $dolibarr_main_data_root."/dolibarr_ai.log";
545 $fp = fopen($outputfile, "a");
546
547 if ($fp) {
548 fwrite($fp, "Error: ".$errormessagelog."\n");
549
550 fclose($fp);
551 dolChmod($outputfile);
552 }
553 }
554 }
555
556 return array(
557 'error' => true,
558 'message' => $errormessage,
559 'code' => (empty($response['http_code']) ? 0 : $response['http_code']),
560 'curl_error_no' => (empty($response['curl_error_no']) ? 0 : $response['curl_error_no']),
561 'format' => $format,
562 'service' => $this->apiService,
563 'model' => $model,
564 'function' => $function
565 );
566 }
567 }
568
579 public function getPayloadToConvertRawFileIntoText($filepath)
580 {
581 // Only the OpenAI-compatible services support the raw file mode
582 if (!in_array($this->apiService, array('chatgpt', 'mistral', 'groq', 'deepseek', 'perplexity', 'zai', 'custom'))) {
583 return array('error' => true, 'message' => 'The AI service '.$this->apiService.' does not support sending a raw file');
584 }
585
586 $isimage = preg_match('/\.(png|jpe?g|gif|bmp|tiff?|webp)$/i', $filepath);
587 $ispdf = preg_match('/\.pdf$/i', $filepath);
588 if (!$isimage && !$ispdf) {
589 return array('error' => true, 'message' => 'The AI service '.$this->apiService.' does not support sending a raw file with this file type (only PDF and image files are supported)');
590 }
591
592 $arrayofai = getListOfAIServices();
593 $model = getDolGlobalString('AI_API_'.strtoupper($this->apiService).'_MODEL_DOCPARSING', (empty($arrayofai[$this->apiService]['docparsing']['default']) ? '' : $arrayofai[$this->apiService]['docparsing']['default']));
594 if (empty($model) || $model == 'na') {
595 return array('error' => true, 'message' => 'No AI model defined to parse documents for the service '.$this->apiService);
596 }
597
598 $mimetype = dol_mimetype($filepath);
599 $filecontentb64 = base64_encode((string) file_get_contents(dol_osencode($filepath)));
600
601 if ($this->apiService == 'mistral') {
602 // Mistral OCR endpoint: it accepts images and PDF files as a base64 into a data URI
603 // (into a image_url part for images, a document_url part for PDF) and returns the text of each page as markdown
604 $typedoc = ($isimage ? 'image_url' : 'document_url');
605 $payload = array(
606 'model' => $model,
607 'document' => array('type' => $typedoc, $typedoc => 'data:'.$mimetype.';base64,'.$filecontentb64)
608 );
609 return array('payload' => $payload, 'function' => 'ocr');
610 }
611
612 // Other OpenAI-compatible services: multimodal chat completion with the file into base64
613 if ($isimage) {
614 $filepart = array('type' => 'image_url', 'image_url' => array('url' => 'data:'.$mimetype.';base64,'.$filecontentb64));
615 } else {
616 $filepart = array('type' => 'file', 'file' => array('filename' => dol_basename($filepath), 'file_data' => 'data:'.$mimetype.';base64,'.$filecontentb64));
617 }
618 $payload = array(
619 'messages' => array(
620 array('role' => 'user', 'content' => array(
621 array('type' => 'text', 'text' => 'Convert the content of this document into text. Return only the text of the document, no comment.'),
622 $filepart
623 ))
624 ),
625 'model' => $model
626 );
627 return array('payload' => $payload, 'function' => 'docparsing');
628 }
629
637 public function decodeJsonIntoArray($json, $type)
638 {
639 $tmparray = array();
640
641 //var_dump($json['items']);
642 if ($type == 'supplier_invoice') {
643 // The AI service may put the invoice data into another section than document_info, so we complete document_info with them
644 foreach (array('invoice_details', 'invoice_info', 'invoice') as $keysection) {
645 if (!empty($json[$keysection]) && is_array($json[$keysection])) {
646 $json['document_info'] = array_merge($json[$keysection], (isset($json['document_info']) && is_array($json['document_info'])) ? $json['document_info'] : array());
647 }
648 }
649
650 // Invoice info
651 if (!empty($json['document_info']['reference'])) {
652 $tmparray['supplierref'] = $json['document_info']['reference'];
653 } elseif (!empty($json['document_info']['invoice_number'])) {
654 $tmparray['supplierref'] = $json['document_info']['invoice_number'];
655 } elseif (!empty($json['document_info']['document_ref'])) {
656 $tmparray['supplierref'] = $json['document_info']['document_ref'];
657 }
658
659 if (!empty($json['document_info']['title'])) {
660 $tmparray['invoice_label'] = $json['document_info']['title'];
661 } elseif (!empty($json['recipient']['description'])) {
662 $tmparray['invoice_label'] = $json['recipient']['description'];
663 } elseif (!empty($json['document_info']['invoice_reference'])) {
664 $tmparray['invoice_label'] = $json['document_info']['invoice_reference'];
665 }
666
667 // Issue date
668 if (!empty($json['document_info']['issue_date']) && preg_match('/^[0-9]{4}-[0-9]{2}-[0-9]{2}((\s|T)[0-9]{2}:[0-9]{2}:[0-9]{2}Z?)?$/', $json['document_info']['issue_date'])) {
669 $tmparray['issue_date'] = dol_stringtotime($json['document_info']['issue_date'], 'tzuserrel');
670 } elseif (!empty($json['document_info']['invoice_date']) && preg_match('/^[0-9]{4}-[0-9]{2}-[0-9]{2}((\s|T)[0-9]{2}:[0-9]{2}:[0-9]{2}Z?)?$/', $json['document_info']['invoice_date'])) {
671 $tmparray['issue_date'] = dol_stringtotime($json['document_info']['invoice_date'], 'tzuserrel');
672 } elseif (!empty($json['document_info']['submission_date']) && preg_match('/^[0-9]{4}-[0-9]{2}-[0-9]{2}((\s|T)[0-9]{2}:[0-9]{2}:[0-9]{2}Z?)?$/', $json['document_info']['submission_date'])) {
673 $tmparray['issue_date'] = dol_stringtotime($json['document_info']['submission_date'], 'tzuserrel');
674 } elseif (!empty($json['document_info']['date']) && preg_match('/^[0-9]{4}-[0-9]{2}-[0-9]{2}((\s|T)[0-9]{2}:[0-9]{2}:[0-9]{2}Z?)?$/', $json['document_info']['date'])) {
675 $tmparray['issue_date'] = dol_stringtotime($json['document_info']['date'], 'tzuserrel');
676 }
677
678 // Due date
679 if (!empty($json['document_info']['due_date']) && preg_match('/^([0-9]{4})-([0-9]{2})-([0-9]{2})$/', $json['document_info']['due_date'])) {
680 $tmparray['due_date'] = dol_stringtotime($json['document_info']['due_date'], 'tzuserrel');
681 }
682
683 // Currency
684 // Note: We extract subarrays into local variables, so the is_string() checks do not narrow the type of the $json subarrays for the rest of the method
685 $summary = (is_array($json['summary'] ?? null) ? $json['summary'] : array());
686 $documentinfo = (is_array($json['document_info'] ?? null) ? $json['document_info'] : array());
687 $currencyinsummary = (isset($summary['currency']) && is_string($summary['currency'])) ? $summary['currency'] : '';
688 $currencyindocinfo = (isset($documentinfo['currency_code']) && is_string($documentinfo['currency_code'])) ? $documentinfo['currency_code'] : '';
689 if ($currencyinsummary == '€') {
690 $tmparray['currency_code'] = 'EUR';
691 } elseif (dol_strlen($currencyinsummary) == 3) {
692 $tmparray['currency_code'] = $currencyinsummary;
693 } elseif (dol_strlen($currencyindocinfo) == 3) {
694 $tmparray['currency_code'] = $currencyindocinfo;
695 }
696
697 // Note
698 if (!empty($json['notes'])) {
699 if (is_scalar($json['notes'])) {
700 $tmparray['note_public'] = $json['notes'];
701 } elseif (is_array($json['notes'])) {
702 // Loop on each note
703 $tmparray['note_public'] = '';
704 foreach ($json['notes'] as $val) {
705 if (is_scalar($val)) {
706 $tmparray['note_public'] = dol_concat($tmparray['note_public'], $val);
707 } elseif (is_array($val)) {
708 foreach ($val as $val2) {
709 if (is_scalar($val2)) {
710 $tmparray['note_public'] = dol_concat($tmparray['note_public'], $val2);
711 }
712 }
713 }
714 }
715 }
716 }
717
718 // Vendor
719 if (!empty($json['document_info']['vendor'])) {
720 $arrayforthirdparty = $json['document_info']['vendor'];
721 } elseif (!empty($json['vendor'])) {
722 $arrayforthirdparty = $json['vendor'];
723 } elseif (!empty($json['issuer'])) {
724 $arrayforthirdparty = $json['issuer'];
725 }
726 if (!empty($arrayforthirdparty)) {
727 if (!empty($arrayforthirdparty['name'])) {
728 $tmparray['vendor_name'] = $arrayforthirdparty['name'];
729 }
730 if (!empty($arrayforthirdparty['siren'])) {
731 $tmparray['vendor_profid1'] = $arrayforthirdparty['siren'];
732 }
733 if (!empty($arrayforthirdparty['siret'])) {
734 $tmparray['vendor_profid2'] = $arrayforthirdparty['siret'];
735 }
736 if (!empty($arrayforthirdparty['email'])) {
737 $tmparray['vendor_email'] = $arrayforthirdparty['email'];
738 }
739 if (!empty($arrayforthirdparty['professional_id'])) {
740 $tmparray['vendor_profid1'] = $arrayforthirdparty['professional_id']['siren'];
741 }
742 if (!empty($arrayforthirdparty['vat_number'])) {
743 $tmparray['vendor_vat_number'] = $arrayforthirdparty['vat_number'];
744 } elseif (!empty($arrayforthirdparty['vatnumber'])) {
745 $tmparray['vendor_vat_number'] = $arrayforthirdparty['vatnumber'];
746 } elseif (!empty($arrayforthirdparty['tva_num'])) {
747 $tmparray['vendor_vat_number'] = $arrayforthirdparty['tva_num'];
748 }
749 if (!empty($arrayforthirdparty['address'])) {
750 if (is_array($arrayforthirdparty['address'])) {
751 // The address may be returned as an array with its parts separated
752 $vendoraddress = $arrayforthirdparty['address'];
753 if (!empty($vendoraddress['street'])) {
754 $tmparray['vendor_address'] = $vendoraddress['street'];
755 }
756 if (!empty($vendoraddress['postal_code'])) {
757 $tmparray['vendor_zip'] = $vendoraddress['postal_code'];
758 }
759 if (!empty($vendoraddress['city'])) {
760 $tmparray['vendor_town'] = $vendoraddress['city'];
761 }
762 if (!empty($vendoraddress['country'])) {
763 $tmparray['vendor_country'] = $vendoraddress['country'];
764 }
765 } else {
766 $tmparray['vendor_address'] = $arrayforthirdparty['address'];
767 }
768 }
769 if (!empty($arrayforthirdparty['phone'])) {
770 $tmparray['vendor_phone'] = $arrayforthirdparty['phone'];
771 } elseif (!empty($arrayforthirdparty['phone_number'])) {
772 $tmparray['vendor_phone'] = $arrayforthirdparty['phone_number'];
773 }
774 }
775
776 // Items
777 if (empty($json['items'])) {
778 if (!empty($json['summary']['subtotal_excluding_tax'])) {
779 $tmparray['description'] = 'Undefined';
780 $tmparray['total_ht'] = (float) $json['summary']['subtotal_excluding_tax'];
781 $tmparray['vat_rate'] = (float) $json['summary']['tax']['rate'];
782 }
783 } else {
784 $i = 0;
785 foreach ($json['items'] as $item) {
786 $i++;
787 $tmparray['lines'][$i] = array();
788
789 if (!empty($item['description'])) {
790 $tmparray['lines'][$i]['desc'] = $item['description'];
791 } elseif (!empty($item['service']) && is_string($item['service'])) {
792 $tmparray['lines'][$i]['desc'] = $item['service'];
793 }
794
795 $tmparray['lines'][$i]['qty'] = $item['quantity'] ?? 1;
796 $tmparray['lines'][$i]['vat_rate'] = $item['tax']['vat_rate'] ?? ($item['vat_rate'] ?? ($item['vatrate'] ?? null));
797 $tmparray['lines'][$i]['total_vat'] = $item['tax']['amount'] ?? null;
798 $tmparray['lines'][$i]['subprice'] = $item['unit_price'] ?? null;
799 $tmparray['lines'][$i]['total_ht'] = $item['total_excluding_tax'] ?? null;
800 $tmparray['lines'][$i]['total_ttc'] = $item['total_including_tax'] ?? null;
801
802 if (!empty($item['period_start'])) {
803 $tmparray['lines'][$i]['date_start'] = dol_stringtotime($item['period_start'], 'tzuserrel');
804 }
805 if (!empty($item['period_end'])) {
806 $tmparray['lines'][$i]['date_end'] = dol_stringtotime($item['period_end'], 'tzuserrel');
807 }
808 if (!empty($item['period']) && !empty($item['period']['start_date'])) {
809 $tmparray['lines'][$i]['date_start'] = dol_stringtotime($item['period']['start_date'], 'tzuserrel');
810 }
811 if (!empty($item['period']) && !empty($item['period']['end_date'])) {
812 $tmparray['lines'][$i]['date_end'] = dol_stringtotime($item['period']['end_date'], 'tzuserrel');
813 }
814 }
815 }
816
817 if (!empty($json['summary'])) {
818 // The AI service may use different key names for the same data
819 $tmparray['total_ht'] = $json['summary']['total_excluding_tax'] ?? ($json['summary']['total_excl_tax'] ?? ($json['summary']['total_ht'] ?? ($json['summary']['subtotal'] ?? null)));
820 $tmparray['vat_rate'] = $json['summary']['tax_rate'] ?? ($json['summary']['vat_rate'] ?? ((isset($json['summary']['tax']) && is_array($json['summary']['tax'])) ? ($json['summary']['tax']['rate'] ?? null) : null));
821 $tmparray['total_ttc'] = $json['summary']['total_including_tax'] ?? ($json['summary']['total_incl_tax'] ?? ($json['summary']['total_ttc'] ?? ($json['summary']['total'] ?? null)));
822 }
823
824 // If no global VAT rate found, use the VAT rate of lines when all lines have the same one
825 if (!isset($tmparray['vat_rate']) && !empty($tmparray['lines'])) {
826 $vatratesoflines = array_unique(array_map('strval', array_filter(array_column($tmparray['lines'], 'vat_rate'), 'is_numeric')));
827 if (count($vatratesoflines) == 1 && count(array_filter(array_column($tmparray['lines'], 'vat_rate'), 'is_numeric')) == count($tmparray['lines'])) {
828 $tmparray['vat_rate'] = (float) reset($vatratesoflines);
829 }
830 }
831 } elseif ($type == 'salary') {
832 // The AI service may use different key names for the same data, so we test several of them
833 $docinfo = (isset($json['document_info']) && is_array($json['document_info'])) ? $json['document_info'] : array();
834 $employee = (isset($json['employee']) && is_array($json['employee'])) ? $json['employee'] : array();
835 $summary = (isset($json['summary']) && is_array($json['summary'])) ? $json['summary'] : array();
836
837 if (!empty($employee['name']) && is_string($employee['name'])) {
838 $tmparray['employee_name'] = $employee['name'];
839 }
840 if (!empty($docinfo['title']) && is_string($docinfo['title'])) {
841 $tmparray['label'] = $docinfo['title'];
842 }
843
844 // Dates (period of salary and payment date)
845 $arraydatekeys = array(
846 'period_start' => array('period_start', 'start_date', 'date_start'),
847 'period_end' => array('period_end', 'end_date', 'date_end'),
848 'payment_date' => array('payment_date', 'date_payment', 'paid_on')
849 );
850 foreach ($arraydatekeys as $keydest => $keyssrc) {
851 foreach ($keyssrc as $keysrc) {
852 if (!empty($docinfo[$keysrc]) && is_string($docinfo[$keysrc]) && preg_match('/^[0-9]{4}-[0-9]{2}-[0-9]{2}$/', $docinfo[$keysrc])) {
853 $tmparray[$keydest] = dol_stringtotime($docinfo[$keysrc], 'tzuserrel');
854 break;
855 }
856 }
857 }
858
859 // Amounts
860 $tmparray['gross_salary'] = $summary['gross_salary'] ?? ($summary['gross_pay'] ?? ($summary['gross'] ?? null));
861 $tmparray['net_to_pay'] = $summary['net_to_pay'] ?? ($summary['net_pay'] ?? ($summary['net_paid'] ?? ($summary['net_salary'] ?? null)));
862
863 $currency = (isset($docinfo['currency_code']) && is_string($docinfo['currency_code'])) ? $docinfo['currency_code'] : '';
864 if (dol_strlen($currency) == 3) {
865 $tmparray['currency_code'] = $currency;
866 }
867 } elseif ($type == 'social_contribution') {
868 // The AI service may use different key names for the same data, so we test several of them
869 $docinfo = (isset($json['document_info']) && is_array($json['document_info'])) ? $json['document_info'] : array();
870 $summary = (isset($json['summary']) && is_array($json['summary'])) ? $json['summary'] : array();
871
872 foreach (array('title' => 'label', 'organization' => 'organization', 'reference' => 'reference') as $keysrc => $keydest) {
873 if (!empty($docinfo[$keysrc]) && is_string($docinfo[$keysrc])) {
874 $tmparray[$keydest] = $docinfo[$keysrc];
875 }
876 }
877 if (!empty($json['employee']['name']) && is_string($json['employee']['name'])) {
878 $tmparray['employee_name'] = $json['employee']['name'];
879 }
880
881 // Dates (end of period the tax is related to, and due date)
882 $arraydatekeys = array(
883 'period_start' => array('period_start', 'start_date'),
884 'period_end' => array('period_end', 'end_date'),
885 'due_date' => array('due_date', 'payment_date', 'date_limit')
886 );
887 foreach ($arraydatekeys as $keydest => $keyssrc) {
888 foreach ($keyssrc as $keysrc) {
889 if (!empty($docinfo[$keysrc]) && is_string($docinfo[$keysrc]) && preg_match('/^[0-9]{4}-[0-9]{2}-[0-9]{2}$/', $docinfo[$keysrc])) {
890 $tmparray[$keydest] = dol_stringtotime($docinfo[$keysrc], 'tzuserrel');
891 break;
892 }
893 }
894 }
895
896 // Amount
897 $tmparray['amount'] = $summary['amount_to_pay'] ?? ($summary['total_to_pay'] ?? ($summary['amount'] ?? ($summary['total'] ?? null)));
898
899 $currency = (isset($docinfo['currency_code']) && is_string($docinfo['currency_code'])) ? $docinfo['currency_code'] : '';
900 if (dol_strlen($currency) == 3) {
901 $tmparray['currency_code'] = $currency;
902 }
903 } elseif ($type == 'expense') {
904 // The AI service may use different key names for the same data, so we test several of them
905 $docinfo = (isset($json['document_info']) && is_array($json['document_info'])) ? $json['document_info'] : array();
906 $summary = (isset($json['summary']) && is_array($json['summary'])) ? $json['summary'] : array();
907
908 foreach (array('merchant' => 'merchant', 'title' => 'label', 'category' => 'category') as $keysrc => $keydest) {
909 if (!empty($docinfo[$keysrc]) && is_string($docinfo[$keysrc])) {
910 $tmparray[$keydest] = $docinfo[$keysrc];
911 }
912 }
913
914 // Date of expense
915 foreach (array('date', 'expense_date', 'issue_date') as $keysrc) {
916 if (!empty($docinfo[$keysrc]) && is_string($docinfo[$keysrc]) && preg_match('/^[0-9]{4}-[0-9]{2}-[0-9]{2}$/', $docinfo[$keysrc])) {
917 $tmparray['date'] = dol_stringtotime($docinfo[$keysrc], 'tzuserrel');
918 break;
919 }
920 }
921
922 // Amounts. If VAT rate is not provided, we try to calculate it from amounts.
923 $tmparray['total_ttc'] = $summary['total_including_tax'] ?? ($summary['total_ttc'] ?? ($summary['total'] ?? null));
924 $tmparray['total_ht'] = $summary['total_excluding_tax'] ?? ($summary['total_ht'] ?? null);
925 $tmparray['vat_rate'] = $summary['vat_rate'] ?? ($summary['tax_rate'] ?? null);
926 $vatamount = $summary['vat_amount'] ?? ($summary['tax_amount'] ?? null);
927 if (!is_numeric($tmparray['vat_rate']) && is_numeric($vatamount) && is_numeric($tmparray['total_ht']) && $tmparray['total_ht'] > 0) {
928 $tmparray['vat_rate'] = round(100 * (float) $vatamount / (float) $tmparray['total_ht'], 1);
929 }
930
931 $currency = (isset($docinfo['currency_code']) && is_string($docinfo['currency_code'])) ? $docinfo['currency_code'] : '';
932 if (dol_strlen($currency) == 3) {
933 $tmparray['currency_code'] = $currency;
934 }
935 }
936
937 return $tmparray;
938 }
939}
getListOfAIServices()
Get list of available ai services.
Definition ai.lib.php:69
Class for AI feature.
Definition ai.class.php:37
getApiService()
get API Service
Definition ai.class.php:93
getPayloadToConvertRawFileIntoText($filepath)
Build the payload of the first call of the raw file mode: the call that converts the raw binary file ...
Definition ai.class.php:579
generateContent($instructions, $model='auto', $function='textgeneration', $format='', $moreheaders=array(), $moreendpoint='')
Generate the response of an AI prompt.
Definition ai.class.php:109
__construct($db)
Constructor.
Definition ai.class.php:79
decodeJsonIntoArray($json, $type)
Decode JSON into array.
Definition ai.class.php:637
dol_stringtotime($string, $gm=1, $processnotimeasnoon=0)
Convert a string date into a GM Timestamps date Warning: YYYY-MM-DDTHH:MM:SS+02:00 (RFC3339) is not s...
Definition date.lib.php:452
dol_basename($pathfile)
Make a basename working with all page code (default PHP basenamed fails with cyrillic).
Definition files.lib.php:41
dol_now($mode='gmt')
Return date for now.
dol_mimetype($file, $default='application/octet-stream', $mode=0)
Return MIME type of a file from its name with extension.
dol_strlen($string, $stringencoding='UTF-8', $binarycount=0)
Make a strlen call.
dol_osencode($str)
Return a string encoded into OS filesystem encoding.
dol_nl2br($stringtoencode, $nl2brmode=0, $forxml=false)
Replace CRLF in string with a HTML BR tag.
dolChmod($filepath, $newmask='')
Change mod of a file.
getDolGlobalInt($key, $default=0)
Return a Dolibarr global constant int value.
dol_textishtml($msg, $option=0)
Return if a text is a html content.
dol_print_date($time, $format='', $tzoutput='auto', $outputlangs=null, $encodetooutput=false, $decorate=0)
Output date in a string format according to outputlangs (or langs if not defined).
dol_trunc($string, $size=40, $trunc='right', $stringencoding='UTF-8', $nodot=0, $display=0)
Truncate a string to a particular length adding '...' if string larger than length.
dol_concat($text1, $text2)
Concat 2 strings.
getDolGlobalString($key, $default='')
Return a Dolibarr global constant string value.
dol_syslog($message, $level=LOG_INFO, $ident=0, $suffixinfilename='', $restricttologhandler='', $logcontext=null)
Write log message into outputs.
getURLContent($url, $postorget='GET', $param='', $followlocation=1, $addheaders=array(), $allowedschemes=array('http', 'https'), $localurl=0, $ssl_verifypeer=-1, $timeoutconnect=0, $timeoutresponse=0, $otherCurlOptions=array(), $morelogsuffix='')
Function to get a content from an URL (use proxy if proxy defined).