dolibarr  17.0.3
geturl.lib.php
Go to the documentation of this file.
1 <?php
2 /* Copyright (C) 2008-2020 Laurent Destailleur <eldy@users.sourceforge.net>
3  *
4  * This program is free software; you can redistribute it and/or modify
5  * it under the terms of the GNU General Public License as published by
6  * the Free Software Foundation; either version 3 of the License, or
7  * (at your option) any later version.
8  *
9  * This program is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12  * GNU General Public License for more details.
13  *
14  * You should have received a copy of the GNU General Public License
15  * along with this program. If not, see <https://www.gnu.org/licenses/>.
16  * or see https://www.gnu.org/
17  */
18 
41 function getURLContent($url, $postorget = 'GET', $param = '', $followlocation = 1, $addheaders = array(), $allowedschemes = array('http', 'https'), $localurl = 0, $ssl_verifypeer = -1)
42 {
43  //declaring of global variables
44  global $conf;
45  $USE_PROXY = empty($conf->global->MAIN_PROXY_USE) ? 0 : $conf->global->MAIN_PROXY_USE;
46  $PROXY_HOST = empty($conf->global->MAIN_PROXY_HOST) ? 0 : $conf->global->MAIN_PROXY_HOST;
47  $PROXY_PORT = empty($conf->global->MAIN_PROXY_PORT) ? 0 : $conf->global->MAIN_PROXY_PORT;
48  $PROXY_USER = empty($conf->global->MAIN_PROXY_USER) ? 0 : $conf->global->MAIN_PROXY_USER;
49  $PROXY_PASS = empty($conf->global->MAIN_PROXY_PASS) ? 0 : $conf->global->MAIN_PROXY_PASS;
50 
51  dol_syslog("getURLContent postorget=".$postorget." URL=".$url." param=".$param);
52 
53  //setting the curl parameters.
54  $ch = curl_init();
55 
56  /*print $API_Endpoint."-".$API_version."-".$PAYPAL_API_USER."-".$PAYPAL_API_PASSWORD."-".$PAYPAL_API_SIGNATURE."<br>";
57  print $USE_PROXY."-".$gv_ApiErrorURL."<br>";
58  print $nvpStr;
59  exit;*/
60  curl_setopt($ch, CURLOPT_VERBOSE, 1);
61  curl_setopt($ch, CURLOPT_USERAGENT, 'Dolibarr geturl function');
62 
63  // We use @ here because this may return warning if safe mode is on or open_basedir is on (following location is forbidden when safe mode is on).
64  // We force value to false so we will manage redirection ourself later.
65  @curl_setopt($ch, CURLOPT_FOLLOWLOCATION, false);
66 
67  if (is_array($addheaders) && count($addheaders)) {
68  curl_setopt($ch, CURLOPT_HTTPHEADER, $addheaders);
69  }
70  curl_setopt($ch, CURLINFO_HEADER_OUT, true); // To be able to retrieve request header and log it
71 
72  // By default use tls decied by PHP.
73  // You can force, if supported a version like TLSv1 or TLSv1.2
74  if (!empty($conf->global->MAIN_CURL_SSLVERSION)) {
75  curl_setopt($ch, CURLOPT_SSLVERSION, $conf->global->MAIN_CURL_SSLVERSION);
76  }
77  //curl_setopt($ch, CURLOPT_SSLVERSION, 6); for tls 1.2
78 
79  // Turning on or off the ssl target certificate
80  if ($ssl_verifypeer < 0) {
81  global $dolibarr_main_prod;
82  $ssl_verifypeer = ($dolibarr_main_prod ? true : false);
83  }
84  if (!empty($conf->global->MAIN_CURL_DISABLE_VERIFYPEER)) {
85  $ssl_verifypeer = 0;
86  }
87 
88  // Turning off the server and peer verification(TrustManager Concept).
89  curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, ($ssl_verifypeer ? true : false));
90  curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, ($ssl_verifypeer ? true : false));
91 
92  // Restrict use to some protocols only
93  $protocols = 0;
94  if (is_array($allowedschemes)) {
95  foreach ($allowedschemes as $allowedscheme) {
96  if ($allowedscheme == 'http') {
97  $protocols |= CURLPROTO_HTTP;
98  }
99  if ($allowedscheme == 'https') {
100  $protocols |= CURLPROTO_HTTPS;
101  }
102  }
103  curl_setopt($ch, CURLOPT_PROTOCOLS, $protocols);
104  curl_setopt($ch, CURLOPT_REDIR_PROTOCOLS, $protocols);
105  }
106 
107  curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, empty($conf->global->MAIN_USE_CONNECT_TIMEOUT) ? 5 : $conf->global->MAIN_USE_CONNECT_TIMEOUT);
108  curl_setopt($ch, CURLOPT_TIMEOUT, empty($conf->global->MAIN_USE_RESPONSE_TIMEOUT) ? 30 : $conf->global->MAIN_USE_RESPONSE_TIMEOUT);
109 
110  /*
111  if ($maxsize) {
112  curl_setopt($ch, CURLOPT_MAXFILESIZE_LARGE, $maxsize);
113  } */
114 
115  //curl_setopt($ch, CURLOPT_SAFE_UPLOAD, true); // PHP 5.5
116  curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1); // We want response
117  if ($postorget == 'POST') {
118  curl_setopt($ch, CURLOPT_POST, 1); // POST
119  curl_setopt($ch, CURLOPT_POSTFIELDS, $param); // Setting param x=a&y=z as POST fields
120  } elseif ($postorget == 'POSTALREADYFORMATED') {
121  curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'POST'); // HTTP request is 'POST' but param string is taken as it is
122  curl_setopt($ch, CURLOPT_POSTFIELDS, $param); // param = content of post, like a xml string
123  } elseif ($postorget == 'PUT') {
124  $array_param = null;
125  curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'PUT'); // HTTP request is 'PUT'
126  if (!is_array($param)) {
127  parse_str($param, $array_param);
128  } else {
129  dol_syslog("parameter param must be a string", LOG_WARNING);
130  $array_param = $param;
131  }
132  curl_setopt($ch, CURLOPT_POSTFIELDS, http_build_query($array_param)); // Setting param x=a&y=z as PUT fields
133  } elseif ($postorget == 'PUTALREADYFORMATED') {
134  curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'PUT'); // HTTP request is 'PUT'
135  curl_setopt($ch, CURLOPT_POSTFIELDS, $param); // param = content of post, like a xml string
136  } elseif ($postorget == 'HEAD') {
137  curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'HEAD'); // HTTP request is 'HEAD'
138  curl_setopt($ch, CURLOPT_NOBODY, true);
139  } elseif ($postorget == 'DELETE') {
140  curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'DELETE'); // POST
141  } else {
142  curl_setopt($ch, CURLOPT_POST, 0); // GET
143  }
144 
145  //if USE_PROXY constant set at begin of this method.
146  if ($USE_PROXY) {
147  dol_syslog("getURLContent set proxy to ".$PROXY_HOST.":".$PROXY_PORT." - ".$PROXY_USER.":".$PROXY_PASS);
148  //curl_setopt ($ch, CURLOPT_PROXYTYPE, CURLPROXY_HTTP); // Curl 7.10
149  curl_setopt($ch, CURLOPT_PROXY, $PROXY_HOST.":".$PROXY_PORT);
150  if ($PROXY_USER) {
151  curl_setopt($ch, CURLOPT_PROXYUSERPWD, $PROXY_USER.":".$PROXY_PASS);
152  }
153  }
154 
155  $newUrl = $url;
156  $maxRedirection = 5;
157  $info = array();
158  $response = '';
159 
160  do {
161  if ($maxRedirection < 1) {
162  break;
163  }
164 
165  curl_setopt($ch, CURLOPT_URL, $newUrl);
166 
167  // Parse $newUrl
168  $newUrlArray = parse_url($newUrl);
169  $hosttocheck = $newUrlArray['host'];
170  $hosttocheck = str_replace(array('[', ']'), '', $hosttocheck); // Remove brackets of IPv6
171 
172  // Deny some reserved host names
173  if (in_array($hosttocheck, array('metadata.google.internal'))) {
174  $info['http_code'] = 400;
175  $info['content'] = 'Error bad hostname '.$hosttocheck.' (Used by Google metadata). This value for hostname is not allowed.';
176  break;
177  }
178 
179  // Clean host name $hosttocheck to convert it into an IP $iptocheck
180  if (in_array($hosttocheck, array('localhost', 'localhost.domain'))) {
181  $iptocheck = '127.0.0.1';
182  } elseif (in_array($hosttocheck, array('ip6-localhost', 'ip6-loopback'))) {
183  $iptocheck = '::1';
184  } else {
185  // Resolve $hosttocheck to get the IP $iptocheck and set CURLOPT_CONNECT_TO to use this ip so curl will not try another resolution that may give a different result
186  if (function_exists('gethostbyname')) {
187  $iptocheck = gethostbyname($hosttocheck);
188  } else {
189  $iptocheck = $hosttocheck;
190  }
191  // TODO Resolve ip v6
192  }
193 
194  // Check $iptocheck is an IP (v4 or v6), if not clear value.
195  if (!filter_var($iptocheck, FILTER_VALIDATE_IP, FILTER_FLAG_IPV4 | FILTER_FLAG_IPV6)) { // This is not an IP, we clean data
196  $iptocheck = '0'; //
197  }
198 
199  if ($iptocheck) {
200  if ($localurl == 0) { // Only external url allowed (dangerous, may allow to get malware)
201  if (!filter_var($iptocheck, FILTER_VALIDATE_IP, FILTER_FLAG_NO_PRIV_RANGE | FILTER_FLAG_NO_RES_RANGE)) {
202  // Deny ips like 10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16, 0.0.0.0/8, 169.254.0.0/16, 127.0.0.0/8 et 240.0.0.0/4, ::1/128, ::/128, ::ffff:0:0/96, fe80::/10...
203  $info['http_code'] = 400;
204  $info['content'] = 'Error bad hostname IP (private or reserved range). Must be an external URL.';
205  break;
206  }
207  if (!empty($_SERVER["SERVER_ADDR"]) && $iptocheck == $_SERVER["SERVER_ADDR"]) {
208  $info['http_code'] = 400;
209  $info['content'] = 'Error bad hostname IP (IP is a local IP). Must be an external URL.';
210  break;
211  }
212  if (!empty($conf->global->MAIN_SECURITY_ANTI_SSRF_SERVER_IP) && in_array($iptocheck, explode(',', $conf->global->MAIN_SECURITY_ANTI_SSRF_SERVER_IP))) {
213  $info['http_code'] = 400;
214  $info['content'] = 'Error bad hostname IP (IP is a local IP defined into MAIN_SECURITY_SERVER_IP). Must be an external URL.';
215  break;
216  }
217  }
218  if ($localurl == 1) { // Only local url allowed (dangerous, may allow to get metadata on server or make internal port scanning)
219  // Deny ips NOT like 10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16, 0.0.0.0/8, 169.254.0.0/16, 127.0.0.0/8 et 240.0.0.0/4, ::1/128, ::/128, ::ffff:0:0/96, fe80::/10...
220  if (filter_var($iptocheck, FILTER_VALIDATE_IP, FILTER_FLAG_NO_PRIV_RANGE | FILTER_FLAG_NO_RES_RANGE)) {
221  $info['http_code'] = 400;
222  $info['content'] = 'Error bad hostname '.$iptocheck.'. Must be a local URL.';
223  break;
224  }
225  if (!empty($conf->global->MAIN_SECURITY_ANTI_SSRF_SERVER_IP) && !in_array($iptocheck, explode(',', $conf->global->MAIN_SECURITY_ANTI_SSRF_SERVER_IP))) {
226  $info['http_code'] = 400;
227  $info['content'] = 'Error bad hostname IP (IP is not a local IP defined into list MAIN_SECURITY_SERVER_IP). Must be a local URL in allowed list.';
228  break;
229  }
230  }
231 
232  // Common check on ip (local and external)
233  // See list on https://tagmerge.com/gist/a7b9d57ff8ec11d63642f8778609a0b8
234  // Not evasive url that ar enot IP are excluded by test on IP v4/v6 validity.
235  $arrayofmetadataserver = array(
236  '100.100.100.200' => 'Alibaba',
237  '192.0.0.192' => 'Oracle',
238  '192.80.8.124' => 'Packet',
239  '100.88.222.5' => 'Tencent cloud',
240  );
241  foreach ($arrayofmetadataserver as $ipofmetadataserver => $nameofmetadataserver) {
242  if ($iptocheck == $ipofmetadataserver) {
243  $info['http_code'] = 400;
244  $info['content'] = 'Error bad hostname IP (Used by '.$nameofmetadataserver.' metadata server). This IP is forbidden.';
245  break 2; // exit the foreach and the do...
246  }
247  }
248 
249  // Set CURLOPT_CONNECT_TO so curl will not try another resolution that may give a different result. Possible only on PHP v7+
250  if (defined('CURLOPT_CONNECT_TO')) {
251  $connect_to = array(sprintf("%s:%d:%s:%d", $newUrlArray['host'], empty($newUrlArray['port'])?'':$newUrlArray['port'], $iptocheck, empty($newUrlArray['port'])?'':$newUrlArray['port']));
252  //var_dump($newUrlArray);
253  //var_dump($connect_to);
254  curl_setopt($ch, CURLOPT_CONNECT_TO, $connect_to);
255  }
256  }
257 
258  // Getting response from server
259  $response = curl_exec($ch);
260 
261  $info = curl_getinfo($ch); // Reading of request must be done after sending request
262  $http_code = $info['http_code'];
263 
264  if ($followlocation && ($http_code == 301 || $http_code == 302 || $http_code == 303 || $http_code == 307)) {
265  $newUrl = $info['redirect_url'];
266  $maxRedirection--;
267  // TODO Use $info['local_ip'] and $info['primary_ip'] ?
268  continue;
269  } else {
270  $http_code = 0;
271  }
272  } while ($http_code);
273 
274  $request = curl_getinfo($ch, CURLINFO_HEADER_OUT); // Reading of request must be done after sending request
275 
276  dol_syslog("getURLContent request=".$request);
277  if (!empty($conf->global->MAIN_GETURLCONTENT_OUTPUT_RESPONSE)) {
278  // This may contains binary data, so we dont output reponse by default.
279  dol_syslog("getURLContent response =".$response);
280  }
281  dol_syslog("getURLContent response size=".strlen($response)); // This may contains binary data, so we dont output it
282 
283  $rep = array();
284  if (curl_errno($ch)) {
285  // Add keys to $rep
286  $rep['content'] = $response;
287 
288  // moving to display page to display curl errors
289  $rep['curl_error_no'] = curl_errno($ch);
290  $rep['curl_error_msg'] = curl_error($ch);
291 
292  dol_syslog("getURLContent response array is ".join(',', $rep));
293  } else {
294  //$info = curl_getinfo($ch);
295 
296  // Add keys to $rep
297  $rep = $info;
298  //$rep['header_size']=$info['header_size'];
299  //$rep['http_code']=$info['http_code'];
300  dol_syslog("getURLContent http_code=".$rep['http_code']);
301 
302  // Add more keys to $rep
303  if ($response) {
304  $rep['content'] = $response;
305  }
306  $rep['curl_error_no'] = '';
307  $rep['curl_error_msg'] = '';
308  }
309 
310  //closing the curl
311  curl_close($ch);
312 
313  return $rep;
314 }
315 
316 
325 function getDomainFromURL($url, $mode = 0)
326 {
327  $tmpdomain = preg_replace('/^https?:\/\//i', '', $url); // Remove http(s)://
328  $tmpdomain = preg_replace('/\/.*$/i', '', $tmpdomain); // Remove part after domain
329  if ($mode == 2) {
330  $tmpdomain = preg_replace('/^.*\.([^\.]+)\.([^\.]+)\.([^\.]+)$/', '\1.\2.\3', $tmpdomain); // Remove part 'www.' before 'abc.mydomain.com'
331  } else {
332  $tmpdomain = preg_replace('/^.*\.([^\.]+)\.([^\.]+)$/', '\1.\2', $tmpdomain); // Remove part 'www.abc.' before 'mydomain.com'
333  }
334  if (empty($mode)) {
335  $tmpdomain = preg_replace('/\.[^\.]+$/', '', $tmpdomain); // Remove first level domain (.com, .net, ...)
336  }
337 
338  return $tmpdomain;
339 }
340 
349 function getRootURLFromURL($url)
350 {
351  $prefix = '';
352  $tmpurl = $url;
353  $reg = null;
354  if (preg_match('/^(https?:\/\/)/i', $tmpurl, $reg)) {
355  $prefix = $reg[1];
356  }
357  $tmpurl = preg_replace('/^https?:\/\//i', '', $tmpurl); // Remove http(s)://
358  $tmpurl = preg_replace('/\/.*$/i', '', $tmpurl); // Remove part after domain
359 
360  return $prefix.$tmpurl;
361 }
362 
369 function removeHtmlComment($content)
370 {
371  $content = preg_replace('/<!--[^\-]+-->/', '', $content);
372  return $content;
373 }
getRootURLFromURL
getRootURLFromURL($url)
Function root url from a long url For example: https://www.abc.mydomain.com/dir/page....
Definition: geturl.lib.php:349
getURLContent
getURLContent($url, $postorget='GET', $param='', $followlocation=1, $addheaders=array(), $allowedschemes=array('http', 'https'), $localurl=0, $ssl_verifypeer=-1)
Function to get a content from an URL (use proxy if proxy defined).
Definition: geturl.lib.php:41
dol_syslog
dol_syslog($message, $level=LOG_INFO, $ident=0, $suffixinfilename='', $restricttologhandler='', $logcontext=null)
Write log message into outputs.
Definition: functions.lib.php:1628
getDomainFromURL
getDomainFromURL($url, $mode=0)
Function get second level domain name.
Definition: geturl.lib.php:325
removeHtmlComment
removeHtmlComment($content)
Function to remove comments into HTML content.
Definition: geturl.lib.php:369