? .weather.geany ? weather-967646-15.patch ? translations/mrg.sh ? translations/t Index: CHANGELOG.txt =================================================================== RCS file: /cvs/drupal-contrib/contributions/modules/weather/CHANGELOG.txt,v retrieving revision 1.152.2.59 retrieving revision 1.152.2.61 diff -u -r1.152.2.59 -r1.152.2.61 --- CHANGELOG.txt 23 Sep 2010 16:55:33 -0000 1.152.2.59 +++ CHANGELOG.txt 19 Nov 2010 18:24:50 -0000 1.152.2.61 @@ -1,3 +1,12 @@ +Version: 6.x-trunk Date: UNRELEASED +--------------------------------------------------------------------- + +* Support a timeout for fetching new weather data. Closes: [#967646] +* Backport intelligent download interval calculation from D7 port. + This way, the website will be much more responsive if a request + times out or an ICAO code is no longer valid. + + Version: 6.x-5.15 Date: 2010-09-23 --------------------------------------------------------------------- Index: weather.module =================================================================== RCS file: /cvs/drupal-contrib/contributions/modules/weather/weather.module,v retrieving revision 1.205.2.22 diff -u -r1.205.2.22 weather.module --- weather.module 24 Aug 2010 15:20:35 -0000 1.205.2.22 +++ weather.module 19 Nov 2010 18:44:03 -0000 @@ -1,5 +1,5 @@ * @@ -1989,36 +1989,79 @@ * Fetches the latest METAR data from the database or internet */ function weather_get_metar($icao) { - // see if there's a report in the database + // See if there's a report in the database $icao = strtoupper($icao); $sql = "SELECT * FROM {weather} WHERE icao='%s'"; $result = db_query($sql, $icao); $data = db_fetch_array($result); - // if there is no report, initialize the array - if (!isset($data['metar_raw']) or !isset($data['next_update_on'])) { + // If there is no report, initialize the array + if (!isset($data['next_update_on'])) { $data['next_update_on'] = 0; - $data['metar_raw'] = ''; } - // if the time has come, download again + // If the time has come, try to download again if ($data['next_update_on'] <= time()) { - $data['metar_raw'] = ''; - } - - // fetch data from the internet - if ($data['metar_raw'] == '') { - $data['metar_raw'] = _weather_retrieve_data($icao); - if ($data['metar_raw']) { - $metar = weather_parse_metar($data['metar_raw']); - weather_store_metar($metar); + $metar_raw = _weather_retrieve_data($icao); + if ($metar_raw) { + $metar = weather_parse_metar($metar_raw); + // Calculate the next scheduled update. Use 62 minutes after the + // reported timestamp, to allow the data to propagate to the server. + $data['next_update_on'] = $metar['reported_on'] + 62 * 60; + // However, if the current time is more than 62 minutes + // over the reported timestamp, do not download on every page request. + // Therefore, we use 3, 6, 12, and 24 hours to check for updates. + // From then on, we check once a day for updates. + if ($data['next_update_on'] < time()) { + $last_update = $metar['reported_on']; + $hours = 3 * 60 * 60; + while (($last_update + $hours + 120) < time()) { + if ($hours < 86400) { + $hours = $hours * 2; + } + else { + $hours = $hours + 86400; + } + } + // Add 2 minutes to allow the data to propagate to the server. + $data['next_update_on'] = $last_update + $hours + 120; + } + weather_store_metar($metar, $data['next_update_on']); } else { - // the internet retrieval has not been successful. - // try again in 10 minutes - $sql = "UPDATE {weather} SET next_update_on=%d WHERE icao='%s'"; - db_query($sql, time() + 10*60, $icao); + // The download has not been successful. Calculate the time of next update + // according to last tries. + if (empty($data['metar_raw'])) { + // There is no entry yet, so this is the first download attempt. + // Create a new entry and store the current time in the #raw column. + $metar['icao'] = $icao; + $metar['#raw'] = time(); + $next_update_on = time() + 10 * 60; + weather_store_metar($metar, $next_update_on); + } + else { + // There has been at least one download attempt. Increase the time of + // next update to not download every few minutes. + // If 24 hours are reached, we check once a day for updates. + // This way, we gracefully handle ICAO codes which do no longer exist. + // The time of the last download attempt is stored in the #raw column. + $last_update = $data['metar_raw']; + $hours = 3 * 60 * 60; + while (($last_update + $hours + 120) < time()) { + if ($hours < 86400) { + $hours = $hours * 2; + } + else { + $hours = $hours + 86400; + } + } + // Add 2 minutes to allow the data to propagate to the server. + $next_update_on = $last_update + $hours + 120; + $metar['icao'] = $icao; + $metar['#raw'] = $data['metar_raw']; + weather_store_metar($metar, $next_update_on); + } } } else { @@ -2033,34 +2076,18 @@ /** * Stores parsed METAR data in the database */ -function weather_store_metar($metar) { - // if there's already a record in the database with the same ICAO +function weather_store_metar($metar, $next_update_on) { + // If there's already a record in the database with the same ICAO // overwrite it $sql = "DELETE FROM {weather} WHERE icao='%s'"; db_query($sql, $metar['icao']); - // insert the new data + // Insert the new data $sql = "INSERT INTO {weather} (icao, next_update_on, metar_raw) VALUES ('%s', %d, '%s')"; - // calculate the next scheduled update: normally, we use 62 - // minutes after the reported timestamp, to allow the data - // to propagate to the server servers. - $next_update_on = $metar['reported_on'] + 62*60; - - // However, if the current time is more than 62 minutes - // over the reported timestamp, allow ten more minutes - // to not fetch the data on each page request. - if ($next_update_on < time()) { - $next_update_on = time() + 10*60; - } - - db_query($sql, $metar['icao'], - // make the next update roughly one hour after the report, - // to allow the data to propagate - $next_update_on, - $metar['#raw']); + db_query($sql, $metar['icao'], $next_update_on, $metar['#raw']); } @@ -2071,6 +2098,8 @@ function _weather_retrieve_data($icao) { $icao = strtoupper($icao); $metar_raw = FALSE; + // Specify timeout in seconds + $timeout = 10; // get information about the last successful data download // @@ -2084,17 +2113,20 @@ $url = 'ftp://tgftp.nws.noaa.gov/data/observations/metar/stations/'; $url .= $icao .'.TXT'; if (function_exists('file_get_contents') and ini_get('allow_url_fopen')) { - $metar_raw = file_get_contents($url); + // Create a stream context to be able to use timeouts + $context = stream_context_create(array('http' => array('timeout' => $timeout))); + $metar_raw = file_get_contents($url, FALSE, $context); } else if (function_exists('curl_init')) { $curl = curl_init($url); curl_setopt($curl, CURLOPT_TRANSFERTEXT, TRUE); curl_setopt($curl, CURLOPT_RETURNTRANSFER, TRUE); + curl_setopt($curl, CURLOPT_TIMEOUT, $timeout); $metar_raw = curl_exec($curl); curl_close($curl); } else if (function_exists('exec')) { - exec("wget --quiet -O- $url", $output, $return_val); + exec("wget --quiet --timeout $timeout -O- $url", $output, $return_val); if ($return_val == 0 and is_array($output)) { $metar_raw = join("\n", $output); } @@ -2109,7 +2141,7 @@ // If the FTP access does not work, try via HTTP. if ($metar_raw === FALSE) { $url = 'http://weather.noaa.gov/mgetmetar.php?cccc=' . $icao; - $response = drupal_http_request($url); + $response = weather_drupal_http_request($url, $timeout); // Extract the valid METAR data from the received webpage. if (preg_match("/([0-9]{4}\/[0-9]{2}\/[0-9]{2}\s+[0-9]{2}:[0-9]{2}\n$icao [0-9]{6}Z .+)/m", $response->data, $matches)) { $metar_raw = $matches[1]; @@ -2133,3 +2165,228 @@ return $metar_raw; } + +/** + * Error code indicating that the request made by drupal_http_request() exceeded + * the specified timeout. + */ +define('HTTP_REQUEST_TIMEOUT', 1); + +/** + * Perform an HTTP request. + * + * This is a flexible and powerful HTTP client implementation. Correctly handles + * GET, POST, PUT or any other HTTP requests. Handles redirects. + * + * This function is a modified copy of the original Drupal function, + * which lacks a timeout parameter. See [#967646] + * The code is from CVS version 1.756.2.101 and includes the patch + * #29 from issue [#156582]. + * + * @param $url + * A string containing a fully qualified URI. + * @param $headers + * An array containing an HTTP header => value pair. + * @param $method + * A string defining the HTTP request to use. + * @param $data + * A string containing data to include in the request. + * @param $retry + * An integer representing how many times to retry the request in case of a + * redirect. + * @param $timeout + * A float representing the maximum number of seconds the function call may + * take. The default is 10 seconds. If a timeout occurs, the error code is set + * to the HTTP_REQUEST_TIMEOUT constant. + * @return + * An object containing the HTTP request headers, response code, protocol, + * status message, headers, data and redirect status. + */ +function weather_drupal_http_request($url, $timeout = 10, $headers = array(), $method = 'GET', $data = NULL, $retry = 3) { + global $db_prefix; + + $result = new stdClass(); + + // Parse the URL and make sure we can handle the schema. + $uri = parse_url($url); + + if ($uri == FALSE) { + $result->error = 'unable to parse URL'; + $result->code = -1001; + return $result; + } + + if (!isset($uri['scheme'])) { + $result->error = 'missing schema'; + $result->code = -1002; + return $result; + } + + timer_start(__FUNCTION__); + + switch ($uri['scheme']) { + case 'http': + case 'feed': + $port = isset($uri['port']) ? $uri['port'] : 80; + $host = $uri['host'] . ($port != 80 ? ':'. $port : ''); + $fp = @fsockopen($uri['host'], $port, $errno, $errstr, $timeout); + break; + case 'https': + // Note: Only works for PHP 4.3 compiled with OpenSSL. + $port = isset($uri['port']) ? $uri['port'] : 443; + $host = $uri['host'] . ($port != 443 ? ':'. $port : ''); + $fp = @fsockopen('ssl://'. $uri['host'], $port, $errno, $errstr, $timeout); + break; + default: + $result->error = 'invalid schema '. $uri['scheme']; + $result->code = -1003; + return $result; + } + + // Make sure the socket opened properly. + if (!$fp) { + // When a network error occurs, we use a negative number so it does not + // clash with the HTTP status codes. + $result->code = -$errno; + $result->error = trim($errstr); + + // Mark that this request failed. This will trigger a check of the web + // server's ability to make outgoing HTTP requests the next time that + // requirements checking is performed. + // @see system_requirements() + variable_set('drupal_http_request_fails', TRUE); + + return $result; + } + + // Construct the path to act on. + $path = isset($uri['path']) ? $uri['path'] : '/'; + if (isset($uri['query'])) { + $path .= '?'. $uri['query']; + } + + // Create HTTP request. + $defaults = array( + // RFC 2616: "non-standard ports MUST, default ports MAY be included". + // We don't add the port to prevent from breaking rewrite rules checking the + // host that do not take into account the port number. + 'Host' => "Host: $host", + 'User-Agent' => 'User-Agent: Drupal (+http://drupal.org/)', + ); + + // Only add Content-Length if we actually have any content or if it is a POST + // or PUT request. Some non-standard servers get confused by Content-Length in + // at least HEAD/GET requests, and Squid always requires Content-Length in + // POST/PUT requests. + $content_length = strlen($data); + if ($content_length > 0 || $method == 'POST' || $method == 'PUT') { + $defaults['Content-Length'] = 'Content-Length: '. $content_length; + } + + // If the server url has a user then attempt to use basic authentication + if (isset($uri['user'])) { + $defaults['Authorization'] = 'Authorization: Basic '. base64_encode($uri['user'] . (!empty($uri['pass']) ? ":". $uri['pass'] : '')); + } + + // If the database prefix is being used by SimpleTest to run the tests in a copied + // database then set the user-agent header to the database prefix so that any + // calls to other Drupal pages will run the SimpleTest prefixed database. The + // user-agent is used to ensure that multiple testing sessions running at the + // same time won't interfere with each other as they would if the database + // prefix were stored statically in a file or database variable. + if (is_string($db_prefix) && preg_match("/^simpletest\d+$/", $db_prefix, $matches)) { + $defaults['User-Agent'] = 'User-Agent: ' . $matches[0]; + } + + foreach ($headers as $header => $value) { + $defaults[$header] = $header .': '. $value; + } + + $request = $method .' '. $path ." HTTP/1.0\r\n"; + $request .= implode("\r\n", $defaults); + $request .= "\r\n\r\n"; + $request .= $data; + + $result->request = $request; + + fwrite($fp, $request); + + // Fetch response. + $response = ''; + while (!feof($fp)) { + // Calculate how much time is left of the original timeout value. + $time_left = $timeout - timer_read(__FUNCTION__) / 1000; + if ($time_left <= 0) { + $result->code = HTTP_REQUEST_TIMEOUT; + $result->error = 'request timed out'; + return $result; + } + stream_set_timeout($fp, floor($time_left), floor(1000000 * fmod($time_left, 1))); + $chunk = fread($fp, 1024); + $response .= $chunk; + } + fclose($fp); + + // Parse response. + list($split, $result->data) = explode("\r\n\r\n", $response, 2); + $split = preg_split("/\r\n|\n|\r/", $split); + + list($protocol, $code, $status_message) = explode(' ', trim(array_shift($split)), 3); + $result->protocol = $protocol; + $result->status_message = $status_message; + + $result->headers = array(); + + // Parse headers. + while ($line = trim(array_shift($split))) { + list($header, $value) = explode(':', $line, 2); + if (isset($result->headers[$header]) && $header == 'Set-Cookie') { + // RFC 2109: the Set-Cookie response header comprises the token Set- + // Cookie:, followed by a comma-separated list of one or more cookies. + $result->headers[$header] .= ','. trim($value); + } + else { + $result->headers[$header] = trim($value); + } + } + + $responses = array( + 100 => 'Continue', 101 => 'Switching Protocols', + 200 => 'OK', 201 => 'Created', 202 => 'Accepted', 203 => 'Non-Authoritative Information', 204 => 'No Content', 205 => 'Reset Content', 206 => 'Partial Content', + 300 => 'Multiple Choices', 301 => 'Moved Permanently', 302 => 'Found', 303 => 'See Other', 304 => 'Not Modified', 305 => 'Use Proxy', 307 => 'Temporary Redirect', + 400 => 'Bad Request', 401 => 'Unauthorized', 402 => 'Payment Required', 403 => 'Forbidden', 404 => 'Not Found', 405 => 'Method Not Allowed', 406 => 'Not Acceptable', 407 => 'Proxy Authentication Required', 408 => 'Request Time-out', 409 => 'Conflict', 410 => 'Gone', 411 => 'Length Required', 412 => 'Precondition Failed', 413 => 'Request Entity Too Large', 414 => 'Request-URI Too Large', 415 => 'Unsupported Media Type', 416 => 'Requested range not satisfiable', 417 => 'Expectation Failed', + 500 => 'Internal Server Error', 501 => 'Not Implemented', 502 => 'Bad Gateway', 503 => 'Service Unavailable', 504 => 'Gateway Time-out', 505 => 'HTTP Version not supported' + ); + // RFC 2616 states that all unknown HTTP codes must be treated the same as the + // base code in their class. + if (!isset($responses[$code])) { + $code = floor($code / 100) * 100; + } + + switch ($code) { + case 200: // OK + case 304: // Not modified + break; + case 301: // Moved permanently + case 302: // Moved temporarily + case 307: // Moved temporarily + $location = $result->headers['Location']; + $timeout -= timer_read(__FUNCTION__) / 1000; + if ($timeout <= 0) { + $result->code = HTTP_REQUEST_TIMEOUT; + $result->error = 'request timed out'; + } + elseif ($retry) { + $result = weather_drupal_http_request($result->headers['Location'], $timeout, $headers, $method, $data, --$retry); + $result->redirect_code = $result->code; + } + $result->redirect_url = $location; + + break; + default: + $result->error = $status_message; + } + + $result->code = $code; + return $result; +}