? tests Index: robotstxt.admin.inc =================================================================== RCS file: /cvs/drupal-contrib/contributions/modules/robotstxt/robotstxt.admin.inc,v retrieving revision 1.1.2.2 diff -u -p -r1.1.2.2 robotstxt.admin.inc --- robotstxt.admin.inc 19 Feb 2009 22:49:06 -0000 1.1.2.2 +++ robotstxt.admin.inc 7 Nov 2009 07:21:43 -0000 @@ -12,14 +12,32 @@ * @see system_settings_form() */ function robotstxt_admin_settings() { - $form['robotstxt'] = array( + $form['robotstxt_file'] = array( + '#type' => 'textfield', + '#title' => t('Default robots.txt file'), + '#description' => t('If specified, this will provide the base content for your robots.txt contents. If you wish to completely customize the robots.txt content, leave this field blank and use the field below.'), + '#default_value' => variable_get('robotstxt_file', drupal_get_path('module', 'robotstxt') . '/robots.txt'), + ); + $form['robotstxt_custom'] = array( '#type' => 'textarea', - '#title' => t('Contents of robots.txt'), - '#default_value' => _robotstxt_get_content(), - '#cols' => 60, - '#rows' => 20, + '#title' => t('Custom robots.txt directives'), + '#default_value' => variable_get('robotstxt_custom', ''), + '#rows' => 10, '#wysiwyg' => FALSE, ); + $form['#submit'][] = 'robotstxt_admin_settings_submit'; return system_settings_form($form); } + +function robotstxt_admin_settings_validate(&$form, &$form_state) { + $file = $form_state['values']['robotstxt_file']; + if ($file && (!is_file($file) || !is_readable($file))) { + form_set_error('robotstxt_file', t('Unable to read file %filename. Please make sure it exists and is readable.', array('%filename' => $file))); + } +} + +function robotstxt_admin_settings_submit(&$form, &$form_state) { + variable_del('robotstxt_file_contents'); + variable_del('robotstxt_file_modified'); +} Index: robotstxt.install =================================================================== RCS file: /cvs/drupal-contrib/contributions/modules/robotstxt/robotstxt.install,v retrieving revision 1.1.6.8 diff -u -p -r1.1.6.8 robotstxt.install --- robotstxt.install 19 Mar 2009 19:44:53 -0000 1.1.6.8 +++ robotstxt.install 7 Nov 2009 07:21:43 -0000 @@ -2,22 +2,14 @@ // $Id: robotstxt.install,v 1.1.6.8 2009/03/19 19:44:53 hass Exp $ /** - * Implementation of hook_install(). - */ -function robotstxt_install() { - if (file_exists('./robots.txt')) { - variable_set('robotstxt', file_get_contents('./robots.txt')); - } - elseif (file_exists(drupal_get_path('module', 'robotstxt') .'/robots.txt')) { - variable_set('robotstxt', file_get_contents(drupal_get_path('module', 'robotstxt') .'/robots.txt')); - } -} - -/** * Implementation of hook_uninstall(). */ function robotstxt_uninstall() { variable_del('robotstxt'); + variable_del('robotstxt_file'); + variable_del('robotstxt_file_modified'); + variable_del('robotstxt_file_contents'); + variable_del('robotstxt_custom'); } /** @@ -39,7 +31,7 @@ function robotstxt_requirements($phase) ); } - // Webservers prefer the robots.txt file on disk and does not allow menu path overwrite. + // Webservers prefer the robots.txt file on disk and does not allow menu path overwrite. if (file_exists('./robots.txt')) { $requirements['robotstxt_file'] = array( 'title' => $t('RobotsTxt'), @@ -50,3 +42,45 @@ function robotstxt_requirements($phase) } return $requirements; } + +/** + * Split the robotstxt variable into robotstxt_file and robotstxt_custom. + */ +function robotstxt_update_6100() { + $file = drupal_get_path('module', 'robotstxt') . '/robots.txt'; + $file_contents = file_get_contents($file); + $robotstxt = variable_get('robotstxt', ''); + + // Split default file contents and current robotstxt text into lines. + $file_lines = preg_split('/[\n\r]+/', $file_contents, 0, PREG_SPLIT_NO_EMPTY); + $current_lines = preg_split('/[\n\r]+/', $robotstxt, 0, PREG_SPLIT_NO_EMPTY); + + // Check if each line of the default file is in the current robotstxt text. + $all_in_file = TRUE; + foreach ($file_lines as $line) { + $index = array_search($line, $current_lines); + if ($index === FALSE) { + $all_in_file = FALSE; + break; + } + else { + unset($current_lines[$index]); + } + } + + if ($all_in_file) { + // If all the default files lines were found, use the default file and set + // the custom lines. + $robotstxt = implode("\n", $current_lines); + } + else { + // Not all the lines from the default file were found in the current text, + // so don't use the default file. + $file = ''; + } + + variable_set('robotstxt_file', $file); + variable_set('robotstxt_custom', $robotstxt); + variable_del('robotstxt'); + return array(); +} Index: robotstxt.module =================================================================== RCS file: /cvs/drupal-contrib/contributions/modules/robotstxt/robotstxt.module,v retrieving revision 1.4.4.17 diff -u -p -r1.4.4.17 robotstxt.module --- robotstxt.module 21 Feb 2009 08:59:51 -0000 1.4.4.17 +++ robotstxt.module 7 Nov 2009 07:21:44 -0000 @@ -44,8 +44,30 @@ function robotstxt_menu() { * Show the robots.txt file. */ function robotstxt_robots() { + $content = robotstxt_get_content(); + drupal_set_header('Content-type: text/plain'); + echo $content; + exit; +} + +/** + * Build and format the contents of the robots.txt file. + */ +function robotstxt_get_content() { $content = array(); - $content[] = _robotstxt_get_content(); + + $file = variable_get('robotstxt_file', drupal_get_path('module', 'robotstxt') . '/robots.txt'); + if ($file && variable_get('robotstxt_file_modified', 0) == filemtime($file)) { + $content[] = variable_get('robotstxt_file_contents', ''); + } + else { + $content[] = $file_contents = file_get_contents($file); + variable_set('robotstxt_file_contents', $file_contents); + variable_set('robotstxt_file_modified', filemtime($file)); + } + + // Add any custom directives. + $content[] = variable_get('robotstxt_custom', ''); // Hook other modules for adding additional lines. if ($additions = module_invoke_all('robotstxt')) { @@ -56,30 +78,5 @@ function robotstxt_robots() { $content = array_map('trim', $content); $content = array_filter($content); - drupal_set_header('Content-type: text/plain'); - echo implode("\n", $content); - exit; -} - -/** - * Retrieve contents of robots.txt from the database variable, site root, or - * module directory. - */ -function _robotstxt_get_content() { - $content = variable_get('robotstxt', FALSE); - - if ($content === FALSE) { - $files = array( - './robots.txt', - drupal_get_path('module', 'robotstxt') .'/robots.txt', - ); - foreach ($files as $file) { - if (file_exists($file) && is_readable($file)) { - $content = file_get_contents($file); - break; - } - } - } - - return $content; + return implode("\n", $content); }