? .DS_Store ? auditfiles.install ? auditfiles.patch ? references.admin.inc ? unreferenced.admin.inc Index: CHANGELOG.txt =================================================================== RCS file: /cvs/drupal-contrib/contributions/modules/auditfiles/CHANGELOG.txt,v retrieving revision 1.5 diff -u -p -r1.5 CHANGELOG.txt --- CHANGELOG.txt 5 Dec 2007 22:33:19 -0000 1.5 +++ CHANGELOG.txt 3 Jun 2009 22:11:50 -0000 @@ -1,3 +1,8 @@ +4 September 2008 (mikeryan) +---------------- +Implemented Missing References and Unreferenced reports. +Added Attach actions to Not In Database report. + 30 October 2007 --------------- Module updated for Drupal 6. Index: INSTALL.txt =================================================================== RCS file: /cvs/drupal-contrib/contributions/modules/auditfiles/INSTALL.txt,v retrieving revision 1.4 diff -u -p -r1.4 INSTALL.txt --- INSTALL.txt 5 Dec 2007 22:33:19 -0000 1.4 +++ INSTALL.txt 3 Jun 2009 22:11:50 -0000 @@ -2,6 +2,6 @@ auditfiles.module installation instructi =========================================== Install this module as normal by copying it to your modules directory. -It will add two audit reports to the Administer > Logs menu +It will add four audit reports to the Administer > Logs menu ; $Id: INSTALL.txt,v 1.4 2007/12/05 22:33:19 stuartgreenfield Exp $ \ No newline at end of file Index: README.txt =================================================================== RCS file: /cvs/drupal-contrib/contributions/modules/auditfiles/README.txt,v retrieving revision 1.5 diff -u -p -r1.5 README.txt --- README.txt 5 Dec 2007 22:33:19 -0000 1.5 +++ README.txt 3 Jun 2009 22:11:50 -0000 @@ -2,8 +2,8 @@ Audit files for Drupal 6.x ========================== Audit files is a module that is designed to help keep your upload files in -check. It can run two reports, which are accessed from Administer > Logs > -Audit files not in database / Audit files not on server +check. It can run four reports, which are accessed from Administer > Reports > +Audit files Audit files not on the server @@ -34,6 +34,38 @@ file is no longer needed before erasing If you're not sure what the file is then you can click on the filename to open the file in your browser. +Missing references +------------------ +Listed here are file references embedded in node bodies which do not have +exact correspondences in the {files} and {upload} tables. If there is a +file in the {files} table with a corresponding base name, that is listed. +Scenarios are: + +* No match at all in the {files} table. Go to Files not in database and make + sure any files that exist have been added to the database. If they have, you + should either find and upload the missing file and run this report again, or + remove the reference from the node. +* Multiple matches in the {files} table. This can happen when the same filename + is in multiple directories in the uploded file hierarchy. You can review the + alternate files and delete any that are true duplicates. When different files + have the same basename, you can select the one that goes with the given node + and choose Attach selected files. This will rewrite the reference in the node + to use the canonical relative URL to the file, and if necessary add the reference + to the {upload} table. +* A single match in the {files} table. You can make the attachments between these + nodes and the corresponding files one-by-one by selecting them and choosing Attach + selected files, or automatically apply it to all single-match cases with Attach + all unique matches. + +Unreferenced +------------ +The files listed here are in the {files} table but no nodes are recorded as +referencing them (i.e., there's no entry in the {upload} table). This might mean +the node has been deleted without deleting the file, or that the files were uploaded +by some means other than the upload module (e.g., ftp) and the relationships between +files and nodes have not been made. If you have used the File references report and +accounted for all files that should be referenced, and are sure that the files below +are not needed, you can delete them. Configuration ------------- @@ -42,5 +74,45 @@ that you do not want to be included. You Site configuration > Audit files. By default the audit excludes .htaccess files and the contents of the color directory. +Migration +--------- +In typical usage, the reports can be used independently and the occasional issues +revealed dealt with one-by-one. Another application is the migration of content and +images from another web site (for example, using the node_import module or pasting +HTML content manually into a node creation form), which typically will break any +embedded image references. The file audit tools can automate much of rectifying this +situation. + +A typical workflow for migrating content containing embedded images would be: + +1. Copy all referenced images to the Drupal files directory (typically sites/default/files). +2. Go to the Not In Database report, and after sanity-checking the list execute the Add All + Files to Database action. This will add each file you've copied in to the {files} table. +3. Go to the Missing References report, and after sanity-checking the list execute the + Attach All Unique Matches action. This will rewrite the image references in your content + to properly point to their path on the Drupal server, in every case where there is a + single matching filename. +4. The Missing References report will now show you image references which don't match any + files in the Drupal files directory - review these to see if you missed any. Some cases + may be offsite references that can be safely ignored - in other cases, where you can't + track down the missing image, you can go to the node and edit it to remove the image reference. +5. The Unreferenced report will now show you files that are not in the {files} and {upload} + tables. Review these carefully to make sure they aren't being used in some way not picked + up by the other tools (e.g., through Javascript). Suggestion: take some sample filenames + and query the Drupal database directly: + SELECT * FROM node_revisions WHERE body LIKE '%file.jpg%' + If you find any files you're sure are not being used, you may (after making sure you have + a fresh backup of the directory tree) delete them from the Unreferenced report. + +Issues +------ +Files are associated with nodes using the {upload} table, behind the upload module's back. The +main issue here is that we may associate a single file with multiple nodes, but the upload +module assumes that each file is only associated with a single node and thus deletes the file +when the node is deleted. This suggests that our generated associations should be saved in our +own table, which would require every place that now checks the {upload} table to check two +tables. + +The newer functionality is untested with private downloads. ; $Id: README.txt,v 1.5 2007/12/05 22:33:19 stuartgreenfield Exp $ \ No newline at end of file Index: auditfiles.admin.inc =================================================================== RCS file: /cvs/drupal-contrib/contributions/modules/auditfiles/auditfiles.admin.inc,v retrieving revision 1.1 diff -u -p -r1.1 auditfiles.admin.inc --- auditfiles.admin.inc 5 Dec 2007 22:33:19 -0000 1.1 +++ auditfiles.admin.inc 3 Jun 2009 22:11:50 -0000 @@ -7,26 +7,55 @@ */ function auditfiles_admin_settings() { - $form['auditfiles_exclude_files'] = array( - '#type' => 'textfield', - '#title' => t('Exclude these files'), - '#default_value' => trim(variable_get('auditfiles_exclude_files', '.htaccess')), - '#description' => t('Enter a list of files to exclude, separated by spaces.'), - ); + $form['exclusions'] = array( + '#type' => 'fieldset', + '#title' => t('Exclusions'), + '#collapsible' => TRUE, + '#collapsed' => FALSE, + ); + $form['exclusions']['auditfiles_exclude_files'] = array( + '#type' => 'textfield', + '#title' => t('Exclude these files'), + '#default_value' => trim(variable_get('auditfiles_exclude_files', '.htaccess')), + '#description' => t('Enter a list of files to exclude, separated by spaces.'), + ); - $form['auditfiles_exclude_extensions'] = array( - '#type' => 'textfield', - '#title' => t('Exclude these extensions'), - '#default_value' => trim(variable_get('auditfiles_exclude_extensions', '')), - '#description' => t('Enter a list of extensions to exclude, separated by spaces. Do not include the leading dot.'), - ); + $form['exclusions']['auditfiles_exclude_extensions'] = array( + '#type' => 'textfield', + '#title' => t('Exclude these extensions'), + '#default_value' => trim(variable_get('auditfiles_exclude_extensions', '')), + '#description' => t('Enter a list of extensions to exclude, separated by spaces. Do not + include the leading dot.'), + ); - $form['auditfiles_exclude_paths'] = array( - '#type' => 'textfield', - '#title' => t('Exclude these paths'), - '#default_value' => trim(variable_get('auditfiles_exclude_paths', 'color')), - '#description' => t('Enter a list of paths to exclude, separated by spaces. Do not include the leading slash. Paths are relative to %directorypath', array('%directorypath' => file_directory_path())), - ); + $form['exclusions']['auditfiles_exclude_paths'] = array( + '#type' => 'textfield', + '#title' => t('Exclude these paths'), + '#default_value' => trim(variable_get('auditfiles_exclude_paths', 'color')), + '#description' => t('Enter a list of paths to exclude, separated by spaces. Do not + include the leading slash. Paths are relative to %directorypath', + array('%directorypath' => file_directory_path())), + ); + $form['domains'] = array( + '#type' => 'fieldset', + '#title' => t('Domains'), + '#collapsible' => TRUE, + '#collapsed' => FALSE, + ); + + $form['domains']['auditfiles_include_domains'] = array( + '#type' => 'textfield', + '#title' => t('Include references to these domains'), + '#default_value' => trim(variable_get('auditfiles_include_domains', '')), + '#size' => 80, + '#maxlength' => 1024, + '#description' => t('Enter a list of domains (e.g., www.example.com) pointing + to your website, separated by spaces. When scanning content for file references (such as <img> + tags), any absolute references using these domains will be included and rewritten + to use relative references. Absolute references to domains not in this list will + be considered to be external references and will not be audited or rewritten.'), + ); + return system_settings_form($form); } Index: auditfiles.module =================================================================== RCS file: /cvs/drupal-contrib/contributions/modules/auditfiles/auditfiles.module,v retrieving revision 1.5 diff -u -p -r1.5 auditfiles.module --- auditfiles.module 5 Dec 2007 22:33:19 -0000 1.5 +++ auditfiles.module 3 Jun 2009 22:11:50 -0000 @@ -13,18 +13,68 @@ function auditfiles_help($path, $arg) { switch ($path) { case 'admin/settings/auditfiles': - return t('You can choose to exclude specific files, paths and extensions from the \'files not in database\' audit report by adding them to the relevant lists below.'); + return t('You can choose to exclude specific files, paths and extensions from the \'files + not in database\' audit report by adding them to the relevant lists below.'); case 'admin/help#auditfiles': - return '

'. t('The audit files module performs an audit of the file table and the files directory to check for inconsistencies.') .'

'; - case 'admin/reports/auditfiles/notonserver': - $output .= '

'.t('The files listed below are in the {files} table but the physical files do not exist on the server. This might mean the file has been deleted using a program such as FTP, or it may mean there is an error in the database.

If the file was uploaded using the upload module then you can click on the node number to view the item to which the missing file relates to try and determine what action needs to be taken, or you can click the edit link to open the node editing form directly.').'

'; - $output .= '

'.t('Files in this list are relative to the files directory %directorypath.', array('%directorypath' => file_directory_path())).'

'; - return $output; + return '

'. t('The audit files module performs an audit of the file table and the files + directory to check for inconsistencies.') .'

'; case 'admin/reports/auditfiles': - $output .= '

'.t('The files listed below are in the files directory on the server but appear to have no corresponding entry in the {files} table. Files in "temporary" folders such as those created by the image module are included in order to check that they are not filling up. You can choose to delete files from this report but remember that if you do this the action cannot be undone.').'

'; + $output .= '

'.t('The files listed below are in the files directory on the server but + appear to have no corresponding entry in the {files} table. Files in "temporary" folders + such as those created by the image module are included in order to check that they are not + filling up. You can choose to delete files from this report but remember that if you do + this the action cannot be undone. You can also add files to the {files} table - you might + do this if you have uploaded files outside of Drupal (e.g., by ftp) and will be attaching + them to nodes through the other tools.').'

'; $output .= '

'.t('Files in this list are relative to the files directory %directorypath.', array('%directorypath' => file_directory_path())).'

'; - // If on the delete confirmation form then suppress the help message - if ($_POST['operation'] == 'delete' && $_POST['files']) $output = ''; + // If on a confirmation form then suppress the help message + if ($_POST['operation'] && $_POST['files']) $output = ''; + return $output; + case 'admin/reports/auditfiles/notonserver': + $output .= '

'.t('The files listed below are in the {files} table but the physical files + do not exist on the server. This might mean the file has been deleted using a program such + as FTP, or it may mean there is an error in the database.

If the file was uploaded + using the upload module then you can click on the node number to view the item to which + the missing file relates to try and determine what action needs to be taken, or you can + click the edit link to open the node editing form directly.').'

'; + $output .= '

'.t('Files in this list are relative to the files directory %directorypath.', + array('%directorypath' => file_directory_path())).'

'; + return $output; + case 'admin/reports/auditfiles/references': + $output .= '

'.t('Listed here are file references embedded in node bodies which do + not have exact correspondences in the {files} and {upload} tables. If there is a file + in the {files} table with a corresponding base name, that is listed. Scenarios are: +

').'

'; + $output .= '

'.t('Files in this list are relative to the files directory %directorypath.', + array('%directorypath' => file_directory_path())).'

'; + return $output; + case 'admin/reports/auditfiles/unreferenced': + $output .= '

'.t('The files listed below are in the {files} table but no nodes are recorded + as referencing them (i.e., there\'s no entry in the {upload} table). This might mean the + node has been deleted without deleting the file, or that the files were uploaded by some + means other than the upload module (e.g., ftp) and the relationships between files and + nodes have not been made. If you have used the Missing references report + and accounted for all files that should be referenced, and are sure that the files below + are not needed, you can delete them. Click on the basename to get a list of all nodes + referencing that base filename (the Missing references report only + identifies <img> and window.open references, + this may help you identify if the file is referenced by another means).').'

'; + $output .= '

'.t('Files in this list are relative to the files directory %directorypath.', + array('%directorypath' => file_directory_path())).'

'; return $output; } } @@ -49,30 +99,59 @@ function auditfiles_menu() { $items = array(); $items['admin/reports/auditfiles'] = array( - 'title' => 'Audit files', - 'description' => 'List files that are not on the server or not in the database.', - 'page callback' => 'drupal_get_form', - 'page arguments' => array('auditfiles_notindb'), + 'title' => t('Audit files'), + 'description' => t('List files that are not on the server or not in the database.'), + 'page callback' => 'auditfiles_notindb', 'access arguments' => array('access file audits'), 'file' => 'notindb.admin.inc', ); $items['admin/reports/auditfiles/notindb'] = array( - 'title' => t('Files not in database'), - 'description' => 'List files that are on the server but are not in the database.', + 'title' => t('Not in database'), + 'description' => t('List files that are on the server but are not in the database.'), 'type' => MENU_DEFAULT_LOCAL_TASK, 'weight' => -10, ); $items['admin/reports/auditfiles/notonserver'] = array( - 'title' => 'Audit files not on server', - 'description' => 'List files that are in the database, but are not on the server.', + 'title' => t('Not on server'), + 'description' => t('List files that are in the database, but are not on the server.'), 'page callback' => 'auditfiles_notonserver', 'access arguments' => array('access file audits'), 'type' => MENU_LOCAL_TASK, 'file' => 'notonserver.admin.inc', + 'weight' => -5, ); - + + $items['admin/reports/auditfiles/references'] = array( + 'title' => t('Missing references'), + 'description' => t('List files referenced by nodes, but not properly linked or attached.'), + 'page callback' => 'auditfiles_references', + 'access arguments' => array('access file audits'), + 'type' => MENU_LOCAL_TASK, + 'file' => 'references.admin.inc', + 'weight' => 0, + ); + + $items['admin/reports/auditfiles/unreferenced'] = array( + 'title' => t('Unreferenced'), + 'description' => t('List files that are in the database, but are not referenced by any node.'), + 'page callback' => 'auditfiles_unreferenced', + 'access arguments' => array('access file audits'), + 'type' => MENU_LOCAL_TASK, + 'file' => 'unreferenced.admin.inc', + 'weight' => 5, + ); + + $items['admin/reports/auditfiles/unreferenced/%'] = array( + 'title' => 'Referencing nodes', + 'page callback' => 'drupal_get_form', + 'page arguments' => array('auditfiles_node_references_form', 4), + 'access arguments' => array('access file audits'), + 'type' => MENU_CALLBACK, + 'file' => 'unreferenced.admin.inc', + ); + $items['admin/settings/auditfiles'] = array( 'title' => 'Audit files', 'description' => 'Set file, extension and path exclusions for file audits.', @@ -95,5 +174,17 @@ function auditfiles_theme() { 'arguments' => array('form' => NULL), 'file' => 'notindb.admin.inc', ), + 'auditfiles_unreferenced_form' => array( + 'arguments' => array('form' => NULL), + 'file' => 'unreferenced.admin.inc', + ), + 'auditfiles_node_references_form' => array( + 'arguments' => array('form' => NULL), + 'file' => 'unreferenced.admin.inc', + ), + 'auditfiles_references_form' => array( + 'arguments' => array('form' => NULL), + 'file' => 'references.admin.inc', + ), ); } Index: notindb.admin.inc =================================================================== RCS file: /cvs/drupal-contrib/contributions/modules/auditfiles/notindb.admin.inc,v retrieving revision 1.1 diff -u -p -r1.1 notindb.admin.inc --- notindb.admin.inc 5 Dec 2007 22:33:19 -0000 1.1 +++ notindb.admin.inc 3 Jun 2009 22:11:50 -0000 @@ -10,26 +10,18 @@ /** * Menu callback: audit files not in the database. */ -function auditfiles_notindb($form_state) { - - // If delete action was selected, and files were checked, then return confirmation form - if ($form_state['values']['operation'] == 'delete' && array_filter($form_state['values']['files'])) { - return auditfiles_multiple_delete_confirm($form_state, array_filter($form_state['values']['files'])); - } - - // Otherwise retreive the form - $form['auditfiles_notindb'] = auditfiles_notindb_form(); - - // Return the form - return $form; +function auditfiles_notindb() { + return drupal_get_form('auditfiles_notindb_form'); } /** * Form definition for audit files not in database */ -function auditfiles_notindb_form() { - +function auditfiles_notindb_form(&$form_state) { + if (isset($form_state['storage']['confirm'])) { + return auditfiles_notindb_form_confirm($form_state); + } // Get the list of files that aren't in the database $filesnotindb = _auditfiles_filesnotindb(); @@ -56,6 +48,8 @@ function auditfiles_notindb_form() { $options = array( 'donothing' => t('Do nothing'), 'delete' => t('Delete checked files'), + 'add' => t('Add checked files to database'), + 'addall' => t('Add all files to database'), ); $form['options']['operation'] = array( @@ -84,9 +78,6 @@ function auditfiles_notindb_form() { // Add list of files to checkboxes $form['files'] = array('#type' => 'checkboxes', '#options' => $files); - // Specify theme for form - $form['#theme'] = 'auditfiles_notindb_form'; - // Return form return $form; } @@ -96,7 +87,6 @@ function auditfiles_notindb_form() { * Theme auditfiles_notindb_form */ function theme_auditfiles_notindb_form($form) { - // Render count $output .= drupal_render($form['count']); @@ -108,8 +98,8 @@ function theme_auditfiles_notindb_form($ // Construct table of files $header = array( - t('Delete?'), - t('File') + t('Select'), + t('File'), ); foreach (element_children($form['file']) as $key) { @@ -124,66 +114,126 @@ function theme_auditfiles_notindb_form($ } + $output .= drupal_render($form); + // Return output return $output; } +function auditfiles_notindb_form_submit($form, &$form_state) { + //drupal_set_message("submit form_state: ".dpr($form_state, TRUE)); + if ($form_state['clicked_button']['#id'] == 'edit-submit' && + $form_state['values']['options']['operation'] <> 'donothing') { + $form_state['storage']['confirm'] = TRUE; + $form_state['storage']['values'] = $form_state['values']; + } +} -/** - * Form action: Confirm deletion of selected items - */ -function auditfiles_multiple_delete_confirm(&$form_state, $files = array()) { - - $form['files'] = array('#prefix' => '', '#tree' => TRUE); - - // array_filter in the call returned only elements with TRUE values - foreach ($files as $file) { - $form['files'][$file] = array('#type' => 'hidden', '#value' => $file, '#prefix' => '
  • ', '#suffix' => check_plain($file) ."
    • \n"); +function auditfiles_notindb_form_confirm(&$form_state) { + $values = $form_state['storage']['values']; + //drupal_set_message("form_confirm: values: ".dpr($values, TRUE)); + $operation = $values['operation']; + switch ($operation) { + case 'delete': + $optype = 'delete'; + $seltype = 'selected'; + break; + case 'add': + $optype = 'add'; + $seltype = 'selected'; + break; + case 'addall': + $optype = 'add'; + $seltype = 'all'; + break; + case 'donothing': + default: + return; + } + + $form['changelist'] = array('#prefix' => '', '#tree' => TRUE); + + foreach ($values['files'] as $file => $selected) { + if ($seltype == 'all' || $selected) { + if ($optype == 'delete') { + $message = "$file will be deleted"; + } else { + $message = "$file will be added to the database"; + } + $form['changelist'][$file] = array('#type' => 'hidden', '#value' => $file, '#prefix' => '
  • ', '#suffix' => $message ."
    • \n"); + } else { + // Unsetting the unprocessed files prevents confirm_submit from dealing with them + unset($form_state['storage']['values']['files'][$file]); + } } - $form['operation'] = array('#type' => 'hidden', '#value' => 'delete'); - $form['#submit'][] = 'auditfiles_multiple_delete_confirm_submit'; + + $form['operation'] = array('#type' => 'hidden', '#value' => $operation); + $form['#submit'][] = 'auditfiles_notindb_form_confirm_submit'; return confirm_form( $form, - t('Are you sure you want to delete these items?'), + t('Are you sure you want to make these changes?'), 'admin/reports/auditfiles/notindb', ''. t('This action cannot be undone.') .'', - t('Delete all'), + t('Process all'), t('Cancel') ); } - -/** - * Form action: Delete selected files - */ -function auditfiles_multiple_delete_confirm_submit($form, &$form_state) { +function auditfiles_notindb_form_confirm_submit($form, &$form_state) { +// drupal_set_message('confirm_submit, form: '.dpr($form, TRUE)); +// drupal_set_message('confirm_submit, form_state: '.dpr($form_state, TRUE)); if ($form_state['values']['confirm']) { - foreach ($form_state['values']['files'] as $file) { - if (file_delete(file_create_path($file))) { - watchdog('audit', '%file was deleted', array('%file' => $file)); + $values = $form_state['storage']['values']; + $operation = $values['options']['operation']; + if ($operation != 'delete') { + global $user; + if (function_exists('finfo_file')) { + $finfo = @finfo_open(FILEINFO_MIME); } - else { - drupal_set_message(t('Failed to delete %file', array('%file' => $file ))); + } + //drupal_set_message("in confirm_submit, values: ".dpr($values,TRUE)); + foreach ($values['files'] as $filename => $selected) { + if ($operation == 'delete') { + if (file_delete(file_create_path($filename))) { + watchdog('audit', '%file was deleted', array('%file' => $filename)); + } + else { + drupal_set_message(t('Failed to delete %file', array('%file' => $filename))); + } + } else { + $fullpath = file_create_path($filename); + $file = new stdClass(); + $file->filename = trim(basename($filename)); + $file->filepath = trim($fullpath); + // Adapted from imce + if ($finfo) { + $type = finfo_file($finfo, $fullpath); + } elseif ($info = @getimagesize($fullpath)) { + $type = $info['mime']; + } elseif (function_exists('mime_content_type')) { + $type = mime_content_type($fullpath); + } else { + $type = 'application/x-download'; + } + $file->filemime = $type; + $file->filesize = filesize($fullpath); + $file->uid = $user->uid; + $file->status = FILE_STATUS_PERMANENT; + $file->timestamp = time(); + drupal_write_record('files', $file); } } - drupal_set_message(t('The items have been deleted.')); + if ($operation == 'delete') { + drupal_set_message(t('The files have been deleted.')); + } else { + drupal_set_message(t('The files have been added to the database.')); + } + // Clear so our return to the primary form doesn't think we're going to the confirmation form + unset($form_state['storage']['confirm']); } - $form_state['redirect'] = 'admin/reports/auditfiles/notindb'; - return; -} - - -/** - * Form action: Submit the deletion form - */ -function auditfiles_notindb_submit($form, &$form_state) { - // Force rebuild to populate form data - $form_state['rebuild'] = TRUE; - } - /** * Helper function - retrieve sorted list of files that are on the server * but not in the database