Index: feeds.module =================================================================== RCS file: /cvs/drupal-contrib/contributions/modules/feeds/feeds.module,v retrieving revision 1.17 diff -u -p -r1.17 feeds.module --- feeds.module 31 Oct 2009 15:19:12 -0000 1.17 +++ feeds.module 8 Nov 2009 17:58:16 -0000 @@ -341,6 +341,47 @@ function _feeds_nodeapi_node_processor($ } /** + * Operation callback for FeedsBatchTransporter. + */ +function _feeds_batch_operation($items_num, $batch_size, $offset, &$context) { + // We don't know what objects that are cached in the session. So we must + // include all definitions before we safely can unserialize the objects. + feeds_importer_load_all(); + + // Retain the objects to the next request. + $_SESSION['FeedsParserResult']; + $_SESSION['FeedsSource']; + + // Get the cached objects from the session. + $parserResult = unserialize($_SESSION['FeedsParserResult']); + $source = unserialize($_SESSION['FeedsSource']); + + $end = ($batch_size + $offset); + $end = ($end > $items_num) ? $items_num : $end; + + $batched_values = array(); + for ($i = 0; $i < $end; $i++) { + if ($i >= $offset) { + $batched_values[] = $parserResult->value['items'][$i]; + } + } + + // We only pass the batched values to the processor. + $parserResult->value['items'] = $batched_values; + + // Process the FeedsParserResult. + $source->importer->processor->process($parserResult, $source); +} + +/** + * Finish callback for FeedsBatchTransporter. + */ +function _feeds_batch_finished($success, $results, $operations) { + // Now we don't need to cache the objects any more. + unset($_SESSION['FeedsParserResult'], $_SESSION['FeedsSource'], $_SESSION['FeedsProcessor']); +} + +/** * Implementation of hook_form_alter(). */ function feeds_form_alter(&$form, $form_state, $form_id) { @@ -667,6 +708,9 @@ function feeds_plugin_type($plugin_key) elseif (feeds_plugin_child($plugin_key, 'FeedsParser')) { return 'parser'; } + elseif (feeds_plugin_child($plugin_key, 'FeedsTransporter')) { + return 'transporter'; + } elseif (feeds_plugin_child($plugin_key, 'FeedsProcessor')) { return 'processor'; } Index: feeds.plugins.inc =================================================================== RCS file: /cvs/drupal-contrib/contributions/modules/feeds/feeds.plugins.inc,v retrieving revision 1.2 diff -u -p -r1.2 feeds.plugins.inc --- feeds.plugins.inc 21 Oct 2009 20:01:18 -0000 1.2 +++ feeds.plugins.inc 8 Nov 2009 17:58:16 -0000 @@ -47,6 +47,15 @@ function _feeds_feeds_plugins() { 'path' => $path, ), ); + $info['FeedsTransporter'] = array( + 'hidden' => TRUE, + 'handler' => array( + 'parent' => 'FeedsPlugin', + 'class' => 'FeedsTransporter', + 'file' => 'FeedsTransporter.inc', + 'path' => $path, + ), + ); $info['FeedsProcessor'] = array( 'hidden' => TRUE, 'handler' => array( @@ -76,6 +85,26 @@ function _feeds_feeds_plugins() { 'path' => $path, ), ); + $info['FeedsDirectTransporter'] = array( + 'name' => 'Direct transporter', + 'description' => 'Transports parsed items directly to the processor.', + 'handler' => array( + 'parent' => 'FeedsTransporter', + 'class' => 'FeedsDirectTransporter', + 'file' => 'FeedsDirectTransporter.inc', + 'path' => $path, + ), + ); + $info['FeedsBatchTransporter'] = array( + 'name' => 'Batch transporter', + 'description' => 'Transports parsed items to the processor through the Batch API.', + 'handler' => array( + 'parent' => 'FeedsTransporter', + 'class' => 'FeedsBatchTransporter', + 'file' => 'FeedsBatchTransporter.inc', + 'path' => $path, + ), + ); $info['FeedsCSVParser'] = array( 'name' => 'CSV parser', 'description' => 'Parse data in Comma Separated Value format.', Index: feeds_defaults/feeds_defaults.defaults.inc =================================================================== RCS file: /cvs/drupal-contrib/contributions/modules/feeds/feeds_defaults/feeds_defaults.defaults.inc,v retrieving revision 1.2 diff -u -p -r1.2 feeds_defaults.defaults.inc --- feeds_defaults/feeds_defaults.defaults.inc 28 Oct 2009 00:17:11 -0000 1.2 +++ feeds_defaults/feeds_defaults.defaults.inc 8 Nov 2009 17:58:16 -0000 @@ -176,6 +176,10 @@ function _feeds_defaults_feeds_importer_ 'plugin_key' => 'FeedsSyndicationParser', 'config' => array(), ), + 'transporter' => array( + 'plugin_key' => 'FeedsDirectTransporter', + 'config' => array(), + ), 'processor' => array( 'plugin_key' => 'FeedsNodeProcessor', 'config' => array( @@ -240,6 +244,10 @@ function _feeds_defaults_feeds_importer_ 'plugin_key' => 'FeedsSyndicationParser', 'config' => array(), ), + 'transporter' => array( + 'plugin_key' => 'FeedsDirectTransporter', + 'config' => array(), + ), 'processor' => array( 'plugin_key' => 'FeedsDataProcessor', 'config' => array( @@ -296,6 +304,10 @@ function _feeds_defaults_feeds_importer_ 'delimiter' => ',', ), ), + 'transporter' => array( + 'plugin_key' => 'FeedsDirectTransporter', + 'config' => array(), + ), 'processor' => array( 'plugin_key' => 'FeedsNodeProcessor', 'config' => array( @@ -349,6 +361,10 @@ function _feeds_defaults_feeds_importer_ 'plugin_key' => 'FeedsOPMLParser', 'config' => array(), ), + 'transporter' => array( + 'plugin_key' => 'FeedsDirectTransporter', + 'config' => array(), + ), 'processor' => array( 'plugin_key' => 'FeedsFeedNodeProcessor', 'config' => array( @@ -393,6 +409,10 @@ function _feeds_defaults_feeds_importer_ 'delimiter' => ',', ), ), + 'transporter' => array( + 'plugin_key' => 'FeedsDirectTransporter', + 'config' => array(), + ), 'processor' => array( 'plugin_key' => 'FeedsUserProcessor', 'config' => array( Index: includes/FeedsImporter.inc =================================================================== RCS file: /cvs/drupal-contrib/contributions/modules/feeds/includes/FeedsImporter.inc,v retrieving revision 1.4 diff -u -p -r1.4 FeedsImporter.inc --- includes/FeedsImporter.inc 21 Oct 2009 22:49:47 -0000 1.4 +++ includes/FeedsImporter.inc 8 Nov 2009 17:58:17 -0000 @@ -22,8 +22,9 @@ abstract class FeedsResult { protected $valid_types = array(); // The type of this result. protected $type; - // The value of this result. - protected $value; + // The value of this result is public so FeedsTransporter can access it + // and split it up in batches. + public $value; /** * Constructor: create object, validate class variables. @@ -96,10 +97,10 @@ class FeedsImporter extends FeedsConfigu // Every feed has a fetcher, a parser and a processor. // These variable names match the possible return values of // feeds_plugin_type(). - protected $fetcher, $parser, $processor; + protected $fetcher, $parser, $transporter, $processor; // This array defines the variable names of the plugins above. - protected $plugin_types = array('fetcher', 'parser', 'processor'); + protected $plugin_types = array('fetcher', 'parser', 'transporter', 'processor'); /** * Instantiate class variables, initialize and configure @@ -136,7 +137,7 @@ class FeedsImporter extends FeedsConfigu try { $result = $this->fetcher->fetch($source); $result = $this->parser->parse($result, $source); - $this->processor->process($result, $source); + $this->transporter->transport($result, $source); } catch (Exception $e) { drupal_set_message($e->getMessage(), 'error'); Index: plugins/FeedsBatchTransporter.inc =================================================================== RCS file: plugins/FeedsBatchTransporter.inc diff -N plugins/FeedsBatchTransporter.inc --- /dev/null 1 Jan 1970 00:00:00 -0000 +++ plugins/FeedsBatchTransporter.inc 8 Nov 2009 17:58:17 -0000 @@ -0,0 +1,42 @@ + t('Batch import'), + 'operations' => array(), + 'finished' => '_feeds_batch_finished', + ); + + $items_num = count($parserResult->value['items']); + $operations_num = ceil($items_num / $this->batch_size); + + for ($i = 0; $i < $operations_num; $i++) { + $offset = ($this->batch_size * $i); + $batch['operations'][] = array('_feeds_batch_operation', array($items_num, $this->batch_size, $offset)); + } + // Set the batch. + batch_set($batch); + } + +} \ No newline at end of file Index: plugins/FeedsDirectTransporter.inc =================================================================== RCS file: plugins/FeedsDirectTransporter.inc diff -N plugins/FeedsDirectTransporter.inc --- /dev/null 1 Jan 1970 00:00:00 -0000 +++ plugins/FeedsDirectTransporter.inc 8 Nov 2009 17:58:17 -0000 @@ -0,0 +1,18 @@ +importer->processor->process($parserResult, $source); + } + +} \ No newline at end of file Index: plugins/FeedsTransporter.inc =================================================================== RCS file: plugins/FeedsTransporter.inc diff -N plugins/FeedsTransporter.inc --- /dev/null 1 Jan 1970 00:00:00 -0000 +++ plugins/FeedsTransporter.inc 8 Nov 2009 17:58:17 -0000 @@ -0,0 +1,19 @@ +