Handlers for file / static content migration

Here's what I've got thus far.

/**
 * @file
 * Support for migration from static file sources. 
 */

/**
 * Implementation of MigrateList, for retrieving a list of IDs to be migrated
 * from a directory. 
 */
class MigrateListStatic extends MigrateList {
  /**
   * A URI pointing to a path to begin scanning. 
   *
   * @var string
   */
  protected $listUri;

  protected $fileMask;

  protected $directoryOptions;


  public function __construct($list_uri, $file_mask = NULL, $options = array()) {
    parent::__construct();
    $this->listUri = $list_uri;
    $this->fileMask = $file_mask;
    $this->directoryOptions = $options;
  }

  /**
   * Our public face is the URI we're getting items from
   *
   * @return string
   */
  public function __toString() {
    return $this->listUri;
  }

  /**
   * Retrieve a list of files based on parameters passed for the migration.
   *
   * @return array
   */
  public function getIdList() {
    migrate_instrument_start("Retrieve $this->listUri");
    $files = file_scan_directory($this->listUri, $this->fileMask, $this->directoryOptions);
    migrate_instrument_stop("Retrieve $this->listUri");
    if (isset($files)) {
      return $this->getIDsFromFiles($files);
    }
    $migration = Migration::currentMigration();
    $migration->showMessage(t('Loading of !listuri failed:',
        array('!listuri' => $this->listUri)));
    return NULL;
  }

  /**
   * Given an array generated from file_scan_directory(), parse out the IDs for processing
   * and return them as an array.
   *
   * @param array $files
   *
   * @return array
   */
  protected function getIDsFromFiles(array $files) {
    $ids = array();
    foreach ($files as $file) {
      $ids[] = str_replace($this->listUri, '', (string) $file->uri);
    }
    return array_unique($ids);
  }

  /**
   * Return a count of all available IDs from the source listing.
   */
  public function computeCount() {
    $count = 0;
    $files = $this->getIdList();
    if ($files) {
      $count = count($files);
    }
    return $count;
  }
}

/**
 * Implementation of MigrateItem, for retrieving a file from the file system based on source directory and
 * an ID provided by a MigrateList class.
 */
class MigrateItemStatic extends MigrateItem {

  protected $sourceUri;

  protected $fileMask;

  public function __construct($source_uri) {
    parent::__construct();
    $this->sourceUri = $source_uri;
  }

  /**
   * Implementors are expected to return an object representing a source item.
   *
   * @param mixed $id
   *
   * @return stdClass
   */
  public function getItem($id) {
    $item_uri = $this->sourceUri . $id;
    // Get the file data at the specified URI
    $data = $this->loadFile($item_uri);
    if ($data) {
      $return = new stdClass;
      $return->html = $data;
      return $return;
    }
    else {
      $migration = Migration::currentMigration();
      $message =  t('Loading of !objecturi failed:', array('!objecturi' => $item_uri));
      $migration->getMap()->saveMessage(
        array($this->id), $message, MigrationBase::MESSAGE_ERROR);
      return NULL;
    }
  }

  /**
   * Default File loader.
   */
  protected function loadFile($item_uri) {
    $data = file_get_contents($item_uri);
    return $data;
  }
}

And an example implementation..

class TestMigration extends Migration {
  public function __construct() {
    parent::__construct();
    $this->description = t('Data migration from static shtml.');

    $this->map = new MigrateSQLMap($this->machineName,
      array(
        'sourceid' => array(
          'type' => 'varchar',
          'length' => 255,
          'not null' => TRUE,
        )
      ),
      MigrateDestinationNode::getKeySchema()
    );

    $fields = array(
      'title' => t('Title'),
      'body' => t('Body'),
    );

    $list_url = DRUPAL_ROOT . '/' . drupal_get_path('module', 'test_migrate') . '/htdocs';

    $this->source = new MigrateSourceList(new MigrateListStatic($list_url, '/index\..html$/'),
      new MigrateItemStatic($list_url), $fields);

    $this->destination = new MigrateDestinationNode('page');

    $this->addFieldMapping('title', 'title');
    $this->addFieldMapping('body', 'body')
         ->arguments(array('format' => 'full_html'));
  }

  public function prepareRow($row) {
    require_once drupal_get_path('module', 'test_migrate') . '/libraries/QueryPath/QueryPath.php';
    $row->title = htmlqp($row->html)->find('h1')->text();
    $row->body = htmlqp($row->html)->find('p')->html();
  }
}

Sounds good!

Mark,

What a coincidence, I'd done the same thing ;). http://drupal.org/sandbox/jerdavis/1232588

Yeah, we should stabilize in one of these and then provide patches back to Mike.

Jer

Thanks for the follow up. I'll review there and we can continue work in that issue queue if that works for you.

Updating to needs review, I'll try to get this in my personal list to check too.