Closed (fixed)
Project:
Migrate
Version:
7.x-2.4-beta1
Component:
Code
Priority:
Normal
Category:
Support request
Assigned:
Unassigned
Reporter:
Created:
16 May 2012 at 14:54 UTC
Updated:
31 May 2012 at 14:20 UTC
I updated to Migrate 7.x-2.4-beta1, hoping that it might solve another issue I had run into with an extremely large record set (270k records). After the update I bumped into the fact that my file handling no longer works. I have read the documentation provided at http://drupal.org/node/1540106 but I am not sure how to adapt my code. My confusion stems from the fact that the documentation seems to be geared towards file-specific migrations, while I am merely migrating files as part of a node migration from XML.
Here is my current code:
class DocumentMigration extends XMLMigration {
//private $sourceid;
public function __construct() {
parent::__construct();
$this->description = t('XML feed (multi items) of roles (positions)');
// There isn't a consistent way to automatically identify appropriate "fields"
// from an XML feed, so we pass an explicit list of source fields
$fields = array(
'filename' => t('File name'),
'title' => t('Publication title'),
'country' => t('Publication country'),
'state' => t('Publication state'),
'city' => t('Publication city'),
'date' => t('Publication date'),
'lang' => t('Language'),
'pagenum' => t('Page number'),
'exceptions' => t('Exceptions'),
'id' => t('Image ID'),
);
// The source ID here is the one retrieved from the XML listing file, and
// used to identify the specific item's file
$this->map = new MigrateSQLMap($this->machineName,
array(
'id' => array(
'type' => 'varchar',
'length' => 255,
'not null' => TRUE,
'description' => 'The Image ID',
)
),
MigrateDestinationNode::getKeySchema()
);
// This can also be an URL instead of a file path.
$xml_folder = variable_get('migrate_preservation_document_path_absolute', DRUPAL_ROOT);
$files = file_scan_directory($xml_folder, '/\.xml$/', array('recurse' => FALSE));
$items_url = array_keys($files);
$item_xpath = '/reel/image'; // relative to document
$item_ID_xpath = 'id'; // relative to item_xpath
$this->source = new MigrateSourceXML($items_url, $item_xpath, $item_ID_xpath, $fields);
$this->destination = new MigrateDestinationNode('document');
$this->addFieldMapping('id');
$this->addFieldMapping('sourceid');
$this->addFieldMapping('field_document_docid', 'docid')
->description('Populated by prepareRow');
$this->addFieldMapping('title', 'full_title')
->description('Populated by prepareRow');
$this->addFieldMapping('field_document_country', 'country')
->xpath('country');
$this->addFieldMapping('field_document_state', 'state')
->xpath('state');
$this->addFieldMapping('field_document_city', 'city')
->xpath('city');
$this->addFieldMapping('field_document_pubtitle', 'title')
->xpath('title');
$this->addFieldMapping('field_document_pagenum', 'pagenum')
->xpath('pagenum');
$this->addFieldMapping('field_document_text_raw', 'raw_text') // works!
->description('Populated by prepareRow from a text file');
$this->addFieldMapping('field_document_dirid', 'directory_id')
->description('Populated by prepareRow');
$this->addFieldMapping('field_document_collection', 'collection')
->description('Populated by prepareRow');
$this->addFieldMapping('field_document_filename', 'filename')
->xpath('filename');
$this->addFieldMapping('field_document_pdf', 'pdf');
// Date handled as separate fields because of limitations on php dates.
$this->addFieldMapping('field_document_year', 'year');
$this->addFieldMapping('field_document_month', 'month');
$this->addFieldMapping('field_document_day', 'day');
}
public function prepareRow($row) {
$filename = (string) $row->xml->filename;
$path = $this->source->activeUrl();
$row->docid = $this->getDocId($path, $filename);
$row->raw_text = $this->getTextFile($path, $filename);
$row->directory_id = $this->getDirectoryId($path);
$row->collection = $this->getCollection($path);
$row->full_title = $this->getFullTitle($row);
$row->pdf = $this->getPDF($path, $filename);
if ((string) $row->xml->pagenum == '') {
$row->pagenum = 0;
}
$dateArr = explode('-', (string) $row->xml->date);
$row->year = $dateArr[0];
$row->month = $dateArr[1];
$row->day = $dateArr[2];
}
public function getFullTitle($row) {
return (string) $row->xml->title . ' ' . (string) $row->xml->date . ' - Page ' . (string) $row->xml->pagenum;
}
public function getDocId($xmlpath, $scanfilename) {
$file = basename($xmlpath);
$filename = substr($file, 0, strrpos($file, '.'));
$parts = explode('@#@', $filename);
$docid = $parts[0] . '--' . $parts[1] . '--' . $scanfilename;
return $docid;
}
public function getTextFile($xmlpath, $scanfilename) {
$path = $this->getFilePath($xmlpath, $scanfilename, '.txt');
$data = file_exists($path) ? utf8_encode(file_get_contents($path)) : '';
return $data;
}
public function getPDF($xmlpath, $scanfilename) {
$path = $this->getFilePath($xmlpath, $scanfilename, '.pdf');
$pdf = array();
$pdf['path'] = $path;
$pdf = drupal_json_encode($pdf);
return $pdf;
}
public function getFilePath($xmlpath, $filename, $ext) {
$xmlfile = basename($xmlpath);
$xmlname = substr($xmlfile, 0, strrpos($xmlfile, '.'));
$parts = explode('@#@', $xmlname);
$path = variable_get('migrate_preservation_document_path_absolute', DRUPAL_ROOT) . '/' . $parts[0] . '/';
if (file_exists($path . $parts[1] . '/' . $parts[1] . '/' . $filename . $ext)) { // Example: /Collection/Dates/Dates/File.ext
$path = $path . $parts[1] . '/' . $parts[1] . '/' . $filename . $ext;
} else if (file_exists($path . $parts[1] . '/' . $filename . $ext)) { // Example: /Collection/Dates/File.ext
$path = $path . $parts[1] . '/' . $filename . $ext;
} else if (file_exists($path . $filename . $ext)) { // Example: /Collection/File.ext
$path = $path . $filename . $ext;
} else { // Can't find the file
return FALSE;
}
return $path;
}
public function getDirectoryId($xmlpath) {
$file = basename($xmlpath);
$filename = substr($file, 0, strrpos($file, '.'));
$parts = explode('@#@', $filename);
return $parts[1];
}
public function getCollection($xmlpath) {
$file = basename($xmlpath);
$filename = substr($file, 0, strrpos($file, '.'));
$parts = explode('@#@', $filename);
return $parts[0];
}
}
Any help is greatly appreciated!
Sean
Comments
Comment #1
mikeryanThere's a lot there to parse through, but if I understand correctly field_document_pdf is the file field in question, correct? If so, then changing getPDF() to:
Or just calling getFilePath() directly should do the job. All you have to do is get 'pdf' to be the path to the source file.
Comment #2
pyrello commented@mikeryan - thanks for the speedy response and sorry that I didn't do a better job of pointing out the specific areas of the code that were relevant. This does appear to fix the problem... at least I am not getting the error I was getting before. There appears to be another issue that is coming into play, but I will open a separate support request for that. Thanks for your help!
Sean