Hi!

I am trying to write a field collection field handler for the migrate module. As starter I used the code from the simple value fields fields handler that comes with the migrate module as I thought the field from field collection is pretty the same than an integer field. Instead of storing an integer value from the user, it stores an integer value which is the entity id.

Am I right there?

My first question is, has my array down here the right format? As I understand Field API (I don't know much about Field API) Where the array key says value down here. This is the name of a column in the db that ends with value and where the entity id is saved? In my case "field_ca_competitor_value ".

$return[$language][$delta]['value'] = 5555; // Test with some fake Entity ID 

If I run this in my migration I wolud assume that I get a new row in the field table that looks like this:

== Data for table field_data_field_ca_competitor

|------
|entity_type|bundle|deleted|entity_id|revision_id|language|delta|field_ca_competitor_value
|------
|node|competitor_analysis|0|6278|4818|de|0|5555  <-- This the fake ID from my migrate handler

But this is not happening. The node with the field inside is created, but the field is not populated.

So my second question is:
Can someone tell me if this is because there is no field colletion with the entity id (5555) at this moment? Does field collection check if there is data with this id and than avoiding that inconsistent state. I can't be a feature from the field API or? Because field API can not know that this field stores the entity id to a field collection, or can it? I thought migrate is just calling the field API for creating a row in the db with this field data. But maybe this works not like that, instead the data is written by the field collection module and at this place there is some consitency checking?!

Thanks for anything what helps me to understand this.


/*
 Example from the Migrate Module how this is done with simple value fields
  
  class MigrateValueFieldHandler extends MigrateFieldHandler {
  public function __construct() {
    $this->registerTypes(array('value', 'list', 'list_boolean', 'list_integer',
      'list_float', 'list_text', 'number_integer', 'number_decimal', 'number_float'));
  }

  public function prepare($entity, array $field_info, array $instance, array $values) {
    $migration = Migration::currentMigration();
    $arguments = (isset($values['arguments']))? $values['arguments']: array();
    $language = $this->getFieldLanguage($entity, $field_info, $arguments);
    // Setup the standard Field API array for saving.
    $delta = 0;
    foreach ($values as $value) {
      $return[$language][$delta]['value'] = $value;
      $delta++;
    }
    if (!isset($return)) {
      $return = NULL;
    }
    return $return;
  }
}

 */


// Define a way to get data into the field_collectio field in a node
// This means not getting data in the fileds of the field collection

class MigrateFieldCollectionFieldHandler extends MigrateFieldHandler {
  // Define the constructor of our Handler - it lets Migrate know what *types* of fields this handler is compatible against.
  public function __construct() {
    // Handle the field_collectio field type
    $this->registerTypes(array('field_collection'));
  }
 
 
  // Based on the entity and return an array of field values to get processed.
public function prepare($entity, array $field_info, array $instance, array $values) {
    $migration = Migration::currentMigration();
    $arguments = (isset($values['arguments']))? $values['arguments']: array();
    $language = $this->getFieldLanguage($entity, $field_info, $arguments);
    // Setup the standard Field API array for saving.
    $delta = 0; 
     $return[$language][$delta]['value'] = 5555; // Test with some fake Entity ID    
    
    if (!isset($return)) {
      $return = NULL;
    }
    return $return;
  }
}

CommentFileSizeAuthor
#234 extras.inc_.txt1.19 KBalbarin
#234 update.inc_.txt1.2 KBalbarin
#222 field_collection-migrate-1175082-222.patch6.04 KBfuerst
#220 Screenshot_6_26_13_7_43_PM.png73.64 KBbleen
#209 field_collection-migrate-1175082-209.patch6.5 KBdanylevskyi
#205 field_collection-migrate-1175082-205.patch7.4 KBalanburke
#201 field_collection-migrate-1175082-201.patch5.97 KBgreenjuls
#198 field_collection-migrate-1175082-198.patch5.7 KBasherry
#194 field_collection-migrate-1175082-194.patch5.95 KBgreenjuls
#192 field_collection-migrate-1175082-192.patch5.96 KBgreenjuls
#178 field_collection-migrate-1175082-178.patch6.01 KBdanylevskyi
#174 field_collection-1175082-174.patch5.51 KBstar-szr
#174 interdiff.txt3.72 KBstar-szr
#153 field_collection-migrate-1175082-interdiff-124-142.txt1.73 KBclemens.tolboom
#146 Screen Shot 2013-01-17 at 3.37.02 AM.png52.88 KBsetvik
#142 field_collection-migrate-1175082-142.patch5.46 KBalexweber
#140 field_collection-migrate-1175082-140.patch5.06 KBalexweber
#139 field_collection-migrate-1175082-139.patch419 bytesalexweber
#124 field_collection-migrate-1175082-124.patch5.43 KBdanylevskyi
#120 field_collection_migrate-entity_load-1175082-120.patch5.39 KBxcafebabe
#115 field_collection_migrate-entity_load-1175082-115.patch5.29 KBdarrenmothersele
#110 field_collection_migrate-entity_load-1175082-110.patch4.82 KBPierre Paul Lefebvre
#94 field_collection_migrate-no_host-1175082-94.patch5.23 KBDavid_Rothstein
#94 interdiff-92-94.txt1.53 KBDavid_Rothstein
#92 field_collection_migrate-no_host-1175082-87.patch5.41 KBradiobuzzer
#90 field_collection_migrate-no_host-1175082-86.patch718 bytesradiobuzzer
#85 field_collection_migrate-no_host-1175082-85.patch921 bytesjpklein
#79 demo_field_collection_migration.txt1.74 KBrocket777
#74 field_collection_migrate-1175082-74.patch5.66 KBmrfelton
#73 field_collection_migrate-1175082-73.patch9.8 KBmrfelton
#68 field_collection_migrate-1175082-68.patch5.21 KBmikeryan
#41 0001-Issue-1175082-File-collection-field-handler-for-migr.patch5.88 KBmarvil07
#36 field_collection-migrate-1175082-36.patch4.91 KBjhedstrom
#33 collection_migrate-1175082-33.patch4.92 KBmikeryan
#27 collection_migrate-1175082-27.patch4.92 KBmikeryan
#25 collection_migrate-1175082-24.patch4.75 KBmikeryan
#20 1175082-field_collection_migrate_handler-20.patch3.7 KBpcambra
#17 1175082-field_collection_migrate_handler-17.patch4.37 KBpcambra
#11 1175082-field_collection_migrate_handler-11.patch3.39 KBpcambra
#8 1175082-migrate_module_integration.patch3.74 KBBevan
#6 people_org.inc_.txt1.05 KBgrendzy
#2 1175082.patch3.03 KBgrendzy
#1 1175082.patch1.11 KBgrendzy
Support from Acquia helps fund testing for Drupal Acquia logo

Comments

grendzy’s picture

Title: File collection field handler for migrate module. Do I have the right the format of the field collection filed array? » File collection field handler for migrate module.
Category: support » feature
Status: Active » Needs work
FileSize
1.11 KB

I'm also a bit stuck on this; here's what I have so far.

grendzy’s picture

Status: Needs work » Needs review
FileSize
3.03 KB
mikeryan’s picture

Subscribe

fago’s picture

Probably it would make sense to base this upon #1168196: Add generic-entity integration ?

abeger’s picture

grendzy: I'm currently attempting to do a field collection migration. Do you have any sample code for how to use this class?

Thanks!

grendzy’s picture

FileSize
1.05 KB

example attached:

abeger’s picture

Thanks! I got it up & running yesterday and it's migrating my field collection like a champ.

Also, it's worth noting that this patch requires version Entity API 7.x-1.0-beta9 or later in order to work.

Bevan’s picture

Status: Needs review » Reviewed & tested by the community
FileSize
3.74 KB

This patch has some whitespace fixes and registers the file with Drupal 7's class registry, so that Migrate can discover it.

fago’s picture

Status: Reviewed & tested by the community » Needs work

Please do not RTBC your own patches.

I gave it a visual review:

+   *  Collection object to build.  Pre-filled with any fields mapped in the Migration.

Double spaces before "pre" and why is migration written in upper-case here? else it is't.

+   * Basic initialization

Missing point. Also in other comments.

+   *  Raw source data object - passed through to prepare/complete handlers.
+   * @return array

Minor, but there should be an empty line before @return.

+    foreach ((array)$collection as $field => $value) {

Missing space after (array).

+    $entity = entity_load_single('field_collection_item', $item_id);

Better use field_collection_item_load().

Bevan’s picture

Acknowledged. However, although I did upload that patch I don't consider it "mine". I only made one very minor change in addition to the previous patch.

pcambra’s picture

Status: Needs work » Needs review
FileSize
3.39 KB

Here's a patch applying fago comments and cleaning the file paths.

pcambra’s picture

By the way, I think this should be included in migrate_extras module (http://drupal.org/project/migrate_extras) more than in field collection, other projects are doing it that way (OG, Profile2...)

fago’s picture

Project: Field collection » Migrate Extras
Version: 7.x-1.x-dev » 7.x-2.x-dev
Component: Code » Miscellaneous

Makes sense - so I'm moving this issue to the migrate_extras queue.

mikeryan’s picture

Just copied over to the Migrate Extras project page from Migrate itself, don't know why I didn't have it in there before:

The ideal place to implement migration support for a contributed module is in that module, not in the Migrate or Migrate Extras modules. That way, the migration support is always self-consistent with the current module implementation - it's not practical for the migrate modules to keep up with changes to all other contrib modules.

I would prefer to see this go into the File collection module itself rather than Migrate Extras (but that is ultimately up to fago).

Thanks.

fago’s picture

Project: Migrate Extras » Field collection
Version: 7.x-2.x-dev » 7.x-1.x-dev

I'd be fine having it in field-collection either.

pcambra’s picture

Sorry about the confusion, I really thought that migrate extras was the place for this.

Anyways, I've tried this patch in a couple of migrations and seems fine to go.

pcambra’s picture

Had some issues when doing rollback of non-imported items, attaching fixed version

roderik’s picture

Some patch feedback:

#0
works great. No problems importing.
I was so happy that this is around. Like the whole module. As a non D7-experienced person, I jumped for joy when I discovered that field_collection would solve the big potential 'data problems' for me, in a nicer way then CCK3 could, let alone extra node types + nodereferences...

#1
It's not clear to me why 'hostEntityType' is defined as a destination field. This is for being able to create several different entity_types from items in the same import process?

Anyway, I didn't set it, after discovering that PHP gave a notice if I didn't pass it as an option in the constructor, like this:

$this->destination = new MigrateDestinationFieldCollection(
        MY_FIELDNAME, array('hostEntityType' => $host_entity_type));

And everything worked nicely, though the the "HostEntityType" field is still unmapped (and showing as such in the migration UI...)

It also wasn't clear to me what the HostEntityType should be set to. I understand by looking at the database after doing a test import, but... it would be nice if it was documented.

...actually, it probably needs a 'field_colllection_migrate_example' module with a documented import class?
(You can give me that task if you want... though I don't have time right now. More stuff to migrate urgently.)

#2
the last hunk in patch #18 (the only one for field_collection.module) fails. That's fine; it's in the -dev version already.

That's it.
So actually, I don't have any feedback except for 'documentation issues' (is that an RTBC blocker? I think yes?...) Great work.

roderik’s picture

Oh yeah,

#3
"Processed 1 (0 created, 0 updated, 0 failed, 0 ignored)"...
It seems an update counter is missing somewhere. The 'ignored' does work, but the 'created' doesn't.

pcambra’s picture

Thanks for the feedback and tests @roderik

As for #1 you need to provide a hostEntityType for loading the host entity in the import, I'd say that's required. You can use addUnmigratedDestinations in your _construct method for those mappings you prefer to ignore.

I've fixed #2 and #3 in the patch, I don't know what has happened with the code repeated, I carried that from the original patch, and I've added a call to increase imported elements.

As for the example, I don't know if fago would want to include that here, maybe it has more sense as a documentation page for the project.

roderik’s picture

Concerning the example: I mentioned it because these example migration classes seem to be the best/standard thing for learning migrations. I picked more up from those than from the d.o. handbook pages.
I see the Date module also ships a date_migrate_example module -- though I'm not sure a complete module is necessary. A field_collection_migrate_example.inc file could be enough.

A documentation page would be good too, if linked from the project page.

---

As for the hostEntityType field: I guess I can see the point that you want to provide a mapping for every database field.

You don't need to provide a value/mapping for it, though. I did without, and most people can do without -- since you do need to provide the hostEntityType as an argument to the MigrateDestinationFieldCollection constructor.

That's the main thing I want to clarify for people using this code - be it in a documentation page or an example .inc file.

pcambra’s picture

Maybe you could complete the patch with a field_collection_migrate_example :)

I have no special reason for keeping hostEntityType as a mapping itself if it can be done otherwise, feel free to change it in the patch and I'll give it a try for sure.

roderik’s picture

Assigned: Unassigned » roderik
marcushenningsen’s picture

This works really well, however, I encountered a few problems. After migration the status show only 13 of 81 rows imported, however, all rows have been imported. Also, rollback has no effect.

Have you experienced these problems? Are you interested in more details?

mikeryan’s picture

I'm now doing some field_collection migration myself, here's an updated patch with the following changes:

  1. Added comment describing usage.
  2. Removed hostEntityType field - just use the option to map the host type.
  3. Support for updating previously-imported collections (migrate-import --update).

Status: Needs review » Needs work

The last submitted patch, collection_migrate-1175082-24.patch, failed testing.

mikeryan’s picture

Status: Needs work » Needs review
FileSize
4.92 KB

Rerolled - fixed a problem I had where the host node loaded by entity_load_single had an instance of my collection field with an empty value, and ultimately the real field value ended up at [1] with the empty [0] still present, so I ended up with nothing.

Status: Needs review » Needs work

The last submitted patch, collection_migrate-1175082-27.patch, failed testing.

Ari Gold’s picture

Status: Needs work » Needs review

Status: Needs review » Needs work

The last submitted patch, collection_migrate-1175082-27.patch, failed testing.

jhedstrom’s picture

Testing the patch from #27, I'm getting these warnings (which trigger an exception, so the migration fails)

Argument 1 passed to MigrateSimpleFieldHandler::prepare() must be an instance of stdClass, instance of FieldCollectionItemEntity given
[PATH]/migrate/plugins/destinations/fields.inc,
line 140

mikeryan’s picture

Hmm, looks like when MigrateSimpleFieldHandler was added to the Migrate module, it did not obey the change to the API from #1132034: Don't assume entity is an stdClass. I'll fix that in Migrate - in the meantime, guess I need to reroll this patch.

mikeryan’s picture

Status: Needs work » Needs review
FileSize
4.92 KB

Rerolled patch.

jhedstrom’s picture

Status: Needs review » Needs work
+++ b/field_collection.migrate.incundefined
@@ -0,0 +1,144 @@
+      if (empty($host_entity->{$this->bundle}->{$host_entity->language}[0]['value'])) {
+        unset($host_entity->{$this->bundle});

I may be using this incorrectly (I'm using a query that provides one row per collection item, rather than one row per host id), but the code from the patch above works in such a way as to only save the last collection to be imported for a given host id.

jhedstrom’s picture

Status: Needs work » Needs review

Didn't mean to change status.

jhedstrom’s picture

Same patch as #33, but fixed the host entity structure (language is an array on the bundle rather than an object).

+++ b/field_collection.migrate.incundefined
@@ -0,0 +1,144 @@
+      if (empty($host_entity->{$this->bundle}[$host_entity->language][0]['value'])) {
+        unset($host_entity->{$this->bundle});
azinck’s picture

#36 is good except my host entity is a taxonomy term which doesn't have a language. This works for me:

if (empty($host_entity->{$this->bundle}[(isset($host_entity->language) ? $host_entity->language : LANGUAGE_NONE)][0]['value'])) {
alphageekboy’s picture

I am glad to see this patch for field_collection. Having I think it is currently missing one part that may be an edge case but probably should be accounted for. I can't seem to get this to work for a Field collection that has a file field with multiple files.

I currently have:

Field Collection Group
---- Meta Data About File
---- File Path

Field Collection Group
---- Meta Data About File
---- File Path
...

It seems that only the first file is associated with the collection and the additional files are ignored. I'm in the process of coming up with a solution for my migration, but thought I would throw this out there and see if anyone had a good idea on how to implement this.

Also, perhaps field_collection is working correctly and the issue is with Migrate updating existing nodes and has more to do with this issue: http://drupal.org/node/1348322.

bastnic’s picture

@alphageekboy, i had a similar case but it was due to this test

if (empty($host_entity->{$this->bundle}[$host_entity->language][0]['value'])) {
  unset($host_entity->{$this->bundle});
}

$host_entity->language equals "en", but the field_collection was saved with "und", so the last saved field collection item was always unset.

Hope that's will help you.

alphageekboy’s picture

@bastnic -

I noticed this too when tracing through the code. However, this did not fix my issue. It seems like the issue is deeper and related to the fact that my file is in a multi-value field within a collection. In tracing through the code, it doesn't look like the field_collection migrate takes into consideration there could be more than one value passed to it that it needs to account for. In my case, I'm passing a multi-value "description" and "file path". I'm still tracing through trying to figure out if this is a Migrate module issue or a field_collection migrate issue.

I am looking for alternative methods at this point, but if I find a solution for this I will post back here.

**UPDATE***
It's kind of a hack for right now, but the solution was to change where the for each loop begins and modify the code accordingly. Essentially, I need the entity_load and entity_create to be in the loop so that for additional values will have their entity created. To simplify troubleshooting I also modified my requirements so that I only need to add the file_path and not the file_descriptive_name. I will try to carve out some time to clean up the code and hopefully submit a patch back that works.

If you have any questions about this before I do that, please don't hesitate to contact me and I can explain in more detail.

marvil07’s picture

This is the patch at comment 36, but it also includes an implementation of hook_migrate_api().

aendra’s picture

Any update on an example file or some documentation? I need to import a tonne of content into field collections and I'd be super appreciative of any help you could give... Thanks!

aendra’s picture

I've modified the example code at http://drupal.org/files/issues/people_org.inc_.txt, and when I try to use the UI to migrate my field collection, I get the following notice:

Notice: Undefined index: host_entity_type in MigrateDestinationFieldCollection->__construct() (line 50 of /home/username/sites/all/modules/field_collection/field_collection.migrate.inc).

Any clue what I'm doing wrong? Thanks!

azinck’s picture

this:

$this->destination = new MigrateDestinationFieldCollection('field_person_section', array('hostEntityType' => 'node'));

needs to become this:

$this->destination = new MigrateDestinationFieldCollection('field_person_section', array('host_entity_type' => 'node'));

You'll also find you need to change this:

 $this->addFieldMapping('hostEntityId', 'people_id')->sourceMigration('People');

to this:

 $this->addFieldMapping('host_entity_id', 'people_id')->sourceMigration('People');
aendra’s picture

Thank you so much! That helped a lot; at least it seems like I'm progressing.

Not to turn this into too much of a support ticket, but now I'm having issues with it detecting dependency migrations. Given the code in the top comments of field_collection.migrate.inc, I've tried adding the following:

$this->dependencies = array('Stories');

However, this gives me the following error:

  • Skipped Bylines due to unfulfilled dependencies
  • Processed 5 (5 created, 0 updated, 0 failed, 0 ignored) in 2.4 sec (123/min) - done with 'Stories'

If I remove the dependency line, I get the following lines:

  • Processed 5 (5 created, 0 updated, 0 failed, 0 ignored) in 0.3 sec (987/min) - done with 'Stories'
  • array_flip() [function.array-flip]: Can only flip STRING and INTEGER values! File /home/gauntlet/webapps/d7/includes/entity.inc, line 179
  • array_flip() [function.array-flip]: Can only flip STRING and INTEGER values! File /home/gauntlet/webapps/d7/includes/entity.inc, line 179
  • array_flip() [function.array-flip]: Can only flip STRING and INTEGER values! File /home/gauntlet/webapps/d7/includes/entity.inc, line 179
  • array_flip() [function.array-flip]: Can only flip STRING and INTEGER values! File /home/gauntlet/webapps/d7/includes/entity.inc, line 179
  • array_flip() [function.array-flip]: Can only flip STRING and INTEGER values! File /home/gauntlet/webapps/d7/includes/entity.inc, line 179
  • Processed 5 (0 created, 0 updated, 5 failed, 0 ignored) in 0.6 sec (499/min) - done with 'Bylines'

I put dpm( debug_backtrace() ); at line 178 of entity.inc and it seems like it's being given a empty array or something.

Any idea what I'm doing wrong? If I can figure this out, I'd probably be willing to write an example module if that would help the cause...

Thanks!

Full pastebin of my code at http://pastebin.com/EpTmg0nV

azinck’s picture

@aendrew: You're running Stories before Bylines, right?

I'm not 100% sure as to the solution to your problem, but I could guess that maybe your DB records in Bylines aren't providing valid sid values. Here are a couple links that might help:
http://drupal.org/node/1323442
http://stackoverflow.com/questions/4798047/array-flipcan-only-flip-strin...

aendra’s picture

Thanks for the help. Apparently:

$this->softDependencies = array('Stories');

...Was what I was looking for. I found I was able to import *some* stories (some errored up) if I removed the dependency line, imported a bunch of stories first, then imported a bunch of bylines. Making it a soft-dependency ensures the correct order is used, without requiring my entire 15,000 entry table to be imported first.

Now I just need to figure out why my byline names aren't importing as taxonomy terms and I'll be set!

agentrickard’s picture

It doesn't appear to me that this current patch supports a single Field Collection that is attached to different node bundles.

For example:

* Field collection 'foo' is attached to both Page and Article nodes.

The fieldMapping for 'host_entity_id' expects a string, which would lock each migration in to a single target bundle.

Is that correct? If so, it needs to be fixed, since this is a very common use case.

aendra’s picture

Hi!
Apologies for once again hijacking this thread, but I'm running into the "flip STRING and INTEGERS" issue again -- though somewhat spastically, which is why I'm having problems troubleshooting it (Even though I've since used the patch at http://drupal.org/node/1003788#comment-5195682 to improve the level of error detail.).

To start: I'm still using the two migration classes listed above (stories and bylines), as well as a third for images (field collection with a term reference, plus two text fields). When I migrate the byline class, everything succeeds except the very first record, and I get the following messages via migrate_ui:

1	Informational	Trying to get property of non-object File /sites/all/modules/field_collection/field_collection.migrate.inc, line 90(file: /sites/all/modules/field_collection/field_collection.migrate.inc, line 90)

1	Error	Missing bundle property on entity of type node. (/includes/common.inc:7501)

It doesn't matter if I migrate 5 or 500 (or 10 consecutive migrations of 50), each migration will always return one error.

When I migrate images, it happens anywhere from 12 to 24% of the time, with the same error.

What's up? Is it just unable to find a corresponding record? Any help you could give me would be great...

Nevermind, I think it's because there are non-existent nids being pointed to by the authors table. Any way I can tell Migrate to skip non-existent entities? Thanks!

agentrickard’s picture

Check the value in prepareRow() and return FALSE to skip a bad record. Ran into the same problem.

coredumperror’s picture

@ agentrickard:
I believe that you could accomplish what you're trying to do by defining a different *Migration class for each pair of source/destination content types. As far as I know that's what you have to do for any other type of field migration, anyway. Just tweak your source SQL query to only grab the source data that's associated with a single source content type.

agentrickard’s picture

I know you can do that, but I don't believe you should have to do that.

Being entities, Field Collections are a different beast than other fields, and should be treated that way.

coredumperror’s picture

I've been using the code from #41 for a while now, and it works great. But there's a minor bug with how it treats errors during migration. When I'm developing my migration classes, my code often throws exception when I do test migrates, causing the individual field's migration to fail. However, those exception are apparently not counted as "failed" migrations by MigrateDestinationFieldCollection. Even if all 10 of my test migrations throw exceptions, I see:

Processed 10 (10 created, 0 updated, 0 failed, 0 ignored) in 1.5 sec (392/min) - done with 'SmallImageField'

When I do drush mr SmallImageField, though, it outputs

Rolled back 0 in 0 sec (0/min) - done with 'SmallImageField'

Since no fields were actually migrated due to the exceptions.

I'd be happy to fix this myself, but I have no idea how.

coredumperror’s picture

I'm curious: would it be possible to write an alternative version of MigrateDestinationFieldCollection to make it act as a field handler, instead of a migrate destination? It would be really handy to be able to migrate my field collections like fields, instead of like independent entities that happen to be parented by nodes. I realize that that's exactly what Field Collections *are*, but aren't Field instances essentially the same thing?

I'd be willing to spend the time to learn how to re-write MigrateDestinationFieldCollection as FieldCollectionHandler, but right now I'm essentially clueless. I can't afford to waste several days on a hopeless project if turns out that it's not possible to migrate Field Collections using MigrateFieldHandler.

agentrickard’s picture

I suspect that it's not possible, having tried to write a custom field-based migration for a field collection before going with this patch instead.

bastnic’s picture

It would also be very much more efficient, because actually each field_collection entity saved causes the parent's node update.

coredumperror’s picture

Yeah, that's the major reason I'd like this to work, since updating the parent node alters the 'changed' date. This forces me to store that date somewhere so that I can change the date back to the original value during the complete() proc for each field collection.

mikeryan’s picture

My 2 cents: right now, the Migrate API handles fields that have internal structure - field collections with several sub-fields, file entities with alt/title/description/etc. values - very awkwardly. Doing the separate entity migration is much more convenient. On the other hand, as bastnic points out, it's inefficient because in the overall migration process you end up saving the parent node twice.

I certainly wouldn't want a field handler version to replace the current destination handler approach - if implemented, it would be an alternative which would make more sense in some applications than in others. I would suggest for now, though, that we go ahead and pursue getting the destination handler committed - the field handler alternative should be pursued as a separate issue in the queue.

zhensley’s picture

subscribe

artis.bajars’s picture

How would one use this to migrate data into a multi-valued field collection? I can't wrap my head around how to create more than one instance of the collection per host entity. My collection itself has 2 fields (single value) - a node reference and a textfield but I need to attach multiple instances of it to the target node. Am I missing something basic here or is this not possible with the current implementation?

Any help is appreciated, thanks!

coredumperror’s picture

How would one use this to migrate data into a multi-valued field collection?

Here's a (slightly) stripped down version of my ImageMigration class, which does what you're asking about:

class ImageMigration extends BaseMigration {

  public function __construct() {
    parent::__construct();
    
    // Ensure that all the Press Releases have already been migrated before doing these, since we need to
    // associate them to already-migrated nodes.
    $this->dependencies = array('PressRelease');
    $this->description = t('Migrate image assets from the database into Field Collections, which allows up to include the caption and photo credit fields.  Also attaches each Field Collection to the approprate Press Release.');
    
    $this->map = new MigrateSQLMap(
      $this->machineName,
      array(
        'id' => array(
          'type' => 'int',
          'not null' => TRUE,
          'description' => 'Asset ID',
        ),
      ),
      MigrateDestinationFieldCollection::getKeySchema()
    );

    // Include the image metadata, blob field, and associated press release ID in one row per image.
    $this->source_query = <big, complicated DBTNG thingy>
    
    $this->source = new MigrateSourceSQL($this->source_query, array(), NULL, array('map_joinable' => FALSE));
    $this->destination = new MigrateDestinationFieldCollection('field_image_fc', array('host_entity_type' => 'node'));

    ///////////////////////////////////////////////////////
    // Mapped fields
    ///////////////////////////////////////////////////////
    // Attach each field collection to the Press Release to which the attached image originally belonged.
    $this->addFieldMapping('host_entity_id', 'category_id')
      ->sourceMigration('PressRelease');
        
    $file_args = MigrateFileFieldHandler::arguments(array('source_field' => 'filename'), 'file_blob', FILE_EXISTS_ERROR, NULL,
      array('source_field' => 'alt'), array('source_field' => 'alt'), NULL, NULL, TRUE);
    $this->addFieldMapping('field_image_fc_image', 'data')
         ->description(t("Migrates each image blob directly into the file field attached to each field collection.  Uses the source alt text as both alt and title attributes."))
         ->arguments($file_args);
		
    $this->addFieldMapping('field_image_fc_caption', 'caption');
    $this->addFieldMapping('field_image_fc_photo_credit', 'photo_credit');
    
    $this->addFieldMapping(NULL, 'filename')
         ->description(t('This is migrated into the image field.'))
         ->issueGroup(t('Done'));
    $this->addFieldMapping(NULL, 'alt')
         ->description(t('This is migrated into the image field.'))
         ->issueGroup(t('Done'));
  }
}

field_image_fc is an unlimited-value Field Collection instance attached to the Press Release content type, while field_image_fc_image, field_image_fc_caption, and field_image_fc_photo_credit are the elements in the Field Collection. More than one source row can potentially have the same category_id, which allows multiple copies of the Field Collection to be migrated into one Press Release node.

artis.bajars’s picture

Thanks @coredumperror, that worked. Also had a problem where only the last row per host node was being attached, but that turned out to be the same node/field language mismatch issue as already reported in #37. Shouldn't that line be included in the patch as well?

valderama’s picture

I can confirm comment #62 - already imported field collection items get unset, when another field_collection item is attached to the same host entity, _if_ the language of the host enity does not match the language of the field collection.

This check needs to be more solid... the suggestion in comment #37 is a start...

drinkingcoffee’s picture

Just wanted to say thanks to everyone for these patches and example code -- it was extremely helpful.
Like #62 and #63, I had an issue with multiple field_collection items not being inserted. Trying the changes in #37 and other changes didn't work for me.

In the end (just when I was about to give up and write a script to insert directly into the db), I figured out a poor, hack-ish workaround, which I'll post here in case anyone finds themselves stuck in a similar situation:

- migrate in first set of field_collection items (so, all unique host entity ids)
- change the entity and revision id on the collection object in field_data_... and field_revision_... in some reversible way. I just added 100000 to all of them, and increment the delta value
- clear the migrate map table for this field_collection import (but leave the previous ones alone, so the host entity ids are still there)
- clear the cache
- migrate a second set of field_collection items
--repeat until all items are imported
- update the entity_id and revision_id back to the correct value (so, subtract 100000)
-??
-profit!

in my case, each of the items i was importing had between 1-4 attached field_collection items, so this whole process only looped 4 times, and while tedious (and easy to mess up at 2am), it only took 10-15 minutes, which is way faster than the alternatives.
obviously ymmv, and this would be a lot more complicated if you had to import objects that have a whole lot of field_collection_itmes in them.

mikeryan’s picture

Status: Needs review » Needs work

Last patch appears to need some work, based on comments since then. As it happens, I'm using this on another project now, one with multiple instances per host entities, so I'll be submitting a fresh patch soon.

cameron prince’s picture

How can a field_collection field be populated in conjunction with a node migration? I.E. using MigrateDestinationNode instead of MigrateDestinationFieldCollection. Is there a reason why there is no FieldHandler in the patch? I'm going to try and write one, but thought I'd ask first in case there's an architectural reason that prevents it.

I'm attempting to import some WordPress articles which have multiple images with captions. I have a field_collection field called field_article_images which is configured for unlimited values. The field_article_images collection contains a long text field called field_caption and a file field called field_image. This is my code from the migration's prepareRow function where I make a query back to the WordPress tables to gather all the attachment rows:


    $article_images = array();

    db_set_active('tuprod');
    $results = db_query("SELECT post_excerpt, guid FROM {wp_posts} WHERE post_type = 'attachment' AND post_parent = :pid ORDER BY post_name", array(':pid' => $row->id));
    db_set_active('default');

    foreach ($results as $r) {
      $article_images[] = array(
        'field_caption' => $r->post_excerpt,
        'field_image' => '/site/html/img/wp/' . basename($r->guid),
      );
    }

    $row->images = $article_images;

Can someone tell me if I'm on the right track here or if there's a better way?

aendra’s picture

@cameronbprince:

Re: no FieldHandler -- see the earlier posts, http://drupal.org/node/1175082#comment-5670794 and http://drupal.org/node/1175082#comment-5672314.

Having just done a migration similar to what you're wanting to do (except from a shoddily constructed custom DB instead of WP's), I get the feeling you should probably just use MigrateDestinationFieldCollection. I personally found it easier to troubleshoot migration issues when compartmentalizing the process between stories and field collections — that way, when the latter had problems, I was able to debug them fairly easily.

I'm hoping to write some more thorough documentation on using the MigrateDestinationFieldCollection class once I get a few moments to breathe this month; until then, there's a good example at http://drupal.org/files/issues/people_org.inc_.txt, though make sure to pay attention to the issue at http://drupal.org/node/1175082#comment-5529172 as some stuff has changed.

mikeryan’s picture

Status: Needs work » Needs review
FileSize
5.21 KB

Finally my latest version of the patch - only slightly tweaked from the previous one.

Status: Needs review » Needs work

The last submitted patch, field_collection_migrate-1175082-68.patch, failed testing.

bastnic’s picture

@mikeryan, I speed up this script a little bit with that trick. With that, while updating, it don't save the parent node.

//$status = entity_save('field_collection_item', $entity);
$status = $entity->save($updating);
mrfelton’s picture

I don't quite understand the host_entity_id part of this, and can't get it work. In my source system (a drupal site) I have a content type with a title and body, standard. In the target system, the title maps to the title of a new content type, but the body needs to go into a field on a field collection attached to that content type.

So I have exampleChapterMigration which runs a MigrateDestinationNode Migration, creating a new node for each in the old system. And then I have exampleChapterBrandingMigration which I'm trying to use to add a field collection to each of the previously created nodes. In exampleChapterBrandingMigration I have


    // Create a MigrateSource object, which manages retrieving the input data.
    $this->source = new MigrateSourceSQL($query, $source_fields, $count_query, array('map_joinable' => FALSE));

    // Set up our destination - node in this case.
    $this->destination = new MigrateDestinationFieldCollection('field_sac_chapter_contact', array('host_entity_type' => 'node'));

    // Attach each field collection to the OG Group to which it belongs.
    $this->addFieldMapping('host_entity_id', 'nid')->sourceMigration('exampleChapter');

But when it runs, it gives me something like this:

ResponseText: Undefined property: stdClass::$host_entity_id File /Users/tom/workspace/sac2/sites/all/modules/contrib/field_collection/field_collection.migrate.inc, line 89(file: /Users/tom/workspace/sac2/sites/all/modules/contrib/field_collection/field_collection.migrate.inc, line 89)Undefined property: stdClass::$nid File /Users/tom/workspace/sac2/sites/all/modules/contrib/migrate/plugins/sources/sqlmap.inc, line 301(file: /Users/tom/workspace/sac2/sites/all/modules/contrib/migrate/plugins/sources/sqlmap.inc, line 301

followed by a buch of array_flip errors.

Any ideas how it is that $host_entity_id is not set, yet I have set it in my field mapping?

mrfelton’s picture

Turns out I needed the nid included in the selected fields for the exampleChapterBrandingMigration class:

  ->fields('n', array('nid', 'vid'))

Now I'm just left with the array_flip errors :(

mrfelton’s picture

Status: Needs work » Needs review
FileSize
9.8 KB

Reroll to apply cleanly against latest codebase.

mrfelton’s picture

Not quit sure what happened with that last one... Got double inclusion of the new file. Try this one instead.

jpcurley25’s picture

Is there any idea as to an eta for this feature to be rolled out in a new version? (ie days, weeks or more?)

zorroposada’s picture

Thanks a lot for this patch #74.
I am using it as an include file in my custom migrate module.
It has so far worked perfect for migrating a field collection with a media field in it.

rocket777’s picture

subscribe

agentrickard’s picture

@rocket777 Please use the "Follow" button at the top of the page.

rocket777’s picture

The patch #74 works just fine for field_collections migration.
Thanks for this.

I have attached a demo file of my script as an example.

attiks’s picture

Status: Needs review » Needs work
+++ b/field_collection.migrate.incundefined
@@ -0,0 +1,155 @@
+      $language = isset($host_entity->language) ? $host_entity->language : LANGUAGE_NONE;
+      if (empty($host_entity->{$this->bundle}[$language][0]['value'])) {
+        unset($host_entity->{$this->bundle});
+      }

This gives problems with multi value fields, in my case the $host_entity->language is 'nl', but the field language is 'und', so all previous values get removed :/

Removing those lines fixes it (for me)

jmenard’s picture

How do I install the patch to get this working?

thx

dynamicdan’s picture

Title: File collection field handler for migrate module. » Field collection field handler for migrate module.

Um.. isn't this for the field collection and not file collection module?

Also, I don't believe that the field collection module currently supports multiple instances of the same field collection field. The closest solution is to create 1 field collection field and allow unlimited values. This however does not allow the fc field to be re-used with another label for example and is therefore not providing the ability to instance an existing fc field. Only the fc field entity can be instanced.

[EDIT] - ignore that last bit, this is drupal default functionality

alexrayu’s picture

Is there a way to migrate textfields sharing the same person id table into multiple values of a node? I am specifying this:

$this->addFieldMapping('host_entity_id', 'content_id')
         ->sourceMigration('Person');

In this code, person facts are to be migrated into a node Person, that had been migrated previously, creating multi-value fields inside a field collection. But it only applies 1 fact, skipping the rest.

Who can suggest what;s wrong? No errors, no "skipped" messaged on import.

brianV’s picture

Just to echo Alexei's concern in #83 - has anyone managed to import multiple field collection entites against a multivalue field collection field on a content type?

In our situation, we are trying to migrate a Person content type (working flawlessly). Each person has a collection of facts associated with them.

When we try to use the code as indicated, it only migrates the first Fact for each person. The remainder silently fail with no messages. This appears to be tied to the lack of a highwater mark.

jpklein’s picture

Here's an addition to the patch in #74 to fail gracefully in the case that the host entity of the field collection can't be identified during the migration.

mducharme’s picture

I've managed to import multiple field collection items onto a single content type by commenting out the language check altogether. I tried the extended language handler patch to handle taxonomies as well which still only imported a single item per node:

      // if (empty($host_entity->{$this->bundle}[$language][0]['value'])) {
      //   unset($host_entity->{$this->bundle});
      // }

This works for me but results in the field collection items being imported as language neutral on a node that hase a language set. I'm in a rush to get this done for a project and this is good enough for me right now. I'll hopefully have some time in the next two weeks to help out on this effort since I use field collections all the time.

mgifford’s picture

Status: Needs work » Needs review

Setting the bot on #85 - would be great if we could get this patch into the next dev release.

Status: Needs review » Needs work

The last submitted patch, field_collection_migrate-no_host-1175082-85.patch, failed testing.

Fidelix’s picture

This patch has wrong paths:

It tries to apply to /docroot/sites/all/modules/contrib/field_collection/field_collection.migrate.inc

radiobuzzer’s picture

Status: Needs work » Needs review
FileSize
718 bytes

Since it has already been 3 weeks from jpklein 's patch, I am re-submitting it with correct paths

Status: Needs review » Needs work

The last submitted patch, field_collection_migrate-no_host-1175082-86.patch, failed testing.

radiobuzzer’s picture

Status: Needs work » Needs review
FileSize
5.41 KB

woa, the patch should have been applied on the existing dev branch and not against the previous patch. Let's see now. I am submitting this patch without having reviewing it myself, it's just a repackaging of jpklein 's from 3 weeks ago.

radiobuzzer’s picture

EDIT: Removed an issue to another thread

David_Rothstein’s picture

Here is a reroll that removes the offending $entity->language code discussed in previous comments. This is the code which has been causing people trouble with multi-valued fields (it happened to me too), and I don't see the reason for the code being there in the first place. If the entity that is being loaded is broken and has incomplete field collections attached to it, it seems like that must be a bug somewhere else? Plus, I don't think that assuming "field language" = "entity language" is correct (as far as I understand the field API in Drupal 7, there is no such relationship).

This patch also makes a couple small improvements to the documentation.

Finally, I didn't change this:

+  static public function getKeySchema() {
+    return array(
+      'cid' => array(
+        'type' => 'int',
+        'unsigned' => TRUE,
+        'not null' => TRUE,
+        'description' => 'ID of field collection item',
+      ),
+    );

But wondered: Why is this using 'cid' rather than 'item_id' (i.e., shouldn't the name match the primary key from the {field_collection_item} table)?

Pierre Paul Lefebvre’s picture

Thank you, this patch works great with multiple values.

Angry Dan’s picture

I've just been using this patch and so far so good, although it did take me ages to notice that second argument to the constructor 'host_entity_type'.

I would suggest (sorry, I don't have time to re-roll the patch now) that we throw an exception in the constructor. For example:

  public function __construct($bundle, array $options = array()) {
    parent::__construct('field_collection_item', $bundle, $options);
    if (!isset($options['host_entity_type'])) {
      throw new MigrateException('MigrateDestinationFieldCollection requires setting the host_entity_type option at construction time.'); 
    }
    $this->hostEntityType = $options['host_entity_type'];
  }

Edited: fix syntax error

jaydub’s picture

I have a question regarding the multiple value field collection issue for migration. The patches in the last dozen or so comments seem to be concerned with multiple values but I am wondering if this is in reference to multiple value fields _within_ a field collection item or multiple value field collection items?

An example to clarify:

Scenario 1
- field collection item consists of 1 select list field (text) and 1 textfield
- properties for the select list field and/or textfield are set to allow 'unlimited' values
- So a single field collection item could include 2 selections for the select list field and 3 values for the textfield
- field collection item in host entity is set to limit of 1 value
- array('field_collection_item' => array('field_1' => array(1, 2), 'field_2' => array('foo', 'bar', 'baz')));

Scenario 2
- field collection item consists of 1 select list field (text) and 1 textfield
- properties for the select list field and textfield are set to limit of 1 value
- field collection item in host entity is set to allow 'unlimited' values
- array('field_collection_item' => array('field_1' => array(1), 'field_2' => array('foo')), 'field_collection_item' => array('field_1' => array(2), 'field_2' => array('bar')));

We are needing to migrate field collection data as described in scenario 2 and the patches don't seem to work with that. If I manually add two field collection item rows (each with 1 value for select list field and textfield) I actually get 2 field collection entities created in Drupal. I am guessing that the reason the patch to add migrate support doesn't work in our case is that migrate is based on creating 1 single field collection entity per migration source row.

If I am right in my thoughts what would be the method for us to migrate source field collection data given scenario 2?

Pierre Paul Lefebvre’s picture

@jaydub In my case I am in scenario 2. It's right all patches wasn't addressing this issue, until the last one. Feel free to contact me via my contact form if you need help.

jaydub’s picture

Thanks for the help @Pierre Paul Lefebvre

I believe that the patch as it currently stands needs to be rerolled as the changes made to field collection module and to entity module to support revisions means the $status of a migration via call to entity_save() will no longer return the expected value.

For background the changes to field collection to support revisions
(http://drupalcode.org/project/field_collection.git/commit/b21ecfc0b46a72...)

changed the save() method in the FieldCollectionItemEntity so that the method returns the result of an entity_save() of the host entity rather than the result of the entity_save() of the field collection entity as before.

before:

  public function save($skip_host_save = FALSE) {
    $transaction = db_transaction();
    try {
      // Make sure we have a host entity during creation.
      if (!empty($this->is_new) && !(isset($this->hostEntityId) || isset($this->hostEntity))) {
        throw new Exception("Unable to create a field collection item without a given host entity.");
      }
      $is_new = !empty($this->is_new);
      $return = entity_get_controller($this->entityType)->save($this, $transaction);

      // Create / update the link to the host entity.
      $host_entity = $this->hostEntity();
      if ($is_new && !$skip_host_save) {
        $delta = $this->delta();
        if (isset($delta)) {
          $host_entity->{$this->field_name}[$this->langcode][$delta] = array('value' => $this->item_id);
        }
        else {
          $host_entity->{$this->field_name}[$this->langcode][] = array('value' => $this->item_id);
        }
      }
      // Always save the host entity, so modules are able to react upon changes
      // to the content of the host and any 'last updated' dates of entities get
      // updated.
      if (!$skip_host_save) {
        entity_save($this->hostEntityType, $host_entity);
      }
      return $return;
    }
    catch (Exception $e) {
      $transaction->rollback($this->entityType, $e->getMessage(), array(), WATCHDOG_ERROR);
      return FALSE;
    }
  }

after:

  public function save($skip_host_save = FALSE) {
    // Make sure we have a host entity during creation.
    if (!empty($this->is_new) && !(isset($this->hostEntityId) || isset($this->hostEntity) || isset($this->hostEntityRevisionId))) {
      throw new Exception("Unable to create a field collection item without a given host entity.");
    }

    // Only save directly if we are told to skip saving the host entity. Else,
    // we always save via the host as saving the host might trigger saving
    // field collection items anyway (e.g. if a new revision is created).
    if ($skip_host_save) {
      return entity_get_controller($this->entityType)->save($this);
    }
    else {
      $host_entity = $this->hostEntity();
      if (!$host_entity) {
        throw new Exception("Unable to save a field collection item without a valid reference to a host entity.");
      }
      // If this is creating a new revision, also do so for the host entity.
      if (!empty($this->revision) || !empty($this->is_new_revision)) {
        $host_entity->revision = TRUE;
        if (!empty($this->default_revision)) {
          entity_revision_set_default($this->hostEntityType, $host_entity);
        }
      }
      // Set the host entity reference, so the item will be saved with the host.
      // @see field_collection_field_presave()
      $delta = $this->delta();
      if (isset($delta)) {
        $host_entity->{$this->field_name}[$this->langcode][$delta] = array('entity' => $this);
      }
      else {
        $host_entity->{$this->field_name}[$this->langcode][] =  array('entity' => $this);
      }
      return entity_save($this->hostEntityType, $host_entity);
    }
  }

For the migration patch the calling code in question is:

    migrate_instrument_start('field_collection_save');
    $status = entity_save('field_collection_item', $entity);
    migrate_instrument_stop('field_collection_save');

Before the changes for revision support the value of $status would be the result of the entity_save() of the field collection item, after the changes the value of $status would be the result of the entity_save() of the host entity. In my case the host entity is a node.

The code for entity_save() in entity.module is:

function entity_save($entity_type, $entity) {
  $info = entity_get_info($entity_type);
  if (method_exists($entity, 'save')) {
    return $entity->save();
  }
  elseif (isset($info['save callback'])) {
    $info['save callback']($entity);
  }
  elseif (in_array('EntityAPIControllerInterface', class_implements($info['controller class']))) {
    return entity_get_controller($entity_type)->save($entity);
  }
  else {
    return FALSE;
  }
}

In the case of the entity_save() made in the new version of field_collection.module the resultant call in entity_save() is the 2nd if clause ($info['save callback']($entity);) which for a node is node_save().

Notice that the 2nd conditional in entity_save() does not return anything and node_save() which is what is called does not return anything (http://api.drupal.org/api/drupal/modules%21node%21node.module/function/n...).

So what this means is that the $status variable in the field_collection.migrate.inc patch will be NULL in this case which means the migration is seen as failing when in fact the field collection entity is successfully created and the host entity is successfully saved.

Now what I don't have here is a patch since w/o the call to entity_save() returning anything from the entity_save() call on the host entity there is no reliable way to say the migration succeeded or failed. I suppose we could check for the $entity->item_id since if the migration didn't successfully create the field collection entity then there wouldn't be an $entity->item_id but that test wouldn't work if we were in an update context.

marcoka’s picture

i am trying to get some names into a field collection that is attached to a node. somehow i dont get it. does the node already have to exist for importing the data into the field collection?

in my case its a node type called root_word, it has a field collection that consists of 3 fields containing two int fields and one field for a word.

Ignore the entityReference field at first i havent found out how it works yet
http://www.root.artwaves.de/screensnapr/1350233577-XpJwHU.jpg

is it needed to use the fields() function somehow?

alexrayu’s picture

Right now, the code needs a host entity id. Which means, there should already be a node for that field collection.

So, your migration should go in 2 steps. In the first step you migrate the node, and then the field collection(s) as a dependent migration in the second step.

marcoka’s picture

ok, that dependency stuff is standard migrate core i guess. i will give it a try. when i have some working code i will post it.

marcoka’s picture

hm ok i am having some problems here. i created the nodes where the field collection is attached to. the second step now would be importing the field collection itself.
but here is the problem. how do i know to what node a collection belongs? i would have to somehow load the node with id X and to that node attach the data from query "get data for field collection for node x".
can do that with some temporary field on the node and custom foreach code but i am not sure here. i want to use migrate the right way. any tips/samples?

mducharme’s picture

If you're going to attach the field collection to a node that you manually created (not through migrate) then you will need to get the id of the node that you want to attach the field collection to. Assuming that you've already setup your "$this->destination", your field mapping will then need to provide the id. Otherwise, your source data needs to contain the id of the node that you want to attach the field collection to.

If that doesn't move you forward, post some code and I can try to help sort it out.

alexrayu’s picture

@e-anima Ideally, it should not work this way. Aren't there host entities in your migration source? Like, say, if you are migrating a page with maps, you will migrate the pages into a nodes in the first step, and then, as you are migrating maps locations, you specify the page-to-node migration as a source for your host entity it. And then, what it does, it automatically adds these field collections to the corresponding nodes.

And if you are making changes at import, and adding host entities where there were none in the migration source, then yes - you will need to write a script that creates nodes, and then feed those nodes to the FC import script. You will have to modify the FC script to do that.

You will also have to abstract the logic that will relate nodes with the database records for the Field Collections.

marcoka’s picture

I am migrating words (node type root_word) and the field collection holds words similar to the root word. Like root word is "cat" and then the field collection holds "kitty, pussy ...".

So the above code gets the root words from an external DB and creates the nodes (first step). The second step then should be migrating the Field Collections right? Question is is the second step inside this class or a new one?

class DudenRootWordNodeMigration extends DudenMigration {

  public function __construct() {
    //the group
    parent::__construct(MigrateGroup::getInstance('duden'));

    $this->description = t("Root Word");

    $query = db_select('Wort_Link', 'wlk');
    $query->join('Wort', 'w', 'wlk.wort_id=w.wort_id');
    $query->fields('w',  array('wort'));
    $query->fields('wlk',  array('wort_link_id'))
          ->condition('wlk.root_word', 1);

    $this->source = new MigrateSourceSQL($query);

    //into the fiel collection "field_collection_word"
    //$this->destination = new MigrateDestinationFieldCollection('field_collection_word', array('host_entity_type' => 'root_word'));

    $this->destination = new MigrateDestinationNode('root_wort');

    //To track migration status, and allow rollback, the migrate module needs to remember what
    //source record resulted in the creation of which destination object ..
    $this->map = new MigrateSQLMap($this->machineName,
      array(
        'wort_link_id' => array(
          'type' => 'int',
          'not null' => TRUE,
          'unsigned' => TRUE,
          'description' => 'Wort Link ID',
        )
      ),
      MigrateDestinationNode::getKeySchema()
    );

    //@todo entity reference
    //http://drupal.org/node/1748716

    // MAPPINGS
    $this->addFieldMapping('title', 'wort');
  }
}
mducharme’s picture

It's good that your root word node is migrated as well, this will be the host_entity that your field collections will be attached to. Your "key" field is wort_link_id which you set it your MigrateSQLMap. Your similar words table contains a foreign key to the wort_link_id , right? This is how the field collection items will know what to map to (if not, see Alexei's comment above). Your field collection migration code should look similar to these snippets - and assumes that your query includes the wort_link_id field.

    // Set the destination to the Root Word node
    $this->destination = new MigrateDestinationFieldCollection('root_word',
      array('host_entity_type' => 'node')););

    // Add field mapping for the host entity id to GET the node to attach the field collection to
    $this->addFieldMapping('host_entity_id', 'wort_link_id')
      ->sourceMigration('DudenRootWordNode');

...and YES, do this in a new migration class with this root word class as a dependency!

marcoka’s picture

the "wort" table
http://www.root.artwaves.de/screensnapr/1350732962-RuRLui.jpg

the table where all is linked together "wort_link"
http://www.root.artwaves.de/screensnapr/1350732991-Cu887m.jpg

wort_link_id is the PK and hauptgruppe_id is the field that groups the words with the root word (root_word = 1)
the tables are linked but not with wort_link_id

i used wort_link_id to import where root_node = 1 so i have the host entities. using wort_link_id in the migrate class for the FC doesnt seem to work. i think i am on the wrong path here.

"Could not find host entity of the field collection to import."

mducharme’s picture

Sorry, but this is getting too case-specific for this issue. I recommend looking at your migration queries and try to get results that fit the suggestions that have been provided. Here are the generalized fields you'll want your migration queries to return:

*Word Node Migration*
RootWord_ID <- primary key id
RootWord <- text

*Similar Term FC Migration*
RootWord_ID <- foreign key (host_entity_id)
Word_ID <- primary key
Word <- text

If you can get results like this, you'll be able to move forward. Otherwise, you should open a new support request.

Pierre Paul Lefebvre’s picture

I had a problem where the host_entity was loaded from cache which was not valid anymore. Changed entity_single_load for entity_load(*, *, TRUE) (reset = TRUE, so it loads the entity from the DB). It is obviously longer, but at least it stopping clashing with other modules on entity_presave.

brunodbo’s picture

The patch in #110 doesn't work for me:

- It doesn't add files[] = field_collection.migrate.inc to the module's info file (so the class isn't found when running a migration).
- After adding the .inc file to the module's info file and running a migration, I get the following error for every record that Migrate is attempting to import: "File /home/bruno/workspace/yale/yale_git/includes/entity.inc, line 290
Invalid argument supplied for foreach()", and nothing is imported in the fieldgroup.

The patch in #94 did work, and imported my data in the correct fieldgroup for the right nodes. However, with that patch, I got this error: "New object was not saved, no error provided". Migrate reported that all my records had failed to import (even though they got imported), and had no way to roll back the migration.

I also had to upgrade to the latest dev of Entity API, to avoid a fatal error ("Call to undefined function entity_revision_is_default()").

darrenmothersele’s picture

I'm trying out the patch from #110. The "invalid argument supplied for foreach()" error is coming from lines 84 and 95 in field_collection_migrate.inc where entity_load is used incorrectly. It currently looks like this:

entity_load($this->hostEntityType, array($collection->host_entity_id), TRUE);

but it should look like this:

entity_load($this->hostEntityType, array($collection->host_entity_id), array(), TRUE);

Fixing that now leaves me with another error, and no field collection items.

Missing bundle property on entity of type node. (/home/darren/www/noah/htdocs/includes/common.inc:7633)
darrenmothersele’s picture

Update: that error was caused by the fact the host entity was being passed in as an array (as returned by entity_load), rather than a single entity. Adding reset to the code on line 96 fixes it:

$entity->setHostEntity($this->hostEntityType, reset($host_entity));

Although now I have the field collection items created, but migrate doesn't think they are and fails with this error:

New object was not saved, no error provided
darrenmothersele’s picture

This is because entity_save('field_collection_item', $entity); isn't returning a status.

That's because FieldCollectionItemEntity::save delegates saving the item to the host entity, i.e. in my case node_save(), and as we know node_save() doesn't return the nid.

darrenmothersele’s picture

I've updated the patch from #110 with my changes to get this working. The only bit I don't like is where I have to force success if the host entity type == 'node' . If I don't do this then (as mentioned in comment #114) the field collection items are created, but Migrate doesn't know about them.

Patch attached...

Zahor’s picture

This worked great for a DB to DB migration, but didn't work for a migration from an XML file as no other methods seem to be allowed once sourceMigration is used. Any ideas here? Did I miss something?

marcoka’s picture

what about when using a entityreference field inside a field collection?

busla’s picture

Amaaaazing! Worked great :-) thanks for contributing.

agupta’s picture

Tested patch for D6->D7 migration. Works great!

xcafebabe’s picture

Works great!

But, entity_load returns an array and migration crash when you run a migration with option --update

I just added an array_shift in entity_load call to avoid failures when you update a migration.

marcoka’s picture

another problem needing to be documented is how to do that with multiple field collection values and what if an entity reference field is inside the FC. i did a manual hacky script for all that after trying it the clean way with migrate for too much hours.

alexweber’s picture

can we get this committed please?

alexweber’s picture

Status: Needs review » Needs work

Upon further testing with Migrate 2.5 this does not work. No time for a patch at the moment but we need to declare the destination handler in hook_migrate_api() and even after doing that the class is missing some required static methods that results in fatals in the Migrate UI.

danylevskyi’s picture

This is updated patch of #120. Now migration works if host_entity_type is field_collection_item.

Pls’s picture

Thanks for the effort guys, really appreciate patches. However, I've got stuck trying to migrate multivalue imagefield to multivalue field collection. As I understand ->sourceMigration('Type') is required for host_entity_id fieldMapping, but is there any way to bypass sourceMigration() method and still get multivalue field collection?

My problem is that I don't really need to migrate whole nodes, just migrate data from imagefield to multivalue field collection imagefield. Migration is happening in same drupal installation between different fields, so node migration is really not needed.

Maby I should use prepare() or prepareRow() functions for this particular situation? Thanks.

alexrayu’s picture

I ended up writing a custom module subclassing the migration class to have multified fc's migrations. http://drupal.org/sandbox/alexrayu/1795572

Importing field collections in the complete() hook of the node migration is the choice I would choose now though.

alexweber’s picture

@alexrayu that crossed my mind, it's probably much simpler because even with this patch in progress it requires 2 separate migrations 1 for the host entity and another for the field collection.

alexrayu’s picture

@alexweber I started with 'onion-skin' migrations - like, images - one migration. Images into fc's - another, and then fc's into nodes - yet another. If the site is simple, it goes well. But then if there are any problems in the intermediary level of migration, you will be crying bitterly trying to open that onion skin. So I used my module initially, but then, as changes, and fixes were demanded in my case, I ended up anstracting the field collections functionality, images functionality, etc, into functions in my migration class. I ended up importing everything with the node, and this made me happy and in control. Though of course if the site was less complex, all that would be unnecessary.

mikeryan’s picture

Status: Needs work » Needs review

Well, it's been a while, but I now have another project using collection fields, so I'll be giving the latest patch a fresh review.

mikeryan’s picture

Status: Needs review » Needs work

I can see right off the bat that the latest patch doesn't register the handler in hook_migrate_api(), which is required to for Migrate 2.5 and later - I'll add that.

mikeryan’s picture

Status: Needs work » Needs review

Well, that was silly - this isn't a handler, it's a destination class, so doesn't need the hook_migrate_api() registration.

Thus far, the last patch seems to be working fine for me.

alexweber’s picture

Status: Needs review » Needs work

@mikeryan, it still needs to be declared for Migrate to recognise it, doesn't it?

I have this on my local sandbox and it shows up under Migrate Registration ok (doesn't without it)

function field_collection_migrate_api() {
  $api = array(
    'api' => 2,
    'destination handlers' => array(
      'MigrateDestinationFieldCollection',
    ),
  );

  return $api;
}

mikeryan’s picture

Status: Needs work » Needs review

No, it doesn't - it's not a destination handler (derived from MigrateDestinationHandler), it's a destination plugin (derived from MigrateDestination). It's called directly in migration constructors, so there's no need for it to be registered.

alexweber’s picture

@mikeryan, ah i see, awesome, thanks for clearing that up :)

danylevskyi’s picture

Status: Needs review » Reviewed & tested by the community

Guys, it's time to commit it!

alexweber’s picture

@mikeryan I have a question about the implementation but to keep this thread on topic created a new one: #1887092: Source ID for host_entity_id in migration

If you could please take a quick look I'd be really grateful, thanks! :)

alexweber’s picture

So this patch works and the Migration runs ok but it returns a status message after complete: "New object was not saved, no error provided"

Despite what it says, the new object was in fact saved. However, since it doesn't think it saved it, it's not able to rollback the migration and doing that has no effect.

Any clues?

Thanks!

Edit: Figured it out, in line 112 there's the following if statement:

if (in_array($this->hostEntityType, array('node', 'field_collection_item')) || $status) {

The problem with this is that even though the field collection item entity was saved properly in like 109, entity_save() didn't return anything. This is probably due to how the field_collection_item entity controller works.

Therefore, even though the field_collection_item was actually saved and mapped properly Migrate doesn't think so and doesn't fill in the "destid1" field in the migrate map table; which leads to errors when rolling back and also when migrating field collections that are fields of another field collection (inception ftw).

The fix is simple:

if (in_array($this->hostEntityType, array('node', 'field_collection_item')) || ($status !== FALSE)) {

Since entity_save explicitly returns FALSE on an error, I'm thinking we can safely assume that anything other than false means its OK.

This solution now correctly stores "destid1" in the migratemap table and allows rolling back as usual.

Patch coming in a few minutes against dev.

alexweber’s picture

Status: Reviewed & tested by the community » Needs work

For completeness.

alexweber’s picture

Status: Needs work » Needs review
FileSize
419 bytes

Attached patch does the following different from the previos almost working one (#124):

  1. Fixes $status problem as outlined in #137
  2. Use "item_id" in key schema instead of "cid"; for conciseness with field_collection_item schema
  3. Minor code comment typos fixed
alexweber’s picture

Oops, forgot to git add -N the file, correct patch now attached.

Status: Needs review » Needs work

The last submitted patch, field_collection-migrate-1175082-140.patch, failed testing.

alexweber’s picture

Status: Needs work » Needs review
FileSize
5.46 KB

Getting my butt whooped by git. Ok this one works, promise. :)

See #139 for more details.

mikeryan’s picture

@alexweber - I don't get any "New object was not saved" messages, everything works cleanly, I think there's something else going on with your migration.

FWIW, here's my working collection migration. Note this is a Drupal 7 to Drupal 7 migration, based on the migrate_d2d module.


class WvStatementMigration extends DrupalMigration {
  public function __construct($arguments) {
    parent::__construct($arguments);

    $this->source = new MigrateSourceSQL($this->query(), array(), NULL,
      $this->sourceOptions);
    $this->destination = new MigrateDestinationFieldCollection('field_statements',
      array('host_entity_type' => 'node'));
    $this->map = new MigrateSQLMap($this->machineName,
      array(
        'field_statements_value' => array(
          'type' => 'int',
          'not null' => TRUE,
          'description' => 'Unique ID for each statement',
        )
      ),
      MigrateDestinationFieldCollection::getKeySchema()
    );

    $this->addFieldMapping('host_entity_id', 'entity_id')
         ->sourceMigration('Accomplishment');
    $this->addFieldMapping('field_progress_category', 'field_progress_category_tid')
         ->sourceMigration('Progress');
    $this->addFieldMapping('field_progress_category:source_type')
         ->defaultValue('tid');
    $this->addFieldMapping('field_impact_sector', 'field_impact_sector_tid')
         ->sourceMigration('ImpactSector');
    $this->addFieldMapping('field_impact_sector:source_type')
         ->defaultValue('tid');
    $this->addFieldMapping('field_statement', 'field_statement_value');
    $this->addFieldMapping('field_project', 'projects')
         ->separator(',');

    $this->addUnmigratedDestinations(array(
        'field_impact_sector:create_term', 'field_impact_sector:ignore_case',
        'field_progress_category:create_term', 'field_progress_category:ignore_case',
        'field_statement:language',
    ));
  }

  protected function query() {
    // Base table for the collections - for each entity, order by delta so we
    // create the statements in the right order.
    $query = Database::getConnection('default', $this->sourceConnection)
             ->select('field_data_field_statements', 'fs')
             ->fields('fs', array('entity_id', 'field_statements_value'))
             ->orderBy('fs.entity_id')
             ->orderBy('fs.delta');

    $query->leftJoin('field_data_field_progress_category', 'fpc',
      'fs.field_statements_value = fpc.entity_id');
    $query->addField('fpc', 'field_progress_category_tid');

    $query->leftJoin('field_data_field_impact_sector', 'fis',
      'fs.field_statements_value = fis.entity_id');
    $query->addField('fis', 'field_impact_sector_tid');

    $query->leftJoin('field_data_field_statement', 'fs2',
      'fs.field_statements_value = fs2.entity_id');
    $query->addField('fs2', 'field_statement_value');

    // There can be multiple projects per statement - gather them into a
    // comma-separated list.
    $query->leftJoin('field_data_field_project', 'fp',
      'fs.field_statements_value = fp.entity_id');
    $query->addField('fp', 'field_project_value');
    $query->groupBy('fs.field_statements_value');
    $query->addExpression('GROUP_CONCAT(field_project_value)', 'projects');

    return $query;
  }
}
bdone’s picture

also confirming that #124 applies cleanly and works as expected: #1175082-124: Field collection field handler for migrate module.

alexweber’s picture

@mikeryan,

My use case is a bit more complex: it's a migration from CSV of Profile2 profiles that have attached Field Collections.

The profiles and field collections are each in their own CSV files and after a couple hours of banging my head against the wall the above fix was the only thing that worked for me.

Have you tried running your own migration with my patch? I don't think it will have any negative impact on working migrations and might just help some edge cases.

Will be happy to provide any more info if needed!

Thanks,

setvik’s picture

Hi,

I'm using the patch in comment #120 (http://drupal.org/node/1175082#comment-6833952) and am trying to migrate data into a field collection comprised of a date field and two multi-value file fields. The field collection is attached to a node content type ('state').

The year field is migrating correctly but both file fields are not migrating and remain empty. Code is below and a screenshot of dpm($row) from the end of prepareRow() is attached. Any ideas on what's wrong with the code?

class StateFilesMigration extends Migration {
  public function __construct() {
    parent::__construct();

    // Migrate story nodes before quote field collections attached to them
    $this->dependencies = array('State');
    $this->description = t('Migrate State Files into Field Collections. Also attaches each Field Collection to the appropriate State.');

    $this->map = new MigrateSQLMap($this->machineName,
      array(
        'vid' => array(
          'type' => 'int',
          'not null' => TRUE,
          'unsigned' => TRUE,
          'alias' => 's',
        ),
      ),
      MigrateDestinationFieldCollection::getKeySchema()
    );

    $source_query = Database::getConnection('default', 'chr_legacy')->select('content_type_state_files', 's');
    $source_query->fields('s', array('vid', 'nid'));
    $source_query->addField('s', 'field_year_value', 'year');
    $source_query->orderBy('s.nid', 'ASC');
    $source_query->orderBy('s.field_year_value', 'DESC');

    $this->source = new MigrateSourceSQL($source_query, array(), NULL, array('map_joinable' => FALSE));
    $this->destination = new MigrateDestinationFieldCollection('field_state_files', array('host_entity_type' => 'node'));

    $this->addFieldMapping('host_entity_id', 'nid')
      ->sourceMigration('State');

    $this->addFieldMapping('field_statefiles_year', 'year');
    $this->addFieldMapping('field_statefiles_profile', 'profile')
      ->sourceMigration('Files');
    $this->addFieldMapping('field_statefiles_profile:file_class')
      ->defaultValue('MigrateFileFid');
    $this->addFieldMapping('field_statefiles_downloads', 'downloads')
      ->sourceMigration('Files');
    $this->addFieldMapping('field_statefiles_downloads:file_class')
      ->defaultValue('MigrateFileFid');
  }

  public function prepareRow($row) {
    if (parent::prepareRow($row) === FALSE) {
      return FALSE;
    }

    // Adjust D6 "2012-00-00T00:00:00" format to the D7 date field
    // compatible "2012-01-01 00:00:00" format
    $row->year = substr($row->year, 0, 4) .'-01-01 00:00:00';

    // Get state profile files
    $map = array(
      'profile' => 'field_state_profile_fid',
      'profile:list' => 'field_state_profile_list',
      'profile:data' => 'field_state_profile_data',
    );

    $files = Database::getConnection('default', 'chr_legacy')->query("
      SELECT *
      FROM content_field_state_profile
      WHERE vid = :vid
      ORDER BY delta",
      array(':vid' => $row->vid));
    while ($item = $files->fetchObject()) {
      foreach ($map as $destination => $source) {
        $row->{$destination}[] = $item->{$source};
      }
    }

    // Get state download files
    $map = array(
      'downloads' => 'field_file_downloads_fid',
      'downloads:list' => 'field_file_downloads_list',
      'downloads:data' => 'field_file_downloads_data',
    );

    $files = Database::getConnection('default', 'chr_legacy')->query("
      SELECT *
      FROM content_field_file_downloads 
      WHERE vid = :vid
      ORDER BY delta",
      array(':vid' => $row->vid));
    while ($item = $files->fetchObject()) {
      foreach ($map as $destination => $source) {
        $row->{$destination}[] = $item->{$source};
      }
    }
  }
}
mikeryan’s picture

@alexweber: Yes, I have no doubt your patch is harmless in the general case, and helpful if for some reason entity_save() returns nothing. I would just be concerned about the fact that entity_save() is returning nothing for you, that seems to suggest there's something going wrong there.

@setvik: In the dpm, what do you see if you expand profile and downloads? I would check to make sure that your Files migration has successfully migrated the fids you see there (i.e., migrate_map_files has rows with the profile and downloads fid as their sourceid1, with destid1 being the fid that should get filled in). Also try a dpm() in prepare() to make sure the file fields are properly constructed.

alexweber’s picture

@mikeryan, just a quick follow-up: I've been rolling with my patch for a week now and have implemented 5+ migrations including field collections inside other field collections and have had zero issues thus far, they import and rollback beautifully :) Here are the migrations btw: http://drupalcode.org/project/drop_jobs.git/tree/refs/heads/7.x-1.x:/modules/custom/drop_jobs_demo

setvik’s picture

@mikeryan: Thanks for the pointers. The destination files (destid1 fid) weren't present. They're present when I do the files migration and then do the above field_collection migration, but what I'm finding is that when I roll-back the field_collection migration, not only are the field collections deleted, but the corresponding rows in the file_managed table are deleted as well. Is there something I'm doing in the code (or neglecting) that could be causing that?

mikeryan’s picture

That's a known Migrate issue, see #1676652: Find a better way to preserve files (there's a patch for it).

puddyglum’s picture

#124 works for me as well, I think.

A confusing part was getting the field mapping of the host_entity_id and the entity_id working...

I tried using

<?php
    $this->addFieldMapping('host_entity_id', 'entity_id')
         ->sourceMigration('MyNode');
?>

but that was not working correctly... I ended up simply joining the migrate_map_mynode table and pulling the correct destid1 value.

<?php
              ->fields('map', array('destid1'));
    $query->leftJoin('migrate_map_mynode','map','map.sourceid1 = sourcecollection.mynode_id');
...
$this->addFieldMapping('host_entity_id', 'destid1');
?>
setvik’s picture

@mikeryan: Thanks much!

clemens.tolboom’s picture

Should I use #142 or #124

It's not clear enough to me :(

The interdiff shows one essential difference to me.

alexweber’s picture

#124 is the vanilla solution that apparently works for all out of the box. I've had issues with field collections embedded in other field collections and also attached to non-core entities (profile2) and the patch in #142 is what has been working for me.

This has been going back and forth a bit and ultimately Mike will decide which one to commit :)

mikeryan’s picture

@alexweber - your patch looks fine to me, was just cautioning you that it may obscure some other problem in your instance.

I do not have commit privileges here, and as one of several contributors to this patch it's not appropriate for me to RTBC it - some objective observer who's using it successfully should do that.

alexweber’s picture

@mikeryan, thanks for clarifying! :)

I guess that excludes me to RTBC it as well, any takers?

clemens.tolboom’s picture

Tnx ... I'm testing #142 on XML containing Article nodes with embedded Authors (more then 1 per node) ... hope I dare to RTBC this later.

afox’s picture

We'll start testing this in a scenario where we have the same field_collection field included in multiple content types, allowing multiple items, referencing field collections which have both single & multiple value fields. Hoping for the best...

danylevskyi’s picture

Status: Needs review » Reviewed & tested by the community

#142 works great!

afox’s picture

Status: Reviewed & tested by the community » Needs review

Let's hold off the RTBC a little since we have at least two others testing.

bdone’s picture

i've also tested #142, and it works well.

alexweber’s picture

@afox can we RTBC this now?

afox’s picture

Sorry I had to postpone my testing for tomorrow. I'm just concerned about testing with multi-value fields.

afox’s picture

Agh, my migration case a bit more complicated and needs some work from other perspective before I can properly test field_collections.
If somebody else has positive multi-value testing done, let's go with that.

alexweber’s picture

Status: Needs review » Reviewed & tested by the community

According to #159 and #162 we're good to go, feel free to change this if you find any errors.

mikeryan’s picture

FWIW, my working example in #143 has a multiple value field (field_projects).

aendra’s picture

Yay, RTBC!

To everyone who contributed to this issue (Which I've been following seemingly forever) -- you're all awesome and totally deserve a cookie, a coffee, a beer -- or all three (Simultaneously! Cookie-flavoured Coffee Beer!).

alexrayu’s picture

And you also deserve the patch to be included in migrate or migrate_extras module. Why is it still a standalone patch?

alexweber’s picture

@alexrayu, Thanks! However Migrate only intends to handle core and work on Migrate extras has been put on an indefinite hiatus. The logic on their project page is that contrib modules should actually implement migrations themselves in order to keep things more organised and code contained within the respective projects. Come to think of it, it actually makes more sense than concentrating all migration-specific code in one module.

Therefore, this actually belongs in the Field Collection module :)

alexrayu’s picture

@alexweber I hope it gets accepted then. There has been a lot of work done, and it is also used often for migration as well. Should not fall in a hole in between the two modules.

mikeryan’s picture

It's not in a hole, it's right while it belongs - in the field_collection issue queue. It's taken a long time to get to rtbc, because field_collection is a complex module with a wide variety of use cases (it's not like some simple field that can be supported with a dozen lines of code).

Thanks not only to those who've contributed to it, but to those who have tested it out!

clemens.tolboom’s picture

Status: Reviewed & tested by the community » Needs review

I've tested this on dev. It works as expected.

I had trouble with my XML file. I had to split it into a 'node' file and a 'collection' file. And I forgot to add the XPath to $this->addFieldMapping('host_entity_id', 'url')->xpath('url')

RTBC to me :-)

alexweber’s picture

@clemens.tolboom I'm not sure what you did there... you changed the status from RTBM to Needs Review, suggesting its not ok but then in your comment said it WAS ok.

Which is it? And if its fine, please change the status back!

star-szr’s picture

Assigned: roderik » Unassigned
FileSize
3.72 KB
5.51 KB

Some documentation cleanup (http://drupal.org/node/1354), no code changes. Interdiff is from #142.

puddyglum’s picture

FYI, I used patch #124 to migrate a multi-value field collection field:

class BusinessHoursMigration extends OfficeMigration {
  public function __construct($arguments) {
    parent::__construct($arguments);

    $this->dependencies = array('OfficeNode');

    $this->map = new MigrateSQLMap($this->machineName,
      array(
        'hours_id' => array(
          'type' => 'int',
          'not null' => TRUE,
          'description' => 'Unique ID',
          'alias' => 'hours',
        )
      ),
      MigrateDestinationFieldCollection::getKeySchema()
    );

    $source_fields = array();

    $query = db_select('legacy_office_office_hours', 'hours')
              ->fields('hours', array(
                'day',
                'hours_id',
                'office_id',
                'time_end',
                'time_start',
                'type',
              ))
              ->fields('map', array('destid1'));
    $query->leftJoin('migrate_map_officenode','map','map.sourceid1 = hours.office_id');

    // By default, MigrateSourceSQL derives a count query from the main query -
    // but we can override it if we know a simpler way
    $count_query = db_select('legacy_office_hours', 'hours');
    $count_query->addExpression('COUNT(hours_id)', 'cnt');

    $this->source = new MigrateSourceSQL($query, $source_fields, $count_query);
    $this->destination = new MigrateDestinationFieldCollection('field_office_hours',
      array('host_entity_type' => 'node'));
    
    $this->addFieldMapping('host_entity_id', 'destid1');
    $this->addFieldMapping('field_office_day','day');
    $this->addFieldMapping('field_office_type','type');
    $this->addFieldMapping('field_business_hours','hours');
  }
  
  public function prepareRow($row) {
    // Format hours for Date module
    $date = array(
      'from' => (string) $row->time_start,
      'to' => (string) $row->time_end,
    );
    
    $row->hours = drupal_json_encode($date);
    
    return TRUE;
  }
}
danylevskyi’s picture

Status: Needs review » Reviewed & tested by the community

#174 is ready for commit.

fago’s picture

Status: Reviewed & tested by the community » Needs work

I've not test this (yet), but I'm happy to commit it if it helps and works.

Also, I think we should have a short and fully working example somewhere? Maybe just describe it in the migrate-handbooks and put the link into the code-comments also?

+++ b/field_collection.migrate.inc
@@ -0,0 +1,160 @@
+ * @code
+ * $this->dependencies = array('Article');
+ * ...
+ * $this->destination = new MigrateDestinationFieldCollection('field_attached_data',
+ *   array('host_entity_type' => 'node'));
+ * ...
+ * $this->addFieldMapping('host_entity_id', 'source_article_id')
+ *   ->sourceMigration('Article');
+ */

This misses an @endcode.

+++ b/field_collection.migrate.inc
@@ -0,0 +1,160 @@
+      // The host entity cannot be reset - we only set it on initial insert.

I'd suggest using entity_get_controller($type)->resetCache() + the method or entity_load_single().

danylevskyi’s picture

Status: Needs work » Needs review
FileSize
6.01 KB

I've created basic documentation page describing field collection migrations.
Here is a patch with the fixes mentioned by fago.

clemens.tolboom’s picture

@danylevskyi in #177 asked for a 'fully working example'. The doc ref in #178 is not. Thanks for the effort though :-)

Let's hope this is still RTBC :p

danylevskyi’s picture

@clemens.tolboom, doc page now contains general info about field collection migrations. Later I'll prepare working example and add it.

alexweber’s picture

@danylevskyi hows it going with the documentation? Need any help?

- Alex

danylevskyi’s picture

@alexweber, I think that the best help is to test #178 patch to mark it as RTBC.

alexweber’s picture

Sounds good, I'm going to run the migrations I've mentioned earlier in the queue and if things go by without a hitch it's good for me!

alexweber’s picture

Status: Needs review » Reviewed & tested by the community

Works for me! I just ran about 10 different migrations, some of them with Field Collections embedded within others and with multiple values too and it all went smoothly!

Great work all!

radiobuzzer’s picture

It ran without a problem, for about 4 migrations. Thanks for this work.
Please commit before lines change again ;-)

evanbarter’s picture

+1.

danylevskyi’s picture

Working example of migration is ready, but page needs review.
http://drupal.org/node/1900640

darrenmothersele’s picture

I have one minor issue, in that the host entity is saved for each field collection item that is added. Which, although most the time might not be an issue, in the case where you have lots of things firing in hooks based on updating the host entity, can cause issues. I got around this by using a custom FieldCollectionItem entity controller that I inserted using hook_entity_info.

alexweber’s picture

@darrenmothersele, AFAIK you gotta save the host entity to attach the field collection item "item_id" to it. Curious as to how you worked around it but, either way, I'm happy with calling this an edge case.

@danylevskyi, thanks! I've added some minor fixes.

I think this is good to go... :) I can haz commit?

darrenmothersele’s picture

@alexweber I agree, this is edge case stuff. I was thinking that long term, a solution might be found that somehow collects up all the field collection items and only saves the host entity once, when they have all been done. I'm not sure there's any way of knowing when you're migrated the last item for a host entity?

Consequently, I didn't prevent the host entity saving, I just added a flag that stops all my custom entity hooks from firing during migration.

alexweber’s picture

@darrenmothersele, maybe the correct way to do that would be to use stubs.

Either way, I think your case should be opened as a follow up. This issue and patches have been going back and forth for ages and we finally have something that everybody agrees upon with documentation, etc. I say commit it and then open a follow-up issue... no?

greenjuls’s picture

Thanx @danylevskyi for the last patch, it worked well for me, but I just had an issue when updating some field collection items with embedded field_collections.

In my case the responsible was in method MigrateDestinationFieldCollection::import() that use the whole original entity when updating. This cause all the original entity fields to be sent into functino prepare().

To fix my issue, I had to use the same behavior as the existing MigrateDestinationNode, copy non-existent fields after invoking migration prepare handlers.

I had to clone the $entity object at first and then use the following code:

 $this->prepare($entity, $row);

// Restore fields from original field_collection_item if updating
if ($updating) {
  foreach ($entity as $field => $value) {
    if ('field_' != substr($field, 0, 6)) {
      continue;
    }
    elseif (isset($entity_old->$field) && !isset($collection->$field)) {
      $entity->$field = $entity_old->$field;
    }
  }
}

Status: Reviewed & tested by the community » Needs work

The last submitted patch, field_collection-migrate-1175082-192.patch, failed testing.

greenjuls’s picture

My fault, here is the corrected patch.

greenjuls’s picture

Status: Needs work » Needs review
Ronino’s picture

The patch in #194 works great for me.

It took me some time to get my image data into a multi-value field collection field. I wasn't aware that the main source table with the node data and the image data table don't have the same primary key and so only the first image record per node was imported. After recognizing this, the migration worked great.

guile2912’s picture

#194 worked fine for me too, I have updated the documentation page fixing the $this->dependencies and $this->sourceMigration parameter in the first part of the page.

asherry’s picture

I'm not sure if it's just because I'm running a d6 - d7 migration, but #194 didn't work for me. It maps to an item_id, which is weird because field collections don't really have an item_id they have a host entity_id and a delta.

I changed the patch slightly and this works for me.

Let me know if I'm missing something though.

alexweber’s picture

@asherry field collection items do have an item_id, it's the primary key

asherry’s picture

Right, ok, sorry. There is an item_id in the field_collection_item table, that then gets referenced as [field_name]_value in the actual field table.

My issue was mapping a custom drupal 6 multiple value field to a new field collection item, it shouldn't have been a patch, more like a custom class proposed solution if ever anybody also encounters that scenario.

Thanks.

greenjuls’s picture

Just a little fix from the #194 : I replaced isset() calls by property_exists(), otherwise field values can't get erased when updating a field collection item.

Jumoke’s picture

Title: Field collection field handler for migrate module. » Failure to sort migration list - most likely due to circular dependencies

Can anyone help me with this error I am getting here:

MigrateException: Failure to sort migration list - most likely due to circular dependencies involving Casefile in migrate_migrations() (line 80 of /webroot/case/sites/all/modules/contrib/migrate/migrate.module).

Post here: http://drupal.org/node/1944590#comment-7185072
Note: I am Using field_collection and the patch on #194. [Thank you all patching this].

And how do i sort migration list?

grendzy’s picture

Title: Failure to sort migration list - most likely due to circular dependencies » Field collection field handler for migrate module.

Please don't change the issue title for support requests.

Jumoke’s picture

Sorry, didnt realize i did that. Thot i was adding comment title :)

alanburke’s picture

Slight modification to patch.
This now allows the language to be set when attaching to the host entity.

Status: Needs review » Needs work

The last submitted patch, field_collection-migrate-1175082-205.patch, failed testing.

puddyglum’s picture

@alanburke, the patch seems formatted wrong, and additionally after comparing the code to #194 I couldn't spot any differences.

mccrodp’s picture

What is the status of this, does field_collection now provide support for the migrate module? If not, I'll be working on this next week and willing to lend a hand on anything that needs doing.

Thanks,
Paul.

danylevskyi’s picture

Status: Needs work » Needs review
FileSize
6.5 KB

Hi guys!
Here is the patch based on last working variants #194, #201. Some documentation and code formatting are fixed here.
We are all tired with this issue. We were working so much time on it. Let's finish it now!

Jumoke’s picture

I am having trouble getting this logic to work - migrating into unlimited field_collections on a node.
unlimited fc has fields set as:
field_date (one value), field_document (multi-value)
field_date (one value), field_document (multi-value)

Note: Master node has "content_id", it is mapping to that content_id.

Table of data that should go into fc is below:
file_id, content_id, date, file
1, 1, 6/4/13,file
2, 1, 6/4/13,file

3, 1, 6/5/13,file
4, 1, 6/5/13,file

5, 2, 6/5/13,file
6, 2, 6/6/13,file

From data above, there should be 2 fc created on the content_id 1.

I am able to migrate into multiple field_collection for a node, but it looks like it's creating one for each document, so it must be that my logic is wrong (sql query handling).

Here is code: Any ideas, insights? Thanks in advance for your time.

class CasefilesMigration extends CFMigration {
	public function __construct() {
		parent::__construct();
		$this -> description = t('Migrate Casefiles into Field Collections. Also attaches each Field Collection to the appropriate Case.');
    	$this->dependencies = array('Cases');
		
		$this -> map = new MigrateSQLMap($this -> machineName, 
			array(
				'content_id' => array(
					'type' => 'int', 
					'not null' => TRUE,
					'alias' => 'cf',
					)), 
			MigrateDestinationFieldCollection::getKeySchema()
		);

		$query = db_select('casefiles', 'f')
			  ->fields('f', array('content_id', 'file_date'))
			  ->distinct();

		$this->source = new MigrateSourceSQL($query, array(), NULL, array('map_joinable' => FALSE));	
		$this->destination = new MigrateDestinationFieldCollection(
			'field_case_timeline', array('host_entity_type' => 'node'));

		//Mapped fields
    	$this->addFieldMapping('host_entity_id', 'content_id')
			 ->sourceMigration('Cases');

		$this->addFieldMapping('field_date', 'file_date');	  
    	$this->addFieldMapping('field_document', 'file_upload_filename');
// More mappings for field_document ...
}
	public function prepareRow($current_row) {
		if (parent::prepareRow($row) === FALSE) {
			return FALSE;
		}
		
		$source_id = $current_row->content_id;
		$source_date = $current_row->file_date;
		
		$result = db_select('casefiles', 'f')
			->fields('f', array('file_upload_filename', 'file_upload_filepath', 'file_upload_title'))
			->condition('content_id', $source_id)
			->condition('file_date', $source_date)
			->groupBy('f.file_date')
			->execute();
					  
		foreach ($result as $row) {
			$year_folder = date('Y', strtotime($row->file_date));
			$month_folder = date('m', strtotime($row->file_date));

	      	$current_row->file_upload_filename[] = $row->file_upload_filename;
		    $current_row->file_upload_filepath[] = $row->file_upload_filepath;
		    $current_row->file_upload_title[] = $row->file_upload_title;
			$current_row->file_destination[] = 'public://documents/cases/'.$year_folder.'/'.$month_folder;
			
		}			
	return TRUE;
	}
}
alexrayu’s picture

Hello Jumoke,

Test your query with Devel and see if it's correct. Also, remember, that the files is a multi-value field (if I understood you correctly).

Jumoke’s picture

Thanks Alexei. My mistake was:

        $this -> map = new MigrateSQLMap($this -> machineName,
            array(
                'content_id' => array(

It should have been the unique id of the data, not the content_id that maps back to the node (and usually there are more than 1 of these).

        $this -> map = new MigrateSQLMap($this -> machineName,
            array(
                'casefile_id' => array(
Jumoke’s picture

So i have another question.
Has anyone successfully migrated into a field_collection with a node_reference? Shouldn't it be the same method? I'm spending too much time on this, i need ideas please :)

$this->addFieldMapping('field_press_release_reference', 'pr_entity_id');
danylevskyi’s picture

Jumoke,

$this->addFieldMapping('reference_field', 'reference_id')->sourceMigration('ReferencedMigration');
Jumoke’s picture

Status: Reviewed & tested by the community » Needs review

Thanks Dany,
Turns out there were some other problems with my migration.

puddyglum’s picture

For some reason when I import my field collection values attached to a node, it inadvertently deletes the Geofield value. It's simply doing a MigrateDestinationFieldCollection, so it shouldn't be altering any of the other fields, but it is. Anybody experiencing similar issues?

bleen’s picture

Status: Needs review » Needs work
+++ b/field_collection.migrate.incundefined
@@ -0,0 +1,180 @@
+      $host_entity = entity_load_single($this->hostEntityType, $collection->host_entity_id);

When trying to migrate a field collection whose host is also a field collection item, then entity_load_single returns FALSE and everything craps out:

drush mi mymigration
Missing bundle property on entity of type field_collection_item. (..../docroot/includes/common.inc:7663)Missing bundle property on entity of type field_collection_item. (..../docroot/includes/common.inc:7663)                                                                                                           [error]
bleen’s picture

ignore my previous comment. I had a typo in my sourceMigration value.

That said, I've been doing a pretty complex migration with nested field_collections and (aside from my fat fingers) this has been working fabulously!! RTBC from me

Edit: I had a typo in the word "typo" above .... oh the irony

fuerst’s picture

Status: Needs work » Reviewed & tested by the community

#209 works for me following the example at https://drupal.org/node/1900640. Thanks a lot!

bleen’s picture

Status: Needs review » Reviewed & tested by the community
FileSize
73.64 KB
+++ b/field_collection.migrate.incundefined
@@ -0,0 +1,180 @@
+        throw new MigrateException('Could not find host entity of the field collection to import.');

I dont think this is worth holding up committing this patch, but for what its worth whenever I see this exception thrown I see the message displayed twice

Screenshot_6_26_13_7_43_PM.png

geerlingguy’s picture

Status: Reviewed & tested by the community » Needs work
+++ b/field_collection.migrate.incundefined
@@ -0,0 +1,180 @@
+}

Missing newline at end of file. Other than that, this patch looks good to me—and even this is just a whitespace problem. Looks like this issue's been open a while—can we get this accepted?

fuerst’s picture

Status: Needs work » Reviewed & tested by the community
FileSize
6.04 KB

Added the missing newline using the patch from #209 as a basis.
Let's get this patch committed and no longer hold it back for minor reasons.

danylevskyi’s picture

It works great! RTBC!

weseze’s picture

We are using this patch in development on a several field_collection migrations. (migrating from noderefernces in D6 to field collections in D7)
It is working perfect, but very very slow...

The total number of field_collections migrated is around 25.000 and this takes over 3 hours to complete. I know creating that much entities will take time, but this is really a lot... We are also migrating about 100.000 nodes in the same project with lots of fields and fancy stuff and that migration is finished in 30-40mins.

What can be the cause of this and can we optimize somewhere?

jaydub’s picture

Not sure if it's something you're running into but you may want to check out #1839744: Add index on {$field_collection_field}_revision_id column.

busla’s picture

I´m creating about 100 field collection items in 65-85 seconds, with 6 fields on each item. This is a legacy MySQL db. Most of fields on the fc are taxonomy terms. I´m also running a rule that processes every item.

25.000 items would then take me 5.9 hours. I might be doing something wrong but everything works as it should be, and I´m not using the patch since I´m not migrating files.

I use the Drush --feedback and --limit parameters when I´m testing so the final migration doesn´t hurt as much :-)

rooby’s picture

busla:
I saw ridiculous times like that before once and the reason was that my databases were across the network, not on my local machine.

Moving the databases to my local machine, which removes the need for network traffic, made things way way faster.

That and making sure to use drush, not the migrate UI, for migrations saves a lot of time.

busla’s picture

rooby: I do all migrations locally but I´m creating several joins in some cases. That might be the cause. Also, the rules conditions might take their toll. I never use the MIgrate UI. I want all the feedback possible from the console... aswell as using --debug.

rooby’s picture

I'm just doing a migration that has the entities (from a csv)

user
user > profile2_1
user > profile2_1 > field_collection_1
user > profile2_1 > field_collection_2
user > profile2_2

And the field collections don't really seem to be going any slower than the rest.
Maybe marginally but nothing really noticeable.

All the migrations have some custom code in prepareRow() but it is not doing any heavy lifting.

Seems it is possibly related to your customisations or maybe you have much larger and more complex field collections than I (mine are pretty small). Or maybe there is a difference because I am coming from CSV.

bleen’s picture

My fairly complex field collection migration does go noticeably slower but not to the degree that busla is describing. That said, this should not prevent the patch in #222 from being committed. Better to have working migrations that might be slower than no migrations. :)

rooby’s picture

I agree.

When I ran my migration on the production server, which is much faster & better than my development one, I was easily getting well over 1000 items per minute (I can't remember the exact figure). My non-field_collection ones were still faster though.

I had a (very - & it's late, I could be talking complete rubbish :)) quick look at the code in the patch now and there is nothing outrageous in there.
It's possible some of it is on the field collection end of things.
Also, it has to deal with the host entity too, which other migrations don't, but there is likely nothing much that can be done about that.

It could end up just being one of those things that is a known issue.

nlisgo’s picture

Migrating nicely with patch #222 :)

Thanks for the good work everyone. I used to avoid field collection because it just seemed like it was going to be painful to migrate content to it. I favoured the relation module. I will still use the relation module for other purposes but it's great to be using this and have migrate working!!

fuerst’s picture

Just a note: While working on a migration I got a PHP error in MigrateDestinationFieldCollection::import(): __clone method called on non-object in field_collection/field_collection.migrate.inc, line 96

Problem here was: the Field Collection in question was imported earlier so there was a Destination ID ($row->migrate_map_destid1) available. Later the Field Collection was removed from the node. When migrating again the assumption in MigrateDestinationFieldCollection::import() still was to update the existing because of $row->migrate_map_destid1 although the Field Collection did not exist anymore.

I propose to just create a new Field Collection in this case. Any thoughts?

albarin’s picture

Category: feature » support
FileSize
1.2 KB
1.19 KB

Hi, I'm not able to update existing field collection.
I have added this lines to the update migration class:
$this->systemOfRecord = Migration::DESTINATION;
$this->addFieldMapping('host_entity_id', 'id')->sourceMigration('Extras');

I always get this error message: Could not find host entity of the field collection to import.

I can't find any example.
I attach the initial migration and the update migration classes.
I would appreciate any help.
Thanks!!

rooby’s picture

Category: support » feature
pbuyle’s picture

The patch in #222 provides a working solution. The instructions are given in the comment of the the field_collection.migrate.inc file. The host entities have to be migrated first (in their own migration class), then the field collections are migrated in a second migration class.

markdorison’s picture

I am attempting to use the patch in #222 to migrate into a multi-value field collection. I create an array of the objects in prepareRow() but when the migration completes and I review the nodes, only the first item in the array has populated the field collection and no subsequent items have been added. Am I missing something important?

<?php
class IngredientsMigration extends BasicMigration {
  public function __construct($arguments) {
    parent::__construct($arguments);
    $this->description = t('Migrate recipe ingredients from the source database into field collections.');
    $this->dependencies = array('Recipe');

    $query = Database::getConnection('default', 'legacy')
      ->select('recipes', 'r')
      ->fields('r', array('id', 'ingredients'))
	    ->condition('r.id', array(2), 'IN');

    $this->source = new MigrateSourceSQL($query);
    $this->destination = new MigrateDestinationFieldCollection(
      'field_collection_ingredients',
      array('host_entity_type' => 'node')
    );

    $this->map = new MigrateSQLMap($this->machineName,
      array(
        'id' => array('type' => 'int',
          'length' => 11,
          'not null' => TRUE,
          'description' => 'Recipe ID',
          'alias' => 'r',
          ),
      ),
      MigrateDestinationFieldCollection::getKeySchema()
    );

    $this->addFieldMapping('host_entity_id', 'id')
      ->sourceMigration('Recipe')
      ->issueNumber(16);
    $this->addFieldMapping('field_ingredient_text', 'ingredients')
      ->description(t('See prepareRow method'))
      ->issueNumber(16);

    $this->addUnmigratedDestinations(array(
      'path',
      )
    );
  }

  public function prepareRow($row) {
    // Always include this fragment at the beginning of every prepareRow()
    // implementation, so parent classes can ignore rows.
    if (parent::prepareRow($row) === FALSE) {
      return FALSE;
    }

    $ingredients_array = preg_split('/$\R?^/m', $row->ingredients);
    $row->ingredients = array();
    foreach ($ingredients_array as $ingredient) {
      $row->ingredients[] = $ingredient;
    }
    return TRUE;
  }
}
markdorison’s picture

Status: Reviewed & tested by the community » Needs work

I have been unable to get this to work with multiple value fields in field collections. I am going to mark this as needs work unless someone can point out what I am missing. I am continuing to look at this.

aidanlis’s picture

Here's an example migration from a D6 flexifield to a field_collection in the hope that it helps someone.

class BulletinMigration extends Migration {
  public function __construct(array $arguments) {
    parent::__construct($arguments);

    $this->description = t('Migrate bulletin field collection.');

    $this->dependencies = array('ExamplesiteNodeupdate', 'ExamplesiteTerm7');

    $query = Database::getConnection('default', 'legacy')
           ->select('node', 'n');
    $query->join('content_field_bulletins', 'f', 'n.nid = f.nid');
    $query->fields('f', array('nid', 'delta', 'field_bulletins_value'))
          ->condition('status', 1)
          ->condition('type', 'update')
          ->orderBy('nid')
          ->orderBy('delta');

    // Create a MigrateSource object, which manages retrieving the input data.
    $extrafields = array('body' => 'Computed field', 'body_format' => 'Computed field', 'category' => 'Computed field', 'title' => 'Computed field');
    $this->source = new MigrateSourceSQL($query, $extrafields, NULL, array('map_joinable' => FALSE));

    $this->map = new MigrateSQLMap($this->machineName,
      array(
        'nid' => array(
          'type' => 'int',
          'not null' => true,
        ),
        'delta' => array(
          'type' => 'int',
          'not null' => true,
        ),
      ),
      MigrateDestinationFieldCollection::getKeySchema()
    );

    $this->destination = new MigrateDestinationFieldCollection(
        'field_bulletins',
        array('host_entity_type' => 'node')
    );

    $this->addFieldMapping('host_entity_id', 'nid')
        ->sourceMigration('ExamplesiteNodeupdate');

    $this->addFieldMapping('field_bulletin_title', 'title');
    $this->addFieldMapping('field_bulletin_body', 'body');
    $this->addFieldMapping('field_bulletin_body:format', NULL)->defaultValue('filtered_html');

    $this->addFieldMapping('field_bulletin_category', 'category')
        ->sourceMigration('ExamplesiteTerm7')
        ->arguments(array('source_type' => 'tid'));
  }

  function prepareRow($row) {
    echo $row->nid,"\n";
    // Always include this fragment at the beginning of every prepareRow()
    // implementation, so parent classes can ignore rows.
    if (parent::prepareRow($row) === FALSE) {
      return FALSE;
    }

    $data = unserialize($row->field_bulletins_value);
    $row->body = $data['field_body'][0]['value'];
    $row->body_format = $data['field_body'][0]['format'];
    $row->category = $data['field_category'][0]['value'];
    $row->title = $data['field_title'][0]['value'];
  }
}

You can see I accessed the serialised field data rather than creating a query which used the content_type_flexifield table, this seemed easier.

fuerst’s picture

Status: Needs work » Reviewed & tested by the community

@markdorison: The rule is: one field collection item per database row. Have a look at the documentation for MigrateDestinationFieldCollection::import(). That means you have to modify your source query to return one record for every ingredient. May be you have to normalize your data first. Alternatively you can create the additional Field Collections per recipe in the complete() method.

markdorison’s picture

@fuerst: This did the trick. Thank you for pointing me in the right direction!

fago’s picture

Status: Reviewed & tested by the community » Fixed

Thanks everyone for the hard work on this + documentation - this looks good now -> Committed.

aidanlis’s picture

Amazing! Great to get this committed!

Automatically closed -- issue fixed for 2 weeks with no activity.

wonder95’s picture

Has anyone implemented this handler with migrate_d2d? I'm trying to do that with an image field, but all the examples I'm finding around the interwebs are for migrate, not migrate_d2d.

harpss’s picture

@markdorison: What was your solution? I'm at the point you were before you said this did the trick. What was your final code?

islalobo’s picture

@markdorison: I'm also curious about how your code looked in the end.

daniroyo’s picture

¿Is it possible to migrate multilanguage fields with this handler?

I have tried to migrate some field_collections with this approach and I couldn´t.

I paste some of my code to view if I am doing something wrong

class TabsBarriosMigration extends ContentBaseMigration {
  public function __construct() {
    parent::__construct();
    $this->dependencies = array('Barrios');

    $this->fields = array(
      'title' => 'Titulo de la pestaña',
      'content' => 'Descripción de la pestaña',
    );

    $this->source = new MigrateSourceSQL($this->query(), $this->fields);
    $this->destination = new MigrateDestinationFieldCollection('field_district_tabs', array('host_entity_type' => 'node'));

    $this->map = new MigrateSQLMap(
      $this->machineName,
      array( 'id' => array( 'type' => 'int', 'length' => 11, 'not null' => TRUE, 'alias' => 'cst', )),
      MigrateDestinationFieldCollection::getKeySchema()
    );
    $this->addFieldMapping('language', 'localization_id')->callbacks(array($this, 'mapLanguagesIds'));

    $this->addFieldMapping('host_entity_id', 'content_section_id')->sourceMigration('Barrios');
    $this->addFieldMapping('field_tab_title', 'title');
    $this->addFieldMapping('field_tab_title:language', 'localization_id')->callbacks(array($this, 'mapLanguagesIds'));
    $this->addFieldMapping('field_tab_description', 'content');
    $this->addFieldMapping('field_tab_description:format')->defaultValue('full_html');
    $this->addFieldMapping('field_tab_description:language', 'localization_id')->callbacks(array($this, 'mapLanguagesIds'));
  }
}

The migration is working almost fine, field collectios are created, but with "und" language.

I have to say that languages provided matched perfectly, so I am not sure what is wrong

daniroyo’s picture

Is it known something about migrate languages with this destination?

Thank you!

gopinath1586’s picture

Any one tell me the final working migration code multi value field collection entity? @markdorison and @fuerst

osopolar’s picture

If someone experience the loss of field collection assignment on subsequent update migrations of the host entity, I added a comment with an example to the Migrate Field Collections handbook article.

herved’s picture

Thank you all and thank you @mikeryan your example with migrate_d2d (#143) helped me a lot.

isalmanhaider’s picture

Which one is the final working patch that works? Anyone, please
Thanks

ccarnnia’s picture

this is a old tread, i am posting because i see a lot heavy hitters here and am desperately trying to amplify my problem: https://www.drupal.org/project/field_collection/issues/3057027#comment-13120718

ccarnnia’s picture

UPDATE:
Thank you @osopolar @isalmanhaider and @Jumoke for desk checking this WITH me.
this worked:

<?php
class MemberVotesMigration extends ZaqMigration {
  public function __construct($arguments) {
    parent::__construct($arguments);

    $this->description = t('casey will migrate this field cllection even if it kills him!!!!!!'); 
    // class of prviously imported node, this class name was generated by d2d_migrate module
    // that is why the name is diffrent
    $this->dependencies = array('7f4f1882cNodeorg_vote'); 

    $query = $this->getConnection()
    // start with the field that embeds the field_collection
    ->select('field_data_field_org_member_votes', 'nodeField')
    ->fields('nodeField', array('entity_id', 'field_org_member_votes_value', 'delta'))
    ->groupBy('nodeField.field_org_member_votes_value')
    ->orderBy('nodeField.entity_id')
    ->orderBy('nodeField.delta');
 
    // join it to the fields that make up the field_collection, in my case there were 3 fields
    $query->leftJoin('field_data_field_org_votes_member', 'member',
    'nodeField.field_org_member_votes_value = member.entity_id');
    $query->addField('member', 'field_org_votes_member_nid');

    $query->leftJoin('field_data_field_org_votes_vote', 'vote',
    'nodeField.field_org_member_votes_value = vote.entity_id');
    $query->addField('vote', 'field_org_votes_vote_value');

    $query->leftJoin('field_data_field_org_votes_majority', 'majority',
    'nodeField.field_org_member_votes_value = majority.entity_id');
    $query->addField('majority', 'field_org_votes_majority_value');


    $this->source = new MigrateSourceSQL($query);
 
    $this->destination = new MigrateDestinationFieldCollection(
        'field_org_member_votes',
        array('host_entity_type' => 'node')
    );
 
    $this->map = new MigrateSQLMap($this->machineName,
        array(
            'field_org_member_votes_value' => array( 
                'type' => 'int',
                'not null' => TRUE,
                'description' => 'Unique ID for each commitee vote',
                'alias' => 'nodeField',
        )
      ),
      MigrateDestinationFieldCollection::getKeySchema()
    ); //map END

    $this->addFieldMapping('host_entity_id', 'entity_id') ->sourceMigration('7f4f1882cNodeorg_committee_vote');
 
    $this->addFieldMapping('field_org_votes_member', 'field_org_votes_member') ->sourceMigration('7f4f1882cNodemember');
    $this->addFieldMapping('field_org_votes_member:source_type') ->defaultValue('nid');

    $this->addFieldMapping('field_org_votes_vote', 'field_org_votes_vote');

    $this->addFieldMapping('field_org_votes_majority', 'field_org_votes_majority'); 

  }//construct END
 
  function prepareRow($row) {
   // this is a good wat to test and make sure 
   // A) you have connected to the db 
   // B) you are getting proper values from your query
   // drush_print_r($row);
  }//prepareRow END
  function complete($row) {
   // drush_print_r($row);
  }
  public function prependTextNotes($row) {
    return 'Imported notes: ' . $row;
  }//prependTextNotes   END

}