apachesolr.index.inc | 112 +++++++++++++++++++++++++-------------------------- 1 file changed, 56 insertions(+), 56 deletions(-) diff --git a/apachesolr.index.inc b/apachesolr.index.inc index b7479cf..62f26e6 100644 --- a/apachesolr.index.inc +++ b/apachesolr.index.inc @@ -46,21 +46,21 @@ function apachesolr_index_entities($env_id, $limit) { $indexed = apachesolr_index_send_to_solr($env_id, $documents); if ($indexed !== FALSE) { - $documents_submitted += count($documents); - $index_position = apachesolr_get_last_index_position($env_id, $entity_type); - $max_changed = $index_position['last_changed']; - $max_entity_id = $index_position['last_entity_id']; - foreach ($rows as $row) { - if (!empty($row->status)) { - if ($row->changed > $max_changed) { - $max_changed = $row->changed; - } - if ($row->entity_id > $max_entity_id) { - $max_entity_id = $row->entity_id; - } + if (count($documents)) { + $documents_submitted += count($documents); + $last_row = end($rows); + if (!empty($last_row->changed) && !empty($last_row->entity_id)) { + apachesolr_set_last_index_position($env_id, $entity_type, $last_row->changed, $last_row->entity_id); + } + else { + watchdog( + 'Apache Solr', + 'Failure recording indexing progress. Last entitiy id processed: %entity_id with timestamp %last_changed', + array('%entity_id' => $last_row->entity_id, '%last_changed' => $last_row->changed), + WATCHDOG_ERROR + ); } } - apachesolr_set_last_index_position($env_id, $entity_type, $max_changed, $max_entity_id); apachesolr_set_last_index_updated($env_id, REQUEST_TIME); } } @@ -96,6 +96,7 @@ function apachesolr_index_entities_document($row, $entity_type, $env_id) { } return $documents; } + /** * Returns the total number of documents that are able to be indexed and the * number of documents left to be indexed. @@ -126,26 +127,7 @@ function apachesolr_index_status($env_id) { $query = db_select($table, 'asn')->condition('asn.status', 1)->condition('asn.bundle', $bundles); $total += $query->countQuery()->execute()->fetchField(); - // Get $last_entity_id and $last_change. - extract(apachesolr_get_last_index_position($env_id, $entity_type)); - // Find the next batch of entities to index for this entity type. Note that - // for ordering we're grabbing the oldest first and then ordering by ID so - // that we get a definitive order. - $query = db_select($table, 'aie') - ->condition('aie.bundle', $bundles) - ->condition('aie.status', 1) - ->condition(db_or() - ->condition('aie.changed', $last_changed, '>') - ->condition(db_and() - ->condition('aie.changed', $last_changed, '<=') - ->condition('aie.entity_id', $last_entity_id, '>'))) - ->addTag('apachesolr_index_' . $entity_type); - - - if ($table == 'apachesolr_index_entities') { - // Other, entity-specific tables don't need this condition. - $query->condition('aie.entity_type', $entity_type); - } + $query = apachesolr_index_get_entities_to_index_get_query($env_id, $entity_type); $remaining += $query->countQuery()->execute()->fetchField(); } return array('remaining' => $remaining, 'total' => $total); @@ -440,29 +422,7 @@ function apachesolr_index_get_entities_to_index($env_id, $entity_type, $limit) { return $rows; } - $table = apachesolr_get_indexer_table($entity_type); - // Get $last_entity_id and $last_change. - extract(apachesolr_get_last_index_position($env_id, $entity_type)); - // Find the next batch of entities to index for this entity type. Note that - // for ordering we're grabbing the oldest first and then ordering by ID so - // that we get a definitive order. - // Also note that we fetch ALL fields from the indexer table - $query = db_select($table, 'aie') - ->fields('aie') - ->condition('aie.bundle', $bundles) - ->condition(db_or() - ->condition('aie.changed', $last_changed, '>') - ->condition(db_and() - ->condition('aie.changed', $last_changed, '<=') - ->condition('aie.entity_id', $last_entity_id, '>'))) - ->orderBy('aie.changed', 'ASC') - ->orderBy('aie.entity_id', 'ASC') - ->addTag('apachesolr_index_' . $entity_type); - - if ($table == 'apachesolr_index_entities') { - // Other, entity-specific tables don't need this condition. - $query->condition('aie.entity_type', $entity_type); - } + $query = apachesolr_index_get_entities_to_index_get_query($env_id, $entity_type); $query->range(0, $limit); $records = $query->execute(); @@ -484,6 +444,46 @@ function apachesolr_index_get_entities_to_index($env_id, $entity_type, $limit) { } /** + * Internal function to avoid duplicating logic that identifies entities + * still due to be indexed + * + * @param string $env_id Environment ID + * @param string $entity_type + * @return SelectQuery + */ +function apachesolr_index_get_entities_to_index_get_query($env_id, $entity_type) { + $table = apachesolr_get_indexer_table($entity_type); + $last_index_position = apachesolr_get_last_index_position($env_id, $entity_type); + $bundles = apachesolr_get_index_bundles($env_id, $entity_type); + + // Find the next batch of entities to index for this entity type. + // Also note that we fetch ALL fields from the indexer table + $query = db_select($table, 'aie') + ->fields('aie') + ->condition('aie.bundle', $bundles) + ->condition(db_or() + // Entities that have been changed more recently than the most up to date indexed entity + ->condition('aie.changed', $last_index_position['last_changed'], '>') + // Tie breaker for entities that were changed at exactly the same second as the last indexed entity + ->condition(db_and() + ->condition('aie.changed', $last_index_position['last_changed'], '=') + ->condition('aie.entity_id', $last_index_position['last_entity_id'], '>') + ) + ) + // It is important that everything is indexed in order of changed date + // then entity id, or the conditions above will not match correctly + ->orderBy('aie.changed', 'ASC') + ->orderBy('aie.entity_id', 'ASC') + ->addTag('apachesolr_index_' . $entity_type); + + if ($table == 'apachesolr_index_entities') { + // Other, entity-specific tables don't need this condition. + $query->condition('aie.entity_type', $entity_type); + } + return $query; +} + +/** * Delete the whole index for an environment. * * @param string $env_id