Index: apachesolr_stats.module =================================================================== RCS file: /cvs/drupal-contrib/contributions/modules/apachesolr_stats/apachesolr_stats.module,v retrieving revision 1.1.2.3 diff -u -r1.1.2.3 apachesolr_stats.module --- apachesolr_stats.module 9 Sep 2009 19:19:44 -0000 1.1.2.3 +++ apachesolr_stats.module 9 Sep 2009 21:17:33 -0000 @@ -1,13 +1,10 @@ l('module configuration page', 'admin/settings/apachesolr/stats'))); } + // Get the facets array to report on + $facets = apachesolr_stats_get_facets(); + $granularities = array( 'minute' => array( 'name' => t('minute'), 'timespan' => 60, 'time_before' => 60*60*24, // One day before + 'format' => '%H:%M', ), 'hour' => array( 'name' => t('hour'), 'timespan' => 60*60, 'time_before' => 60*60*24*7, // One week before + 'format' => '%m/%d %H:%M', ), 'day' => array( 'name' => t('day'), 'timespan' => 60*60*24, 'time_before' => 60*60*24*7*3, // 3 weeks before + 'format' => '%m/%d', ), ); // Decide what granularity to use: minute, hour or day @@ -168,6 +171,9 @@ $result = db_query("SELECT * FROM {apachesolr_stats} WHERE timestamp > %d ORDER BY timestamp DESC", time()- $granularity['time_before']); $start_timeslot = 0; $last_timeslot = 0; + $time['max'] = -1; + $time['min'] = 9999.999; + // Scan the logfile and build statistics arrays while ($record = db_fetch_object($result)) { $timeslot = intval($record->timestamp / $granularity['timespan']); @@ -179,22 +185,45 @@ if ($record->suggestions) { $suggestions++; } - $total_queries++; - $time['total'] += $record->total_time; - $time['prepare'] += $record->prepare_time; - $time['process'] += $record->process_time; + $total_requests++; + $time['total'] += $record->total_time; + #$time['prepare'] += $record->prepare_time; + #$time['process'] += $record->process_time; + + // Track max and min response times + $time['max'] = ($time['max'] < $record->total_time ? $record->total_time : $time['max']); + $time['min'] = ($time['min'] > $record->total_time ? $record->total_time : $time['min']); // Field usage; only when on first results page (meaning it's a fresh search) if ($record->page == "") { - if (trim($record->keywords) != "") { - $field_usage['keyword']++; - } - foreach (explode(' ', $record->filters) as $filter) { - list($fieldname, $value) = explode(':', $filter); - if ($fieldname) { - $field_usage[$fieldname]++; + unset($facet_processed_flag); + $ok = preg_match_all('/[^ ]+/', $record->filters, $matches); + foreach($matches[0] as $word) { + $fieldname = apachesolr_stats_determine_field_from_query($facets, $word); + if ($fieldname != false) { + // Count this facet field only once per query + if ($facet_processed_flag[$fieldname] != true) { + // Add 1 to usage of term from vocabulary + $facets[$fieldname]['usage']++; + // Mark so we don't count it again for this query + $facet_processed_flag[$fieldname] = true; + } + } + + if (trim($record->keywords) != "") { + if ($facet_processed_flag['kw'] != true) { + $facets['kw']['usage']++; + $facet_processed_flag['kw'] = true; + } } } + // Count each unique query + $facets["any"]['usage']++; + + // Keep track of how many fields were active per query + $simultaneous_fields[sizeof($facet_processed_flag)]++; + + $total_queries++; } // Sort usage; count only the first page of results @@ -220,16 +249,16 @@ $s[t('Queries')] = $total_queries; $s[t('Spellchecker suggestions')] = $suggestions; - $s[t('Sessions')] = sizeof($sessions); + $s[t('Unique sessions')] = sizeof($sessions); $s[t('Unique users')] = sizeof($users); $s[t('Average queries per user')] = sprintf("%.1f", $total_queries / sizeof($users)); $s[t('Average queries per session')] = sprintf("%.1f", $total_queries / sizeof($sessions)); - $s[t('Average time per query')] = sprintf("%.3f sec", $time['total'] / $total_queries / 1000); - $s[t('Average time per query: prepare')] = sprintf("%.3f sec", $time['prepare'] / $total_queries / 1000); - $s[t('Average time per query: process')] = sprintf("%.3f sec", $time['process'] / $total_queries / 1000); + $s[t('Solr service response time per query')] = sprintf("%s: %.3f sec / %s: %.3f sec / %s: %.3f sec", t('Average'), $time['total'] / $total_queries / 1000, t('Minimum'), $time['max'] / 1000, t('Maximum'), $time['min'] / 1000); + #$s[t('Average time per query: prepare')] = sprintf("%.3f sec", $time['prepare'] / $total_queries / 1000); + #$s[t('Average time per query: process')] = sprintf("%.3f sec", $time['process'] / $total_queries / 1000); // Chart for queries per timeslot - $chart = apachesolr_stats_chart($data_per_granularity['queries'], $start_timeslot, $last_timeslot, $total_queries, $total_queries / ($last_timeslot-$start_timeslot+1)); + $chart = apachesolr_stats_chart($granularity, $data_per_granularity['queries'], $start_timeslot, $last_timeslot, $total_queries, $total_queries / ($last_timeslot-$start_timeslot+1)); $s[t('Total queries, by @granularity', array('@granularity' => $granularity['name']))] = $chart; // Chart for average time per timeslot @@ -238,22 +267,15 @@ $data[$timeslot] = $value / $count_per_granularity[$timeslot]; } // Call with average_empty = FALSE - $chart = apachesolr_stats_chart($data, $start_timeslot, $last_timeslot, $total_queries, $time['total'] / $total_queries); + $chart = apachesolr_stats_chart($granularity, $data, $start_timeslot, $last_timeslot, $total_queries, $time['total'] / $total_queries); $s[t('Average time per query, by @granularity (miliseconds)', array('@granularity' => $granularity['name']))] = $chart; // Chart for users per timeslot - $chart = apachesolr_stats_chart($data_per_granularity['users_per_slot'], $start_timeslot, $last_timeslot, $total_queries, sizeof($users) / ($last_timeslot-$start_timeslot+1)); + $chart = apachesolr_stats_chart($granularity, $data_per_granularity['users_per_slot'], $start_timeslot, $last_timeslot, $total_queries, sizeof($users) / ($last_timeslot-$start_timeslot+1)); $s[t('Total unique users executing queries, by @granularity', array('@granularity' => $granularity['name']))] = $chart; // Chart for field usage - foreach ($field_usage as $fieldname => $count) { - $chd = ""; - } - $chl = implode('|', array_keys($field_usage)); - $chd = implode(',', $field_usage); - $height = 30 + sizeof($field_usage) * 35; - $chart = ""; - $s[t('Field usage', array('@granularity' => $granularity['name']))] = $chart; + $s[t('Field usage', array('@granularity' => $granularity['name']))] = apachesolr_stats_facet_usage_graph($facets); // Chart for sort usage foreach ($sort_usage as $fieldname => $count) { @@ -289,6 +311,10 @@ ); } $output .= theme('table', array(), $rows); + + $output .= "

Facet usage detail

"; + $output .= apachesolr_stats_facet_usage_table($facets); + return $output; } @@ -306,7 +332,7 @@ * @param $average * Boolean flag: show an average value in the chart. */ -function apachesolr_stats_chart($data, $start_timeslot, $last_timeslot, $total_queries, $average = FALSE) { +function apachesolr_stats_chart($granularity, $data, $start_timeslot, $last_timeslot, $total_queries, $average = FALSE) { // Sample: http://chart.apis.google.com/chart?cht=lc&chs=350x100&chdlp=b&chma=10,10,10,10&chd=s:[encoded chart data] #drupal_set_message("start = $start_timeslot, last = $last_timeslot"); @@ -335,20 +361,37 @@ $image_url = $chart_prefix . apachesolr_stats_encodedata($chd, $chd_min, $chd_max); // Add labels + $chxl = ""; if ($chd_max > 0) { - $image_url .= "&chxl=0:|". intval($chd_min) ."|". intval($chd_max); + $chxl .= "0:|". intval($chd_min) ."|". intval($chd_max); // Show average value in a label - if ($average !== FALSE) { - $image_url .= sprintf("|1:|%s=%.2f", t('average'), $average); + if ($average !== FALSE) { $image_url .= "&chxp=1," . intval($average / $chd_max * 100); + $chxl .= sprintf("|1:|%s=%.2f", t('average'), $average); $chxt = "y,r"; } else { $chxt = "y"; } - $image_url .= "&chxt=$chxt"; + + // Add time/date labels + $earliest_timestamp = $start_timeslot * $granularity['timespan']; + $last_timestamp = $last_timeslot * $granularity['timespan']; + $mid_timestamp = ($last_timestamp + $earliest_timestamp) / 2; + $time_msg_1 = strftime($granularity['format'], $earliest_timestamp); + $time_msg_2 = strftime($granularity['format'], $mid_timestamp); + $time_msg_3 = strftime($granularity['format'], $last_timestamp); + if ($chxt) { + $chxt = "{$chxt},x"; + $chxl .= "|2:|{$time_msg_1}|{$time_msg_2}|{$time_msg_3}"; + } else { + $chxt = "x"; + $chxl .= "|1:|{$time_msg_1}|{$time_msg_2}|{$time_msg_3}"; + } + + $image_url .= "&chxl={$chxl}&chxt={$chxt}"; } - + // Return the image tag return ""; } @@ -387,6 +430,208 @@ */ /* TODO: change results here to track clickthrus? function apachesolr_stats_apachesolr_search_result_alter($doc) { - dprint_r($doc); + dsm($doc); +} +*/ + +/** + * Determine the facet field from a word: im_vid_XX, kw or XXX (anything before ":", like tid, uid, etc) + */ +function apachesolr_stats_determine_field_from_query($facets, $word) { + if (strpos($word, ":") !== false) { + list($fieldname, $value) = explode(":", $word); + if ($fieldname == "tid") { + // Replace tid with the correct facet field name (im_vid_XXX) where XXX = vid + $term = taxonomy_get_term(intval(substr($word, 4))); + $fieldname = "im_vid_" . $term->vid; + } else { + // Check if $fieldname is really a facet, if not count as keyword + if (empty($facets[$fieldname])) { + $fieldname = false; + } + } + } else { + // No ":" found, so it's a keyword + $fieldname = false; + } + return $fieldname; +} + +/** + * Returns a themed table for facet usage + */ +function apachesolr_stats_facet_usage_table($facets) { + // Report usage in table + $header = array( + array('data' => 'Facet ID', 'Xfield' => 'id', 'sort' => 'asc'), + array('data' => 'Facet info', 'Xfield' => 'info', 'sort' => ''), + array('data' => 'Queries containing this facet', 'Xfield' => 'queries', 'sort' => ''), + #array('data' => 'Clickthrus from queries containing this facet', 'Xfield' => 'ct', 'sort' => ''), + #array('data' => 'Clickthru rate for facet', 'Xfield' => 'ctp', 'sort' => ''), + ); + foreach($facets as $fieldname => $facet) { + $rows[$fieldname][] = $fieldname; + $rows[$fieldname][] = $facet['info']; + $rows[$fieldname][] = $facet['usage']; + #$rows[$fieldname][] = $facet['clickthru']; + #if ($facet['usage']>0) { + # $rows[$fieldname][] = sprintf("%2.1f%%", ($facet['clickthru'] / $facet['usage'])*100); + #} + } + $output = theme('table', $header, $rows); + return $output; +} + +/** + * Returns the tag for a Google chart with the facet usage + */ +function apachesolr_stats_facet_usage_graph($facets) { + // Chart for field usage + $leyend = array(); + $data = array(); + $label_cutoff = 40; + foreach($facets as $fieldname => $facet) { + #if ($facet['usage']) { + $tmp = preg_replace("/^.*ilter by /", "", $facet['info']); + if (strlen($tmp) > $label_cutoff) { + $tmp = substr($tmp,0,$label_cutoff) . "..."; + } + $leyend[] = $tmp; + $data[] = $facet['usage']; + #} + } + $chd = apachesolr_stats_encodedata($data, 0, $facets['any']['usage']); + // array_reverse() in next line due to apachesolr_stats_encodedata() encoding data backwards + $chl = implode('|', array_reverse($leyend)); + $height = 30 + sizeof($leyend) * 28; + // Percentage labels + $chm = "N*f0*%,000000,0,-1,11"; + $chart = ""; + return $chart; +} + +/** + * Returns the facet array to report on. + */ +function apachesolr_stats_get_facets() { + $all_facets = module_invoke_all('apachesolr_facets'); + // Keep only those enabled according to apachesolr_get_enabled_facets() + $facets = array(); + foreach (apachesolr_get_enabled_facets() as $module => $enabled) { + foreach ($enabled as $facet_id) { + $facets[$facet_id] = $all_facets[$facet_id]; + } + } + + // Add some "virtual" facets to report on. + $facets['kw'] = array( + 'facet_field' => 'kw', + 'info' => 'Keyword search', + ); + $facets['any'] = array( + 'facet_field' => 'any', + 'info' => '[All queries including any filter and/or keywords]', + ); + /* + $facets['none'] = array( + 'facet_field' => 'none', + 'info' => '[Clickthrus with no previous queries]', + ); + */ + return $facets; +} + +/** + * Gather facet usage statistics by traversing logfiles + * THIS FUNCTION IS JUST HERE for some code, will not be actually used... + */ +function apachesolr_stats_get_facet_usage() { + // Get all facets to report on + $facets = apachesolr_stats_get_facets(); + + // Traverse logs and collect statistics + $sessions = 0; + $prev_hostname = ""; + $prev_timestamp = -1; + $timeout = 500; + $limit=""; + $result = db_query("SELECT * FROM {watchdog} WHERE type IN('search', 'KHUB clickthru') ORDER BY uid, hostname, timestamp LIMIT 2000"); + while ($data = db_fetch_object($result)) { + #dprint_r($data); + if ($data->hostname != $prev_hostname || $data->timestamp >$prev_timestamp+$timeout) { + if ($prev_timestamp > 0) { + // Different session + $sessions++; + $session_total_time = $prev_timestamp - $session_start; + $sessions_total_time += $session_total_time; + + // Reset current flags + unset($facet_processed_flag); + $session_start = $data->timestamp; + + /* + echo "Duration: $session_total_time
"; + echo "Totals: All searches: {$facet_usage[any]} Clickthrus: ". ($facet_clickthru[any] + $facet_clickthru[none]) ."
"; + echo "Session #$sessions: Start: ". strftime("%c", $prev_timestamp) ." {$data->hostname} {$data->uid}
"; + */ + } else { + // First session in log traversal, be gentle. + $prev_timestamp = $data->timestamp; + $session_start = $prev_timestamp; + } + } + + // Event: clickthru + if ($data->type == "KHUB clickthru") { + $session_clickthru[$sessions]++; + + // Count "clickthru" for currently active facets (from last search): NOTE this includes "kw", "any", "none", and any other "virtual" facets defined above + if (is_array($facet_processed_flag)) { + foreach ($facet_processed_flag as $fieldname => $dummy) { + $facets[$fieldname]['clickthru']++; + } + } else { + $facets["none"]['clickthru']++; + } + } + + // Event: search + if ($data->type == "search") { + /** + * variables[%keys] examples: + * filters=tid:16119 + * managementfilters=tid:13590 + * management human resourcesfilters=tid:1369 uid:1 + */ + #dprint_r($data); + $variables = unserialize($data->variables); + $query = $variables['%keys']; + $query = str_replace('filters=', ' ', $query); + $ok = preg_match_all('/[^ ]+/', $query, $matches); + + // Process each query's word + unset($facet_processed_flag); + foreach($matches[0] as $word) { + $fieldname = apachesolr_stats_determine_field_from_query($word); + + // Count this facet field only once per query + if ($facet_processed_flag[$fieldname] != true) { + // Add 1 to usage of term from vocabulary + $facets[$fieldname]['usage']++; + // Mark so we don't count it again for this query + $facet_processed_flag[$fieldname] = true; + } + } + + // Count each unique query + if (!$facet_processed_flag["any"]) { + $facets["any"]['usage']++; + $facet_processed_flag["any"] = true; + } + } + // Store current hostname, timestamp + $prev_hostname = $data->hostname; + $prev_timestamp = $data->timestamp; + } + return $facets; } -*/ \ No newline at end of file