vertex_ai_search-1.0.0-beta4/src/Service/VertexSearchManager.php
src/Service/VertexSearchManager.php
<?php
namespace Drupal\vertex_ai_search\Service;
use Drupal\Component\Plugin\Exception\PluginException;
use Drupal\Core\Url;
use Drupal\Core\Utility\Token;
use Drupal\Core\Flood\FloodInterface;
use Drupal\Core\Pager\PagerManagerInterface;
use Drupal\Core\StringTranslation\StringTranslationTrait;
use Drupal\search\SearchPageRepositoryInterface;
use Drupal\vertex_ai_search\VertexAutocompletePluginManager;
use Drupal\vertex_ai_search\VertexSearchFilterPluginManager;
use Drupal\vertex_ai_search\VertexSearchResultsPluginManager;
use Google\ApiCore\ApiException;
use Google\Cloud\DiscoveryEngine\V1\Client\SearchServiceClient;
use Google\Cloud\DiscoveryEngine\V1\SearchRequest;
use Google\Cloud\DiscoveryEngine\V1\SearchRequest\SpellCorrectionSpec;
use Google\Cloud\DiscoveryEngine\V1\SearchRequest\SpellCorrectionSpec\Mode;
use Psr\Log\LoggerInterface;
use Symfony\Component\HttpFoundation\RequestStack;
/**
* Gets Dynamic List Manager.
*/
class VertexSearchManager implements VertexSearchManagerInterface {
use StringTranslationTrait;
/**
* Search Page Repository Manager.
*
* @var \Drupal\search\SearchPageRepositoryInterface
*/
protected $searchPageRepository;
/**
* Vertex Autocomplete Plugin Manager.
*
* @var \Drupal\vertex_ai_search\VertexAutocompletePluginManager
*/
protected $autocompletePluginManager;
/**
* Vertex Search Filter Plugin Manager.
*
* @var \Drupal\vertex_ai_search\VertexSearchFilterPluginManager
*/
protected $searchFilterPluginManager;
/**
* Vertex Search Results Plugin Manager.
*
* @var \Drupal\vertex_ai_search\VertexSearchResultsPluginManager
*/
protected $searchResultsPluginManager;
/**
* The token service.
*
* @var \Drupal\Core\Utility\Token
*/
protected $tokenManager;
/**
* Flood control instance.
*
* @var \Drupal\Core\Flood\FloodInterface
*/
protected $floodManager;
/**
* The Request stack.
*
* @var \Symfony\Component\HttpFoundation\RequestStack
*/
protected $requestStack;
/**
* PagerManager service object.
*
* @var \Drupal\Core\Pager\PagerManagerInterface
*/
protected $pagerManager;
/**
* Logger service object.
*
* @var \Psr\Log\LoggerInterface
*/
protected $logger;
/**
* Object constructor.
*
* @param \Drupal\search\SearchPageRepositoryInterface $search_page_repository
* Search page repository manager.
* @param \Drupal\vertex_ai_search\VertexAutocompletePluginManager $autocomplete_plugin_manager
* Vertex AI Search Autocomplete Manager.
* @param \Drupal\vertex_ai_search\VertexSearchFilterPluginManager $search_filter_plugin_manager
* Vertex AI Search filter plugin manager.
* @param \Drupal\vertex_ai_search\VertexSearchResultsPluginManager $search_results_plugin_manager
* Vertex AI Search results plugin manager.
* @param \Drupal\Core\Utility\Token $token_manager
* For managing Tokens.
* @param \Drupal\Core\Flood\FloodInterface $flood_manager
* Flood control instance.
* @param \Symfony\Component\HttpFoundation\RequestStack $request_stack
* The request stack.
* @param \Drupal\Core\Pager\PagerManagerInterface $pager_manager
* This is the Pager Manager.
* @param Psr\Log\LoggerInterface $logger
* This is the Logger.
*/
public function __construct(
SearchPageRepositoryInterface $search_page_repository,
VertexAutocompletePluginManager $autocomplete_plugin_manager,
VertexSearchFilterPluginManager $search_filter_plugin_manager,
VertexSearchResultsPluginManager $search_results_plugin_manager,
Token $token_manager,
FloodInterface $flood_manager,
RequestStack $request_stack,
PagerManagerInterface $pager_manager,
LoggerInterface $logger,
) {
$this->searchPageRepository = $search_page_repository;
$this->autocompletePluginManager = $autocomplete_plugin_manager;
$this->searchFilterPluginManager = $search_filter_plugin_manager;
$this->searchResultsPluginManager = $search_results_plugin_manager;
$this->tokenManager = $token_manager;
$this->floodManager = $flood_manager;
$this->requestStack = $request_stack;
$this->pagerManager = $pager_manager;
$this->logger = $logger;
}
/**
* {@inheritdoc}
*/
public function executeSearch(array $search_configuration, array $search_parameters) {
// Get initial results array.
$results = $this->retrieveInitialResultsArray();
// Add a subset of search configuration to the results array.
$this->retrieveResponseConfig($results, $search_configuration);
// Check for Flooding.
if ($this->isWithinFloodThreshold($search_configuration) === FALSE) {
$results = $this->retrieveInitialResultsArray();
$results['error'] = $this->t(
'@floodMessage',
['@floodMessage' => $search_configuration['flood_message']]
);
return $results;
}
// Check if communication with Google Vertex is disabled.
$disableQueries = $search_configuration['disable_google_api_queries'] ?? FALSE;
if ($disableQueries) {
$this->retrieveSampleResult($results);
$this->replaceMessageTokens($results, $search_parameters);
$this->determineDisplayMessages($results);
return $results;
}
// Prepare a request to send to Vertex AI Search.
$searchRequest = $this->prepareSearchRequest(
$search_configuration,
$search_parameters
);
// If a searchRequest could not be prepared.
if (empty($searchRequest)) {
$this->replaceMessageTokens($results, $search_parameters);
$this->determineDisplayMessages($results);
return $results;
}
// Perform Vertex Search and update $results array.
$this->performVertexSearch(
$results,
$search_configuration,
$searchRequest,
$search_parameters
);
// Retrieve curated results and update $results array.
$this->retrievePluginCuratedResults(
$results,
$search_configuration,
$search_parameters
);
// Update custom messages in $results array.
$this->replaceMessageTokens($results, $search_parameters);
// Set display messages in $results array.
$this->determineDisplayMessages($results);
// Register the request with flood control.
$this->registerFloodRequest($search_configuration);
// Create a pager.
$this->createPager($results, $search_configuration);
return $results;
}
/**
* Helper to retrieve the configuration of the relevant custom search page.
*
* @param array $search_page_config
* Configuration of the relevant custom search page.
* @param array $search_parameters
* Array of search parameters (keys, page, etc...).
*
* @return \Google\Cloud\DiscoveryEngine\V1\SearchRequest
* A Vertex Search Request ready to be sent.
*/
protected function prepareSearchRequest(array $search_page_config, array $search_parameters) {
// Configure Search Client with serving configuration.
// @see: https://cloud.google.com/php/docs/reference/cloud-discoveryengine/0.4.0/V1.Client.SearchServiceClient.
$formattedServingConfig = SearchServiceClient::servingConfigName(
$search_page_config['google_cloud_project_id'],
$search_page_config['google_cloud_location'],
$search_page_config['vertex_ai_data_store_id'],
$search_page_config['vertex_ai_serving_config']
);
// Prepare the search request.
$request = (new SearchRequest())->setServingConfig($formattedServingConfig);
// Retrieve any exclusions.
$searchKeys = empty($search_page_config['exclusion_list']) ?
$search_parameters['keys'] :
$this->applyExclusionList(
$search_page_config['exclusion_list'],
$search_parameters['keys']
);
if (empty($searchKeys)) {
return NULL;
}
// Set Keywords to be used in request query.
$request->setQuery($searchKeys);
// Set results per page.
$request->setPageSize($search_page_config['resultsPerPage']);
// Specify if safe search is on or off.
$safeSearch = !empty($search_page_config['safeSearch']) ? TRUE : FALSE;
$request->setSafeSearch($safeSearch);
// Specify the spelling correction mode (automatic or not).
$spellMode = MODE::value($search_page_config['spelling_correction_mode']);
if (
!empty($search_parameters['correction']) &&
$search_parameters['correction'] === 'override'
) {
$spellMode = MODE::value('SUGGESTION_ONLY');
}
$spellCorrection = new SpellCorrectionSpec();
$spellCorrection->setMode($spellMode);
$request->setSpellCorrectionSpec($spellCorrection);
// Set offset (starting point) of search request.
$page = !empty($search_parameters['page']) ? $search_parameters['page'] : 0;
$offset = $search_page_config['resultsPerPage'] * $page;
$request->setOffset($offset);
// Set filter for search request if configured.
$filter = $this->retrievePluginFilter($search_page_config);
if ($filter) {
$request->setFilter($filter);
}
return $request;
}
/**
* Helper to perform a Vertex Search.
*
* @param array &$results
* Array holding response data.
* @param array $search_page_config
* Configuration of the relevant custom search page.
* @param \Google\Cloud\DiscoveryEngine\V1\SearchRequest $search_request
* A Vertex Search Request ready to be sent.
* @param array $search_parameters
* The query string parameters of the search.
*/
protected function performVertexSearch(array &$results, array $search_page_config, SearchRequest $search_request, array $search_parameters) {
$results['page'] = $search_parameters['page'] ?? 0;
// Retrieve credentials and create a search client.
// @see https://github.com/googleapis/google-cloud-php/blob/main/AUTHENTICATION.md.
$credPath = $search_page_config['service_account_credentials_file'];
$transport = $search_page_config['transport_method'] ?? NULL;
// Get client parameters and create datastore Service Client.
$clientParameters['credentials'] = json_decode(
file_get_contents($credPath),
TRUE
);
if (!empty($transport)) {
$clientParameters['transport'] = $transport;
}
// Call the API and handle any network failures.
try {
$searchServiceClient = new SearchServiceClient($clientParameters);
/** @var \Google\ApiCore\PagedListResponse $response */
$response = $searchServiceClient->search($search_request);
$page = $response->getPage();
$responseObject = $page->getResponseObject();
$results['totalEstimatedResults'] = $responseObject->getTotalSize();
$results['nextPageToken'] = $responseObject->getNextPageToken();
$results['pageResultCount'] = $page->getPageElementCount();
$results['correctedQuery'] = $responseObject->getCorrectedQuery();
// Use the lesser of total results and total results limit.
$results['totalResults'] =
($results['totalEstimatedResults'] < $search_page_config['totalResultsLimit'])
? $results['totalEstimatedResults'] : $search_page_config['totalResultsLimit'];
// Determine query correction status.
$results['queryCorrected'] = FALSE;
$correctConfig = MODE::name($search_request->getSpellCorrectionSpec()->getMode());
if (!empty($results['correctedQuery']) &&
($correctConfig !== 'SUGGESTION_ONLY')) {
$results['queryCorrected'] = TRUE;
}
// If option to remove domain is set, strip the domain from result links.
if (!empty($results['configuration']['removeDomain'])) {
/** @var \Google\Cloud\DiscoveryEngine\V1\SearchResponse\SearchResult $result */
foreach ($page as $result) {
$resultDecoded = json_decode($result->serializeToJsonString(), TRUE);
$this->stripDomainFromResult($resultDecoded);
$results['results'][] = $resultDecoded;
}
}
else {
/** @var \Google\Cloud\DiscoveryEngine\V1\SearchResponse\SearchResult $result */
foreach ($page as $result) {
$results['results'][] = json_decode($result->serializeToJsonString(), TRUE);
}
}
}
catch (ApiException $ex) {
// ApiException Exception.
$this->logger->critical(
'Vertex AI Search APIException error occurred: @error.
Message: @message.',
[
'@error' => $ex->getStatus(),
'@message' => $ex->getMessage(),
],
);
}
catch (\Exception $ex) {
$this->logger->critical('Vertex AI Search Exception error occurred.
Message: @message. Details: @details.',
[
'@message' => $ex->getMessage(),
'@details' => $ex->__toString(),
],
);
}
}
/**
* Check Flood Levels.
*/
private function isWithinFloodThreshold(array $search_page_config) {
if (!empty($search_page_config['flood_enable'])) {
if (!$this->floodManager->isAllowed(
'vertex_ai_search.flood_level',
$search_page_config['flood_threshold'],
$search_page_config['flood_window']
)) {
return FALSE;
}
}
return TRUE;
}
/**
* Registers flood requests if food control is on.
*/
private function registerFloodRequest(array $search_page_config) {
if (!empty($search_page_config['flood_enable'])) {
$this->floodManager->register(
'vertex_ai_search.flood_level',
$search_page_config['flood_window']
);
}
}
/**
* Helper to perform a Vertex Search.
*
* @param array &$results
* Array holding response data.
* @param array $search_page_config
* Configuration of the relevant custom search page.
*/
protected function createPager(array &$results, array $search_page_config) {
// Create a pager customized for uncertain total results count.
if (!empty($results['nextPageToken']) || !empty($results['page'])) {
if (!empty($search_page_config['pagerType'])
&& $search_page_config['pagerType'] === 'VERTEX') {
$this->pagerManager->createVertexPager(
$results['totalResults'],
$search_page_config['resultsPerPage']
);
}
else {
$this->pagerManager->createPager(
$results['totalResults'],
$search_page_config['resultsPerPage'],
0
);
}
}
}
/**
* Helper to apply exclusion list to search keys.
*
* @param string $exclusion_list
* Terms to be excluded as configured for the page.
* @param string $keys
* Search Keys from which excluded terms are stripped.
*
* @return string
* The search keys minus excluded words.
*/
private function applyExclusionList(string $exclusion_list, string $keys) {
$excludeArray = preg_split("/\r\n|[\r\n]/", $exclusion_list);
foreach ($excludeArray as $exclusion) {
$exclusion = trim($exclusion);
$keys = preg_replace('/' . $exclusion . '/i', '', $keys);
}
return trim($keys);
}
/**
* Helper to retrieve an initialized results array.
*
* @return array
* An initialized search results array with all expected keys.
*/
private function retrieveInitialResultsArray() {
return [
'totalResults' => 0,
'totalEstimatedResults' => 0,
'pageResultCount' => 0,
'nextPageToken' => NULL,
'correctedQuery' => NULL,
'queryCorrected' => FALSE,
'results' => [],
'curated' => [],
'configuration' => [],
'errorMessage' => NULL,
'page' => 0,
];
}
/**
* Helper function to retrieve a single sample result.
*
* @param array &$results
* Array containing response data.
*/
protected function retrieveSampleResult(array &$results) {
$results['totalResults'] = 1;
$results['totalEstimatedResults'] = 1;
$results['pageResultCount'] = 1;
$results['results'] = [
[
"id" => "0",
"document" => [
"name" => "projects/foo/bar",
"id" => "0",
"derivedStructData" => [
"link" => "https://www.example.com/",
"formattedUrl" => "https://www.example.com/",
"htmlFormattedUrl" => "https://www.example.com/",
"pagemap" => [],
"displayLink" => "www.example.com",
"title" => "Example search result",
"htmlTitle" => "This is an example <b>HTML search result</b>.",
"snippets" => [
[
"htmlSnippet" => "Example <b>HTML snippet</b>.",
"snippet" => "Example text snippet.",
],
],
],
],
],
];
}
/**
* Helper function to prep config array to be included in response.
*
* This is configuration information that may be helpful to the client.
*
* @param array &$results
* Array containing response data.
* @param array $search_page_config
* The full configuration array.
*/
protected function retrieveResponseConfig(array &$results, array $search_page_config) {
$responseConfig = [
'label',
'autocomplete_enable',
'autocomplete_trigger_length',
'autocomplete_max_suggestions',
'resultsPerPage',
'totalResultsLimit',
'result_parts',
'removeDomain',
'pagerType',
];
$responseConfiguration = array_intersect_key($search_page_config, array_flip($responseConfig));
$responseConfig['messages'] = [
'results_message',
'results_message_singular',
'no_results_message',
'no_keywords_message',
'correction_made_message',
'correction_suggestion_message',
];
$responseMessages = array_intersect_key($search_page_config, array_flip($responseConfig['messages']));
$responseConfiguration['messages'] = $responseMessages;
$results['configuration'] = $responseConfiguration;
}
/**
* Gets the formatted filter from the selected plugin.
*
* @param array $search_page_config
* Array containing configuration of relevant custom search page.
*
* @return false|string
* The formatted filter or FALSE.
*/
private function retrievePluginFilter(array $search_page_config) {
if (empty($search_page_config['filter_enable']) || empty($search_page_config['filter_plugin'])) {
return FALSE;
}
$filterPlugin = NULL;
try {
$filterPlugin = $this->searchFilterPluginManager->createInstance(
$search_page_config['filter_plugin'],
$search_page_config
);
}
catch (PluginException $ex) {
$this->logger->error(
'PluginException error occurred for filter_plugin: @id.
Message: @message.',
[
'@id' => $search_page_config['filter_plugin'],
'@message' => $ex->getMessage(),
],
);
return FALSE;
}
return $filterPlugin->getSearchFilter();
}
/**
* Gets results manually curated by search results plugins.
*
* @param array &$results
* Array containing response data.
* @param array $search_page_config
* Array containing configuration of relevant custom search page.
* @param array $search_page_parameters
* Array containing parameters of the search.
*/
private function retrievePluginCuratedResults(array &$results, array $search_page_config, array $search_page_parameters) {
// Curated Results.
$curated = [];
// Retrieve any VertexSearchResultPlugins and curated results.
$resultsPluginDefinitions = $this->searchResultsPluginManager->getDefinitions();
foreach ($resultsPluginDefinitions as $pluginKey => $pluginDefinition) {
$resultsPlugin = $this->searchResultsPluginManager->createInstance(
$pluginKey
);
$curated = $resultsPlugin->retrieveCuratedResults($search_page_config, $search_page_parameters);
}
$results['curated'] = $curated;
}
/**
* Replaces the tokens in the custom messages and sets relevant message.
*
* @param array &$results
* The results array containing the response data from a vertex.
* @param array $search_parameters
* Parameters for performing a search.
*/
private function replaceMessageTokens(array &$results, array $search_parameters) {
// Store originalURL (all parts) and originalKeys.
$originalRequest = $this->requestStack->getCurrentRequest();
$originalURL = $originalRequest->getUri();
// Set the corrected values to original values as default.
$correctedURL = $originalURL;
$correctedParameters = $search_parameters;
if (!empty($correctedParameters['correction'])) {
unset($correctedParameters['correction']);
}
// If correctedQuery is set in the response, make changes.
if (!empty($results['correctedQuery'])) {
$correctedParameters['keys'] = $results['correctedQuery'];
$correctedParameters['page'] = 0;
$correctedURL = Url::fromUri(strtok($originalURL, '?'), ['query' => $correctedParameters]);
$correctedURL = $correctedURL->toString();
}
// If query corrected, add parameter to allow search without correction.
if ($results['queryCorrected'] === TRUE) {
// Do not override the actual request stack url; make copy.
$notCorrectedRequest = clone $originalRequest;
$notCorrectedRequest->query->set('correction', 'override');
$notCorrectedRequest->query->set('page', 0);
$notCorrectedRequest->overrideGlobals();
$notCorrectedURL = $notCorrectedRequest->getUri();
}
// If the 'search-path' parameter is set, modify the URLs accordingly.
if (!empty($search_parameters['search-path'])) {
$correctedURL = Url::fromUserInput($search_parameters['search-path'], ['query' => $correctedParameters])->toString();
$originalURL = Url::fromUserInput($search_parameters['search-path'], ['query' => $search_parameters])->toString();
$notCorrectedParameters = $search_parameters;
$notCorrectedParameters['correction'] = 'override';
$notCorrectedParameters['page'] = 0;
$notCorrectedURL = Url::fromUserInput($search_parameters['search-path'], ['query' => $notCorrectedParameters])->toString();
}
// Retrieve the starting result count and ending count for the page.
$page = !empty($search_parameters['page']) ? $search_parameters['page'] : 0;
$startCount = ($page * $results['configuration']['resultsPerPage']) + 1;
$endCount = $startCount + count($results['results']) - 1;
// Populate the data array used to populate Vertex AI Search custom tokens.
$tokens['vertex_ai_search'] = [
'vertex_ai_search_keywords' => ($results['queryCorrected']) ? $correctedParameters['keys'] : $search_parameters['keys'],
'vertex_ai_search_result_start' => $startCount,
'vertex_ai_search_result_end' => $endCount,
'vertex_ai_search_page' => $results['configuration']['label'],
'vertex_ai_search_original_keyword' => $search_parameters['keys'],
'vertex_ai_search_original_keyword_url' => ($notCorrectedURL) ?? $originalURL,
'vertex_ai_search_corrected_keyword' => $correctedParameters['keys'],
'vertex_ai_search_corrected_keyword_url' => $correctedURL,
'vertex_ai_search_total_result_count' => $results['totalResults'],
'vertex_ai_search_estimated_result_count' => $results['totalEstimatedResults'],
];
// Replace tokens in custom search page messages.
foreach ($results['configuration']['messages'] as $key => $message) {
$message = $this->tokenManager->replace($results['configuration']['messages'][$key], $tokens);
$results['configuration']['messages'][$key] = htmlspecialchars_decode($message);
}
// Add tokens to the results array.
$results['tokens'] = $tokens['vertex_ai_search'];
}
/**
* Sets default messages based on results and corrections.
*
* @param array &$results
* The results array containing the response data from a vertex.
*/
private function determineDisplayMessages(array &$results) {
// Set the default convenience messages.
$results['messages']['correction'] = NULL;
$results['messages']['correction_type'] = NULL;
$results['messages']['results'] = NULL;
$results['messages']['results_type'] = NULL;
// Set the correction message.
if (!empty($results['correctedQuery'])) {
if ($results['queryCorrected'] === TRUE) {
$results['messages']['correction'] = $results['configuration']['messages']['correction_made_message'] ?? '';
$results['messages']['correction_type'] = 'correction_made_message';
}
else {
$results['messages']['correction'] = $results['configuration']['messages']['correction_suggestion_message'] ?? '';
$results['messages']['correction_type'] = 'correction_suggestion_message';
}
}
// Set the results message.
$results['messages']['results'] = NULL;
if (empty($results['tokens']['vertex_ai_search_keywords'])) {
$results['messages']['results'] = $results['configuration']['messages']['no_keywords_message'];
$results['messages']['results_type'] = 'no_keywords_message';
}
elseif (empty($results['results'])) {
$results['messages']['results'] = $results['configuration']['messages']['no_results_message'];
$results['messages']['results_type'] = 'no_results_message';
}
elseif (count($results['results']) === 1) {
$results['messages']['results'] = $results['configuration']['messages']['results_message_singular'];
$results['messages']['results_type'] = 'results_message_singular';
}
else {
$results['messages']['results'] = $results['configuration']['messages']['results_message'];
$results['messages']['results_type'] = 'results_message';
}
}
/**
* Helper function to strip the domain from a search result link.
*
* @param array $result
* Reference to the search result array.
*/
private function stripDomainFromResult(array &$result) {
if (isset($result['document']['derivedStructData']['link'])) {
$url_parts = parse_url($result['document']['derivedStructData']['link']);
$new_link = str_replace(
$url_parts['scheme'] . '://' . $url_parts['host'],
'',
$result['document']['derivedStructData']['link']
);
$result['document']['derivedStructData']['link'] = $new_link;
}
if (isset($result['document']['derivedStructData']['formattedUrl'])) {
$url_parts = parse_url($result['document']['derivedStructData']['formattedUrl']);
$new_link = str_replace(
$url_parts['scheme'] . '://' . $url_parts['host'],
'',
$result['document']['derivedStructData']['formattedUrl']
);
$result['document']['derivedStructData']['formattedUrl'] = $new_link;
}
}
}
