coveo-1.0.0-alpha1/modules/coveo_search_api/src/SyncFields.php
modules/coveo_search_api/src/SyncFields.php
<?php
declare(strict_types=1);
namespace Drupal\coveo_search_api;
use Drupal\Core\Entity\TypedData\EntityDataDefinitionInterface;
use Drupal\Core\Field\FieldDefinitionInterface;
use Drupal\Core\Field\FieldStorageDefinitionInterface;
use Drupal\Core\Logger\LoggerChannelFactoryInterface;
use Drupal\Core\Logger\LoggerChannelTrait;
use Drupal\Core\TypedData\ComplexDataDefinitionInterface;
use Drupal\Core\TypedData\ListDataDefinitionInterface;
use Drupal\coveo\Entity\CoveoOrganizationInterface;
use Drupal\coveo\FieldConverter;
use Drupal\coveo_search_api\Event\CoveoFieldDataAlter;
use Drupal\coveo_search_api\Event\CoveoFieldOperationsAlter;
use Drupal\coveo_search_api\Plugin\search_api\backend\SearchApiCoveoBackend;
use Drupal\search_api\Item\FieldInterface;
use Drupal\search_api\Utility\FieldsHelper;
use Drupal\search_api\Utility\Utility as SearchApiUtility;
use NecLimDul\ArrayCUD\ArrayCUD;
use NecLimDul\ArrayCUD\CUDTuple;
use NecLimDul\Coveo\FieldApi\Api\FieldsApi;
use NecLimDul\Coveo\FieldApi\Model\FieldListingOptions;
use NecLimDul\Coveo\FieldApi\Model\FieldModel;
use Neclimdul\OpenapiPhp\Helper\Logging\Error as ApiError;
use Symfony\Component\EventDispatcher\EventDispatcherInterface;
/**
* Field sync helper class.
*
* @phpstan-type CoveoData array<string, array{
* 'field': \NecLimDul\Coveo\FieldApi\Model\FieldModel
* }>
* @phpstan-type DrupalData array<string, \Drupal\search_api\Item\FieldInterface>
* @phpstan-type FieldData array{
* 'coveo': CoveoData,
* 'drupal': DrupalData
* }
* @phpstan-type FieldOperations array{
* 'create': array<string,\NecLimDul\Coveo\FieldApi\Model\FieldModel>,
* 'update': array<string,\NecLimDul\Coveo\FieldApi\Model\FieldModel>,
* 'delete': array<string,string>
* }
*/
class SyncFields {
use LoggerChannelTrait;
/**
* Construct a field sync helper.
*
* @param \Drupal\search_api\Utility\FieldsHelper $fieldsHelper
* Field helper service.
* @param \Drupal\coveo_search_api\CoveoServers $coveoServers
* Coveo SearchApi servers services.
* @param \Symfony\Component\EventDispatcher\EventDispatcherInterface $eventDispatcher
* Event dispatcher.
* @param \Drupal\Core\Logger\LoggerChannelFactoryInterface $loggerChannelFactory
* Log channel factory set the current logger.
*/
public function __construct(
private readonly FieldsHelper $fieldsHelper,
private readonly CoveoServers $coveoServers,
private readonly EventDispatcherInterface $eventDispatcher,
LoggerChannelFactoryInterface $loggerChannelFactory,
) {
$this->setLoggerFactory($loggerChannelFactory);
}
/**
* Sync fields with a Coveo organization.
*/
public function syncOrganization(CoveoOrganizationInterface $organization): void {
$this->doSyncOrg($organization, $this->getFieldData($organization));
}
/**
* Get field data for an organization.
*
* @param \Drupal\coveo\Entity\CoveoOrganizationInterface $organization
* Coveo organization.
*
* @return FieldData
* Related field data.
*/
public function getFieldData(CoveoOrganizationInterface $organization) {
/** @var FieldData $field_data */
$field_data = ['coveo' => [], 'drupal' => []];
$field_data['coveo'] += $this->getBackendFields($organization);
foreach ($this->coveoServers->getSearchBackends($organization->id()) as $backend) {
// Get drupal data from indexes...
$field_data['drupal'] += $this->getDrupalFields($backend);
}
// Allow modification of field operations before sending them to Coveo.
$this->eventDispatcher->dispatch(new CoveoFieldDataAlter(
$organization,
$field_data,
));
return $field_data;
}
/**
* Get Coveo field data from organization.
*
* @param \Drupal\coveo\Entity\CoveoOrganizationInterface $organization
* Related Coveo organization.
*
* @return CoveoData|false
* Coveo field information.
*/
private function getBackendFields(CoveoOrganizationInterface $organization): array|false {
// @todo we only need to request the org fields once, but we need to
// refactor the service onto the org and move it up a level to make this
// work.
$field_converter = $organization->getFieldConverter();
$fields_api = $organization->fieldApiCreate(FieldsApi::class);
$field_data = [];
$page = 0;
do {
$fields_response = $fields_api->getFields(
$organization->getOrganizationId(),
new FieldListingOptions([
'page' => $page,
]),
);
if (!$fields_response->isSuccess()) {
ApiError::logError(
$this->getLogger('coveo'),
$fields_response,
);
return FALSE;
}
/** @var \NecLimDul\Coveo\FieldApi\Model\PageModelFieldModel $fields */
$fields = $fields_response->getData();
foreach ($fields->getItems() as $field) {
$field_name = $field->getName();
if ($field_converter->isDrupalField($field_name)) {
// Keep track of the server id so we know what to update later.
$field_data[$field_name] ??= [
'field' => $field,
];
}
}
} while ($page++ < $fields->getTotalPages());
return $field_data;
}
/**
* Build a list of drupal fields from indexes.
*
* @param \Drupal\coveo_search_api\Plugin\search_api\backend\SearchApiCoveoBackend $backend
* Coveo search backend.
*
* @return DrupalData
* Drupal index field data.
*/
private function getDrupalFields(
SearchApiCoveoBackend $backend,
): array {
$organization = $backend->getOrganization();
$field_converter = $organization->getFieldConverter();
$coveo_indexes = $backend->getServer()->getIndexes();
$field_data = [];
foreach ($coveo_indexes as $index) {
$index_fields = $index->getFields();
foreach ($index_fields as $field_id => $field) {
// If the field is magically mapped, we avoid syncing it.
if (!$field_converter->isCoveoMagic($field_id)) {
$field_data[$field_id] ??= $field;
}
}
}
return $field_data;
}
/**
* Sync fields with a Coveo organization.
*
* @param \Drupal\coveo\Entity\CoveoOrganizationInterface $organization
* Related organization.
* @param FieldData $field_data
* Calculated field data for an organization.
*/
private function doSyncOrg(CoveoOrganizationInterface $organization, array $field_data): void {
$this->createInternalFields($organization);
$organization_id = $organization->getOrganizationId();
/** @var FieldOperations $ops */
$ops = [
'create' => [],
'update' => [],
'delete' => [],
];
$field_converter = $organization->getFieldConverter();
// re-key drupal fields to the coveo field names to more easily compare.
$mapped_drupal_field_data = array_reduce(
array_combine(array_keys($field_data['drupal']), array_keys($field_data['drupal'])),
function (array $return, string $i) use ($field_converter, $field_data): array {
$return[$field_converter->convertDrupalToCoveo($i)] = $field_data['drupal'][$i];
return $return;
},
[],
);
// Find updates and removals.
(new ArrayCUD($field_data['coveo'], $mapped_drupal_field_data))->compare(
// Create.
function (CUDTuple $d) use (&$ops): void {
$ops['create'][$d->key] = $this->generateField($d->value, $d->key);
},
// Update.
function (CUDTuple $c, CUDTuple $d_tuple) use (&$ops): void {
$ops['update'][$c->key] = $this->generateField(
$d_tuple->value,
$c->value['field']->getName(),
$c->value['field']->getType(),
);
},
// Delete.
function (CUDTuple $c) use (&$ops): void {
$ops['delete'][$c->key] = $c->key;
},
);
// Allow modification of field operations before sending them to Coveo.
$this->eventDispatcher->dispatch(new CoveoFieldOperationsAlter(
$organization,
$field_data,
$ops,
));
$fields_api = $organization->fieldApiCreate(FieldsApi::class);
if (!empty($ops['create'])) {
$response = $fields_api->createFieldsBatch($organization_id, array_values($ops['create']));
if (!$response->isSuccess()) {
// @todo notify the user there was a problem and to check log.
ApiError::logError(
$this->getLogger('coveo'),
$response,
);
}
}
// @todo You cannot change fields short of attributes, Need to look into.
if (!empty($ops['update'])) {
$response = $fields_api->updateFieldsBatch($organization_id, array_values($ops['update']));
if (!$response->isSuccess()) {
// @todo notify the user there was a problem and to check log.
ApiError::logError(
$this->getLogger('coveo'),
$response,
);
}
}
if (!empty($ops['delete'])) {
$response = $fields_api->removeFieldsBatch($organization_id, $ops['delete']);
if (!$response->isSuccess()) {
// @todo notify the user there was a problem and to check log.
ApiError::logError(
$this->getLogger('coveo'),
$response,
);
}
}
}
/**
* Create internal fields in coveo needed for Drupal to function.
*
* @param \Drupal\coveo\Entity\CoveoOrganizationInterface $organization
* Organization object used to sync fields.
*
* @return bool
* Creation success. True if all fields exist or where created.
*/
public function createInternalFields(CoveoOrganizationInterface $organization): bool {
$create_fields = TRUE;
$create_fields &= $this->createInternalField($organization, FieldConverter::COVEO_ITEM_ID_FIELD, 'Search API ID field for Drupal');
$create_fields &= $this->createInternalField($organization, FieldConverter::COVEO_INDEX_ID_FIELD, 'Source tracking ID field for Drupal');
$create_fields &= $this->createInternalField($organization, FieldConverter::COVEO_PREFIX_FIELD, 'Drupal field prefix');
return (bool) $create_fields;
}
/**
* Helper method to create internal tracking fields in coveo.
*
* @param \Drupal\coveo\Entity\CoveoOrganizationInterface $organization
* Organization object used to sync fields.
* @param string $field
* Field name.
* @param string $description
* Field description.
* @param string $type
* Field storage type. String is probably the right value.
*
* @return bool
* Success of field creation.
*/
private function createInternalField(CoveoOrganizationInterface $organization, string $field, string $description, string $type = 'STRING'): bool {
$fields_api = $organization->fieldApiCreate(FieldsApi::class);
$response = $fields_api->createField($organization->getOrganizationId(), new FieldModel([
'name' => $field,
'description' => $description,
'type' => $type,
]));
// Failure that isn't 412 (field already exists) is a problem.
if (!$response->isSuccess() && $response->getResponse()->getStatusCode() !== 412) {
ApiError::logError(
$this->getLogger('coveo'),
$response,
);
return FALSE;
}
return TRUE;
}
/**
* Generate a field sync operations.
*
* Note: this is hard coded to a reasonable state but needs an interface for
* managing it.
*
* @param \Drupal\search_api\Item\FieldInterface $field
* Drupal SearchAPI field definition.
* @param string $name
* The field name.
* @param string|null $existing_type
* Existing type in Coveo if once exists.
*
* @return \NecLimDul\Coveo\FieldApi\Model\FieldModel
* A field model we can send to Coveo.
*
* @see https://docs.coveo.com/en/8/api-reference/field-api#tag/Fields/operation/createField
*/
private function generateField(FieldInterface $field, string $name, ?string $existing_type = NULL): FieldModel {
$cardinality = $this->getPropertyPathCardinality(
$field->getPropertyPath(),
$field->getIndex()->getPropertyDefinitions($field->getDatasourceId()),
);
$has_multiple = $cardinality !== 1;
$field_type = $has_multiple ? 'STRING' : $this->mapCoveoType($field->getType());
$coveo_field = new FieldModel([
'name' => $name,
'type' => $existing_type ?: $field_type,
'description' => $field->getLabel(),
'sort' => TRUE,
]);
// Detect hierarchy and let Coveo know.
if ($this->isHierarchicalField($field)) {
$coveo_field->setHierarchicalFacet(TRUE);
// If the field has hierarchy, the field has to be set to a multi-value
// facet field otherwise the field is mostly unusable.
$coveo_field->setFacet(FALSE);
$coveo_field->setMultiValueFacet(TRUE);
$coveo_field->setMultiValueFacetTokenizers(';');
}
// Make multi-fields multi-value facets.
elseif ($has_multiple) {
$coveo_field->setFacet(FALSE);
$coveo_field->setMultiValueFacet(TRUE);
$coveo_field->setMultiValueFacetTokenizers(';');
}
else {
$coveo_field->setFacet(TRUE);
// @todo This is not always correct, i.e. title / content type.
if ($field_type === 'STRING') {
$coveo_field->setFacet(FALSE);
$coveo_field->setSort(FALSE);
}
}
// @todo mergeWithLexicon to set field as free text if we know that.
return $coveo_field;
}
/**
* Computes the cardinality of a complete property path.
*
* @param string $property_path
* The property path of the property.
* @param \Drupal\Core\TypedData\DataDefinitionInterface[] $properties
* The properties which form the basis for the property path.
* @param int $cardinality
* The cardinality of the property path so far (for recursion).
*
* @return int
* The cardinality.
*/
protected function getPropertyPathCardinality($property_path, array $properties, $cardinality = 1) {
[$key, $nested_path] = SearchApiUtility::splitPropertyPath($property_path, FALSE);
if (isset($properties[$key])) {
$property = $properties[$key];
if ($property instanceof FieldDefinitionInterface) {
$storage = $property->getFieldStorageDefinition();
if ($storage instanceof FieldStorageDefinitionInterface) {
if ($storage->getCardinality() == FieldStorageDefinitionInterface::CARDINALITY_UNLIMITED) {
// Shortcut. We reached the maximum.
return FieldStorageDefinitionInterface::CARDINALITY_UNLIMITED;
}
$cardinality *= $storage->getCardinality();
}
}
elseif ($property->isList() || $property instanceof ListDataDefinitionInterface) {
// Lists have unspecified cardinality. Unfortunately BaseFieldDefinition
// implements ListDataDefinitionInterface. So the safety net check for
// this interface needs to be the last one!
return FieldStorageDefinitionInterface::CARDINALITY_UNLIMITED;
}
// @phpstan-ignore-next-line
if (isset($nested_path)) {
$property = $this->fieldsHelper->getInnerProperty($property);
if ($property instanceof ComplexDataDefinitionInterface) {
$cardinality = $this->getPropertyPathCardinality($nested_path, $this->fieldsHelper->getNestedProperties($property), $cardinality);
}
}
}
return $cardinality;
}
/**
* Checks if a field is (potentially) hierarchical.
*
* Fields are (potentially) hierarchical if:
* - they point to an entity type; and
* - that entity type contains a property referencing the same type of entity
* (so that a hierarchy could be built from that nested property).
*
* @see \Drupal\search_api\Plugin\search_api\processor\AddHierarchy::getHierarchyFields()
*
* @return bool
* TRUE if the field is hierarchical, FALSE otherwise.
*
* @throws \Drupal\search_api\SearchApiException
*/
protected function isHierarchicalField(FieldInterface $field): bool {
$definition = $field->getDataDefinition();
if ($definition instanceof ComplexDataDefinitionInterface) {
$properties = $this->fieldsHelper->getNestedProperties($definition);
// The property might be an entity data definition itself.
$properties[''] = $definition;
foreach ($properties as $property) {
$property = $this->fieldsHelper->getInnerProperty($property);
if ($property instanceof EntityDataDefinitionInterface) {
if ($this->hasHierarchicalProperties($property)) {
return TRUE;
}
}
}
}
return FALSE;
}
/**
* Checks if hierarchical properties are nested on an entity-typed property.
*
* @param \Drupal\Core\Entity\TypedData\EntityDataDefinitionInterface $property
* The property to be searched for hierarchical nested properties.
*
* @return bool
* TRUE if the property contains hierarchical properties, FALSE otherwise.
*
* @see \Drupal\search_api\Plugin\search_api\processor\AddHierarchy::findHierarchicalProperties()
*/
protected function hasHierarchicalProperties(EntityDataDefinitionInterface $property): bool {
$entity_type_id = $property->getEntityTypeId();
// Check properties for potential hierarchy. Check two levels down, since
// Core's entity references all have an additional "entity" sub-property for
// accessing the actual entity reference, which we'd otherwise miss.
foreach ($this->fieldsHelper->getNestedProperties($property) as $property_2) {
$property_2 = $this->fieldsHelper->getInnerProperty($property_2);
if ($property_2 instanceof EntityDataDefinitionInterface) {
if ($property_2->getEntityTypeId() == $entity_type_id) {
return TRUE;
}
}
elseif ($property_2 instanceof ComplexDataDefinitionInterface) {
foreach ($property_2->getPropertyDefinitions() as $property_3) {
$property_3 = $this->fieldsHelper->getInnerProperty($property_3);
if ($property_3 instanceof EntityDataDefinitionInterface) {
if ($property_3->getEntityTypeId() == $entity_type_id) {
return TRUE;
}
}
}
}
}
return FALSE;
}
/**
* Converts a field type into a Coveo type.
*
* @param string $type
* The original type.
*
* @return string
* The new type.
*/
private function mapCoveoType(string $type): string {
return match ($type) {
'date' => 'DATE',
'integer' => 'LONG_64',
// 'boolean', 'string', 'text' => 'STRING',
default => 'STRING',
};
}
}
