external_entities-8.x-2.x-dev/src/Plugin/ExternalEntities/DataAggregator/GroupAggregator.php
src/Plugin/ExternalEntities/DataAggregator/GroupAggregator.php
<?php
namespace Drupal\external_entities\Plugin\ExternalEntities\DataAggregator;
use Drupal\Component\Plugin\ConfigurableInterface;
use Drupal\Component\Plugin\Exception\PluginException;
use Drupal\Component\Plugin\PluginManagerInterface;
use Drupal\Component\Utility\NestedArray;
use Drupal\Core\Form\FormStateInterface;
use Drupal\Core\Logger\LoggerChannelFactoryInterface;
use Drupal\Core\Messenger\MessengerTrait;
use Drupal\Core\Plugin\PluginFormInterface;
use Drupal\Core\StringTranslation\TranslationInterface;
use Drupal\external_entities\DataAggregator\DataAggregatorBase;
use Drupal\external_entities\DataAggregator\DataAggregatorInterface;
use Drupal\external_entities\Entity\ConfigurableExternalEntityTypeInterface;
use Drupal\external_entities\Entity\ExternalEntityInterface;
use Drupal\external_entities\Entity\ExternalEntityType;
use Drupal\external_entities\Entity\ExternalEntityTypeInterface;
use Drupal\external_entities\Form\XnttSubformState;
use Drupal\external_entities\FieldMapper\FieldMapperBase;
use JsonPath\InvalidJsonException;
use JsonPath\InvalidJsonPathException;
use JsonPath\JsonObject;
use Symfony\Component\DependencyInjection\ContainerInterface;
/**
* External entities data aggregator by groups.
*
* @DataAggregator(
* id = "group",
* label = @Translation("Group Aggregator"),
* description = @Translation("Aggregates data from multiple grouped data sources.")
* )
*/
class GroupAggregator extends DataAggregatorBase implements DataAggregatorInterface {
use MessengerTrait;
/**
* Disabled storage client mode.
*/
const STORAGE_CLIENT_MODE_DISABLED = 0;
/**
* Read-only storage client mode.
*/
const STORAGE_CLIENT_MODE_READONLY = 1;
/**
* Write-only storage client mode.
*/
const STORAGE_CLIENT_MODE_WRITEONLY = 2;
/**
* Read-write storage client mode.
*/
const STORAGE_CLIENT_MODE_READWRITE = 3;
/**
* Read flag for storage client mode.
*/
const STORAGE_CLIENT_FLAG_READ = 0b00000001;
/**
* Wrtie flag for storage client mode.
*/
const STORAGE_CLIENT_FLAG_WRITE = 0b00000010;
/**
* The property mapper manager.
*
* @var \Drupal\Component\Plugin\PluginManagerInterface
*/
protected $propertyMapperManager;
/**
* Remapped storage client instances.
*
* @var \Drupal\external_entities\StorageClient\StorageClientInterface[]
*/
protected $remappedStorageClients = [];
/**
* External entity type config restrictions.
*
* @var array
*/
protected $locks;
/**
* Constructs a DataAggregatorBase object.
*
* @param array $configuration
* A configuration array containing information about the plugin instance.
* @param string $plugin_id
* The plugin_id for the plugin instance.
* @param mixed $plugin_definition
* The plugin implementation definition.
* @param \Drupal\Core\StringTranslation\TranslationInterface $string_translation
* The string translation service.
* @param \Drupal\Core\Logger\LoggerChannelFactoryInterface $logger_factory
* The logger channel factory.
* @param \Drupal\Component\Plugin\PluginManagerInterface $storage_client_manager
* The storage client manager.
* @param \Drupal\Component\Plugin\PluginManagerInterface $property_mapper_manager
* The property mapper plugin manager.
*/
public function __construct(
array $configuration,
string $plugin_id,
$plugin_definition,
TranslationInterface $string_translation,
LoggerChannelFactoryInterface $logger_factory,
PluginManagerInterface $storage_client_manager,
PluginManagerInterface $property_mapper_manager,
) {
parent::__construct(
$configuration,
$plugin_id,
$plugin_definition,
$string_translation,
$logger_factory,
$storage_client_manager,
);
$this->propertyMapperManager = $property_mapper_manager;
}
/**
* {@inheritdoc}
*/
public static function create(
ContainerInterface $container,
array $configuration,
$plugin_id,
$plugin_definition,
) {
return new static(
$configuration,
$plugin_id,
$plugin_definition,
$container->get('string_translation'),
$container->get('logger.factory'),
$container->get('plugin.manager.external_entities.storage_client'),
$container->get('plugin.manager.external_entities.property_mapper'),
);
}
/**
* {@inheritdoc}
*
* Aggregation config holds aggregation configuration for each storage client
* and has the following structure (stored for each client in an 'aggr' key):
* @code
* [
* 'groups' => string[] (an array of group prefixes which can be empty),
* 'group_prefix_strip' => bool (TRUE means the group prefix is stripped),
* 'id_mapper' => ['id' => string, 'config' => array] a property
* mapper config to map the ID field name in storage data,
* 'join_mapper' => ['id' => string, 'config' => array] or NULL, a property
* mapper config to map the join field name in storage data,
* 'merge' => string (one of 'keep', 'over', 'oven', 'ovem' and 'sub'),
* 'merge_as_member' => string (field name to use to store returned data),
* 'merge_join' => string (ID field name to use in previous storage data),
* 'mode' => int (see self::STORAGE_CLIENT_MODE_* for values),
* ],
* @endcode
*/
public function defaultConfiguration() {
return [
'storage_clients' => [],
];
}
/**
* Retrieves the aggragation configuration of the given storage client plugin.
*
* @param int|string $client_key
* The key identifier of the client storage.
*
* @return array
* An associative array with the storage client aggragation configuration.
* If the requested client has no config set, returns an empty array as
* default value.
*/
public function getStorageClientAggregationConfig(
int|string $client_key,
) :array {
return $this->configuration['storage_clients'][$client_key]['aggr'] ?? [];
}
/**
* Sets the aggragation configuration of the given storage client plugin.
*
* If the client key identifier has no storage client id set, nothing is set.
*
* @param array $storage_client_aggr_config
* The new aggregation configuration for the storage client.
* @param int|string $client_key
* The key identifier of the client storage.
*
* @return \Drupal\external_entities\Entity\ExternalEntityTypeInterface
* Returns current instance.
*/
public function setStorageClientAggregationConfig(
array $storage_client_aggr_config,
int|string $client_key,
) :self {
if (!empty($this->configuration['storage_clients'][$client_key]['id'])) {
// Update aggregation config.
$this->configuration['storage_clients'][$client_key]['aggr'] =
$storage_client_aggr_config;
}
return $this;
}
/**
* Returns current external entity type remapped storage clients.
*
* A remapped storage client is storage client from which the associated
* external entity type has been replaced by a virtual one (ie. living in
* memory) that has no other storage client than the remapped one and which
* has only an id field mapping that has been adjusted to meet aggregation
* settings.
*
* @param int|null $flags
* Only return remapped clients matching the given mode flag.
*
* @return \Drupal\external_entities\StorageClient\StorageClientInterface[]
* An array of remapped storage clients matching current external entity
* storage clients. If no current external entity is set, returns an empty
* array.
*/
protected function getRemappedStorageClients(?int $flags = NULL) :array {
if (empty($this->externalEntityType)) {
// Warn for invalid initialization.
$this->logger->warning(
'GroupAggregator::getRemappedStorageClients() called without proper initialization of the external entity type member.'
);
return [];
}
if (!$this->externalEntityType instanceof ConfigurableExternalEntityTypeInterface) {
// Warn for unsupported external entity type.
$this->logger->warning(
'GroupAggregator::getRemappedStorageClients() called with an unsupported external entity type'
);
return [];
}
$config = $this->getConfiguration();
if (!$this->remappedStorageClients) {
$storage_clients = $this->getStorageClients();
$virtual_aggr_config = $config;
unset($virtual_aggr_config['storage_clients']);
foreach ($storage_clients as $client_key => $storage_client) {
// Create a specific virtual external entity type for each storage
// client to handle join fields in field mappings.
// Indeed, when we use join fields, we only need one storage client and
// we may need to map id to a different client source field.
$virtual_xntt_type = $this->externalEntityType->createDuplicate();
// Put back original id to have an external entity type name when
// loading external entity instances.
$virtual_xntt_type->set('id', $this->externalEntityType->id());
// Remap ID if needed.
$aggr_settings = $config['storage_clients'][$client_key]['aggr'];
if (!empty($aggr_settings['id_mapper']['id'])) {
$virtual_xntt_type
->setFieldMapperId(
'id',
ExternalEntityType::DEFAULT_FIELD_MAPPER
)
->setFieldMapperConfig(
'id',
[
'property_mappings' => [
'value' => [
'id' => $aggr_settings['id_mapper']['id'],
'config' => $aggr_settings['id_mapper']['config'] ?? [],
],
],
]
);
// Check if we must also remap a join field using "title" field. We
// use the "title" field because it is always existing on entities so
// we avoid a complex procedure to temporary add a dedicated field to
// handle join mapping.
if (!empty($aggr_settings['join_mapper']['id'])) {
$virtual_xntt_type
->setFieldMapperId(
'title',
ExternalEntityType::DEFAULT_FIELD_MAPPER
)
->setFieldMapperConfig(
'title',
[
'property_mappings' => [
'value' => [
'id' => $aggr_settings['join_mapper']['id'],
'config' => $aggr_settings['join_mapper']['config'] ?? [],
],
],
]
);
}
}
// Now create a remapped storage client.
$storage_client_id = $this->getStorageClientId($client_key);
$storage_client_config = $this->getStorageClientConfig($client_key);
// Data aggregator config on virtual xntt type.
$virtual_xntt_type->setDataAggregatorConfig(
[
'storage_clients' => [
[
'id' => $storage_client_id,
'config' => $storage_client_config,
'aggr' => $aggr_settings,
],
],
]
+ $virtual_aggr_config
);
// It's a duplicate of something already existing, we must set it is not
// new otherwise, some elements might not work as expected, like
// StorageClientBase::getSourceIdFieldName().
$virtual_xntt_type->set('enforceIsNew', FALSE);
$storage_client_config[ExternalEntityTypeInterface::XNTT_TYPE_PROP] = $virtual_xntt_type;
$storage_client_config += $this->getStorageClientDefaultConfiguration();
$this->remappedStorageClients[$client_key] = $this->storageClientManager->createInstance(
$storage_client_id,
$storage_client_config
);
if ($debug_level = $this->getDebugLevel()) {
$this->remappedStorageClients[$client_key]->setDebugLevel($debug_level);
}
}
}
if (isset($flags)) {
// Filter out clients not matching the flags.
$clients = array_filter(
$this->remappedStorageClients,
function ($client_index) use ($config, $flags) {
return $config['storage_clients'][$client_index]['aggr']['mode'] & $flags;
},
ARRAY_FILTER_USE_KEY
);
return $clients;
}
else {
return $this->remappedStorageClients;
}
}
/**
* Returns the storage clients matching an entity id for aggregation.
*
* Only return remapped storage clients that are relevant to the provided
* entity identifier. Since some clients may use alternative fields as
* joins, they will not be filtered here but they should be fitered
* afterward, from the related join field values.
* Returned storage clients are still indexed with their initial index used
* in the external entity type.
*
* @param mixed $id
* Identifier value of current external entity considered.
* @param int $flags
* Filter out clients that have none of the given flags set.
*
* @return \Drupal\external_entities\StorageClient\StorageClientInterface[]
* An array of remapped storage clients indexed using their original
* external entity type storage client index. Only storage clients relevants
* to the given identifier are returned which means there could be expected
* index gaps in the returned array as irrelevant storage clients are
* filtered out. If no current external entity is set, returns an empty
* array.
*/
protected function filterAggregationStorageClientsForId(
$id,
?int $flags = NULL,
) :array {
$clients = $this->getRemappedStorageClients($flags);
$aggr_clients = [];
$config = $this->getConfiguration();
foreach ($clients as $client_index => $client) {
$aggr_settings =
$config['storage_clients'][$client_index]['aggr'];
$groups = $aggr_settings['groups'] ?? [];
if (empty($groups)) {
// Not a group-specific client.
$aggr_clients[$client_index] = $client;
}
else {
// Check if a group prefix macthes this entity ID.
$re = '/^\Q' . implode('\E|^\Q', $groups) . '\E/';
if (1 === preg_match($re, $id)) {
$aggr_clients[$client_index] = $client;
}
}
}
return $aggr_clients;
}
/**
* {@inheritdoc}
*/
public function loadMultiple(?array $ids = NULL) :array {
if (empty($this->externalEntityType)) {
// Warn for invalid initialization.
$this->logger->warning(
'GroupAggregator::loadMultiple() called without proper initialization of the external entity type member.'
);
return [];
}
// Filter out non-read clients.
$clients = $this->getRemappedStorageClients(static::STORAGE_CLIENT_FLAG_READ);
$config = $this->getConfiguration();
$no_groups = empty($config['storage_clients'][0]['aggr']['groups']);
// Check if we are working with groups and set groups.
if ($no_groups) {
// If first storage client has no group, then it is the reference.
// Only its entities will be taken into account.
$clients_by_group = ['' => $clients];
}
else {
// Otherwise we work by groups.
// Get each group reference and set of clients.
$clients_by_group = [];
foreach ($clients as $client_index => $client) {
$groups =
$config['storage_clients'][$client_index]['aggr']['groups']
// No specific groups, add client to all groups.
?? array_keys($clients_by_group);
foreach ($groups as $group) {
$clients_by_group[$group][$client_index] = $client;
}
}
}
if (1 <= $this->getDebugLevel()) {
$group_details = '';
foreach ($clients_by_group as $group => $group_clients) {
$group_details .= $this->t(
"- Group '@group': @clients\n",
[
'@group' => $group,
'@clients' => implode(
', ',
array_map(
function ($c) use ($group_clients) {
return $c . '(' . $group_clients[$c]->getPluginId() . ')';
},
array_keys($group_clients)
)
),
]
);
}
$this->logger->debug(
"Requested @count entities\nStart loading with grouped clients:\n@groups",
[
'@count' => isset($ids) ? count($ids) : $this->t('all'),
'@groups' => $group_details,
]
);
}
$entities = [];
foreach ($clients_by_group as $group => $group_clients) {
$group_len = strlen($group);
// Get the reference storage client.
reset($group_clients);
$ref_client_number = key($group_clients);
$ref_client = current($group_clients);
// We will process the reference storage client separately.
// Unset the ref. client to keep indexes but remove it from the set.
unset($group_clients[$ref_client_number]);
$id_field_mapper = $this
->externalEntityType
->getFieldMapper('id');
if (empty($id_field_mapper)) {
// Warn for invalid configuration.
$this->logger->warning(
'GroupAggregator::loadMultiple() the identifier field (id) is not mapped.'
);
return [];
}
$id_field = $id_field_mapper->getMappedSourceFieldName('value');
if (empty($id_field)) {
// Warn for invalid configuration.
$this->logger->warning(
'GroupAggregator::loadMultiple() the identifier field (id) is not directly mapped to raw field name.'
);
return [];
}
// Check if the reference client uses group filtering.
$groups = $config['storage_clients'][$ref_client_number]['aggr']['groups'] ?? [];
$valid_ids = $ids;
if (is_array($valid_ids)) {
if (!empty($group)) {
// Filter requested ids for this group.
$valid_ids = array_filter(
$valid_ids,
function ($id) use ($group_len, $group) {
return (strncmp($id, $group, $group_len) === 0);
}
);
}
// Check if group prefix needs to be stripped.
if (!empty($config['storage_clients'][$ref_client_number]['aggr']['group_prefix_strip'])
&& !empty($group)
) {
// Use virtual pefix: strip.
$valid_ids = array_map(
function ($x) use ($group_len) {
return substr($x, $group_len);
},
$valid_ids
);
}
}
if (!empty($valid_ids) || (NULL === $ids)) {
if (1 <= $this->getDebugLevel()) {
$this->logger->debug(
"Loading @count entities @group from reference client @client",
[
'@count' => isset($ids) ? count($valid_ids) : $this->t('all'),
'@group' => empty($group) ? 'without group' : 'for group ' . $group,
'@client' => '#' . $ref_client_number,
]
);
}
$client_entities = $ref_client->loadMultiple($valid_ids) ?? [];
}
else {
// No id to load from client.
$client_entities = [];
if (1 <= $this->getDebugLevel()) {
$this->logger->debug(
"No valid entities to load @group from reference client @client",
[
'@group' => empty($group) ? 'without group' : 'for group ' . $group,
'@client' => '#' . $ref_client_number,
]
);
}
}
// Filter ids to only loaded ids.
$valid_ids = array_keys($client_entities);
if (!empty($config['storage_clients'][$ref_client_number]['aggr']['group_prefix_strip'])
&& !empty($group)
) {
// Use virtual pefix: unstrip.
$valid_ids = array_map(
function ($x) use ($group) {
return $group . $x;
},
$valid_ids
);
$remapped_entities = [];
foreach ($client_entities as $client_entity_id => $client_entity) {
if (0 < strlen($client_entity_id ?? '')) {
$client_entity[$id_field] =
$client_entity_id =
$group . $client_entity_id;
$remapped_entities[$client_entity_id] = $client_entity;
}
}
$client_entities = $remapped_entities;
}
$valid_ids = array_combine($valid_ids, $valid_ids);
// Merge arrays but keep keys as they are, especially for integer keys as
// keys are identifiers. We should only get new keys and there should
// not be any override here (unless the client returns duplicates).
foreach ($client_entities as $client_entity_id => $client_entity) {
$entities[$client_entity_id] = $client_entity;
}
// Aggregate values from other storage clients.
$this->aggregateSubStorageClientsData(
$entities,
$group_clients,
$valid_ids,
$id_field,
'loadMultiple'
);
}
return $entities;
}
/**
* Aggregate data from storage client of a given group to current data.
*
* @param array &$entities
* Raw reference client entities array keyed by ids that will contain
* aggregated data.
* @param StorageClientInterface[] $group_clients
* Array of remapped storage clients of a given group.
* @param array $valid_ids
* An array of entity identifiers that correspond to the group.
* @param string $id_field
* The name of the reference storage client id field used for entities.
* @param string $caller
* Name of the calling function used by transliterateDrupalFilters context.
*/
protected function aggregateSubStorageClientsData(
array &$entities,
array $group_clients,
array $valid_ids,
string $id_field,
string $caller = 'aggregateSubStorageClientsData',
) :void {
$config = $this->getConfiguration();
foreach ($group_clients as $client_index => $client) {
// First, get the list of "ids" to load from current client.
// $client_id_map keys are reference entity ids, values are array of
// current client data values to match (either ids or join field).
$client_id_map = [];
foreach ($valid_ids as $valid_id) {
$raw_entity = $entities[$valid_id];
if (!empty($config['storage_clients'][$client_index]['aggr']['merge_join'])) {
// Merge using a specific field on current data as id.
$group_join_field = $config['storage_clients'][$client_index]['aggr']['merge_join'];
// Check if we got a JSONPath.
if (preg_match('/^\$[.\[]/', $group_join_field)) {
$jpath = new JsonObject($raw_entity, TRUE);
try {
$join_value = $jpath->get($group_join_field);
if (FALSE === $join_value) {
$join_value = [];
}
$client_id_map[$raw_entity[$id_field]] =
is_array($join_value)
? $join_value
: [$join_value];
}
catch (InvalidJsonException | InvalidJsonPathException $e) {
// JSONPath mapping failed. Report.
$this->logger->error(
'Invalid JSONPath: '
. $e
. "\nExternal Entity Type: "
. $this->externalEntityType->id()
. "\nStorage client: $client_index\nJSONPath: "
. print_r($group_join_field, TRUE)
. "\nData: "
. print_r($raw_entity, TRUE)
);
$client_id_map[$raw_entity[$id_field]] = [];
}
}
elseif (isset($raw_entity[$group_join_field])) {
$client_id_map[$raw_entity[$id_field]] =
is_array($raw_entity[$group_join_field])
? $raw_entity[$group_join_field]
: [$raw_entity[$group_join_field]];
}
// Else, it means the raw entity does not have any value for that
// join field and no value can be added.
}
else {
// Use current entity id as id value for current client.
$client_id_map = array_map(
function ($vid) {
return [$vid];
},
$valid_ids
);
}
}
// Get all matching storage client entitites at once for all currrent
// entities. We will sort out which ones to join to which ones after.
// Note: the given field to filter on is a raw field, not a Drupal
// one. However, the query filter only supports Drupal fields.
// To get around that problem, the client has been configured
// with an altered field mapping that maps the Drupal id field
// to the current client field to use as id.
// That id field may be used for joining unless a "join_mapper" setting
// is set to select a different field for joining. In that case, the
// Drupal entity "title" field is used to map the join field.
// @see self::getRemappedStorageClients().
$match_values = array_unique(array_merge(...array_values($client_id_map)));
// Check if we match client data by id or by another field.
if (!empty($config['storage_clients'][$client_index]['aggr']['id_mapper']['id'])
&& !empty($config['storage_clients'][$client_index]['aggr']['join_mapper']['id'])
) {
// Match on a join field.
$filter_parameters = [
[
// The "title" field is used to map the join field.
// @see self::getRemappedStorageClients().
'field' => 'title',
'value' => $match_values,
'operator' => 'IN',
],
];
$parameters = $client->transliterateDrupalFilters(
$filter_parameters,
['caller' => $caller]
);
$query_entities = [];
if (!empty($parameters['drupal']) || empty($parameters['source'])) {
$this->logger->warning(
'GroupAggregator::aggregateSubStorageClientsData() is unable to filter storage client '
. $client_index
. ' on a secondary join field as the client does not support filtering on that field (on the source side).'
);
}
else {
// Query returns an non-keyed array of matching entries.
$source_entities = $client->querySource(
$parameters['source']
);
// Remap results by join field.
$join_mapper = $this->propertyMapperManager->createInstance(
$config['storage_clients'][$client_index]['aggr']['join_mapper']['id'],
$config['storage_clients'][$client_index]['aggr']['join_mapper']['config']
?? []
);
foreach ($source_entities as $source_entity) {
if (!is_array($source_entity)) {
if (2 <= $this->getDebugLevel()) {
$this->logger->debug(
"Matching returned data is not a valid structure.\nExternal Entity Type: "
. $this->externalEntityType->id()
. "\nStorage client: $client_index\nData: "
. print_r($source_entity, TRUE)
);
}
continue;
}
$source_entity_join_values = $join_mapper
->extractPropertyValuesFromRawData($source_entity)
?? NULL;
if (empty($source_entity_join_values)) {
if (2 <= $this->getDebugLevel()) {
$this->logger->debug(
"No matching id field found in returned data.\nExternal Entity Type: "
. $this->externalEntityType->id()
. "\nStorage client: $client_index\nData: "
. print_r($source_entity, TRUE)
);
}
}
else {
foreach ($source_entity_join_values as $source_entity_join_value) {
$query_entities[$source_entity_join_value][] = $source_entity;
}
}
}
}
}
else {
// Match on ID.
$loaded_entities = $client->loadMultiple($match_values);
// Remap as array of entities.
foreach ($loaded_entities as $loaded_id => $loaded_entity) {
$query_entities[$loaded_id] = [$loaded_entity];
}
}
// Here, $query_entities is supposed to contain all entities matching
// the given "ids" in an array. However, the "id" field used may not
// provide unique matches: it is possible that more than one returned
// entity share the same "id" (especially for a secondary join field).
// Now, we will use $client_id_map array to match current entity id
// with the values to join. If we got more than one matching new record,
// we will handle 2 cases:
// 1) the new records must be added in a subfield, no problem there.
// 2) the new records must be merged to current one. Each record will
// be merged one after the other it the order of the field containing the
// matching ids which is given by $client_id_map values.
// Sets default merge behavior.
$config['storage_clients'][$client_index]['aggr']['merge'] ??= 'keep';
if (in_array($config['storage_clients'][$client_index]['aggr']['merge'], ['sub', 'translation'])) {
// Store corresponding client entities as member.
$member =
($config['storage_clients'][$client_index]['aggr']['merge_as_member'] ?? '')
?: $client->getPluginId() . '_' . $client_index;
foreach ($client_id_map as $source_id => $matching_ids) {
$entities[$source_id][$member] = [];
foreach ($matching_ids as $matching_id) {
if (!empty($query_entities[$matching_id])) {
$entities[$source_id][$member][$matching_id] = $query_entities[$matching_id];
}
}
}
}
else {
foreach ($client_id_map as $source_id => $matching_ids) {
// Gather current client entities matching $source_id.
$matching_entities = [];
foreach ($matching_ids as $matching_id) {
if (!empty($query_entities[$matching_id])) {
$matching_entities = array_merge($matching_entities, $query_entities[$matching_id]);
}
}
if (!empty($matching_entities)) {
// We got something to merge, check for override.
if ('keep' == $config['storage_clients'][$client_index]['aggr']['merge']) {
// No override of fields already set.
foreach ($matching_entities as $matching_entity) {
$entities[$source_id] = static::mergeArrays(
$matching_entity,
$entities[$source_id],
TRUE,
FALSE,
FALSE,
FALSE
);
}
}
elseif ('over' == $config['storage_clients'][$client_index]['aggr']['merge']) {
foreach ($matching_entities as $matching_entity) {
// Override all except id.
unset($matching_entity[$id_field]);
$entities[$source_id] = static::mergeArrays(
$entities[$source_id],
$matching_entity,
TRUE,
FALSE,
FALSE,
FALSE
);
}
}
elseif ('oven' == $config['storage_clients'][$client_index]['aggr']['merge']) {
// Override only existing fields except id.
foreach ($matching_entities as $matching_entity) {
unset($matching_entity[$id_field]);
$entities[$source_id] = static::mergeArrays(
$entities[$source_id],
$matching_entity,
TRUE,
FALSE,
FALSE,
TRUE
);
}
}
elseif ('ovem' == $config['storage_clients'][$client_index]['aggr']['merge']) {
// Override empty fields only.
foreach ($matching_entities as $matching_entity) {
$entities[$source_id] = static::mergeArrays(
$entities[$source_id],
$matching_entity,
TRUE,
TRUE,
FALSE,
FALSE
);
}
}
else {
// Unknown merge method, warn.
$this->logger->warning(
'Unsupported merge method "'
. $config['storage_clients'][$client_index]['aggr']['merge']
. '" for client '
. $client_index
. ' of external entity type '
. $this->externalEntityType->id()
. '.'
);
}
}
}
}
}
}
/**
* {@inheritdoc}
*/
public function save(ExternalEntityInterface $entity) :int {
$return_value = 0;
$sc_config = $this->getConfiguration()['storage_clients'];
// Filter aggregation plugins.
$aggr_plugins = $this->filterAggregationStorageClientsForId($entity->id(), static::STORAGE_CLIENT_FLAG_WRITE);
$original_id = $entity->id();
foreach ($aggr_plugins as $plugin_index => $plugin) {
// Set new id to the orignal one to later detect changes.
$new_id = $original_id;
// Check if identifier should be adjusted (data joins).
if (!empty($sc_config[$plugin_index]['aggr']['id_mapper']['id'])) {
// Using a join field instead of the default identifier field.
$property_mapper = $this->propertyMapperManager->createInstance(
$sc_config[$plugin_index]['aggr']['id_mapper']['id']
?? FieldMapperBase::DEFAULT_PROPERTY_MAPPER,
$sc_config[$plugin_index]['aggr']['id_mapper']['config']
?? []
);
$raw_data = $entity->toRawData();
$new_id = $property_mapper
->extractPropertyValuesFromRawData($raw_data)[0]
?? NULL;
}
elseif (!empty($sc_config[$plugin_index]['aggr']['group_prefix_strip'])) {
// Default identifier should be stipped.
$groups = $sc_config[$plugin_index]['aggr']['groups'] ?? [];
// Remove group prefixes.
$re = '/^\Q' . implode('\E|^\Q', $groups) . '\E/';
$new_id = preg_replace($re, '', $original_id);
}
if (!isset($new_id) || ('' == $new_id)) {
// The join identifier is not available, skip.
continue 1;
}
if ($new_id === $original_id) {
// No id change.
$return_value = max($plugin->save($entity), $return_value);
}
else {
// Id change.
// Save using the given id.
$entity->id = $new_id;
$entity->setOriginalId($new_id);
$return_value = max($plugin->save($entity), $return_value);
// Put back previous value.
$entity->id = $original_id;
$entity->setOriginalId($original_id);
}
}
return $return_value;
}
/**
* {@inheritdoc}
*/
public function delete(ExternalEntityInterface $entity) {
// Filter aggregation clients.
$aggr_clients = $this->filterAggregationStorageClientsForId($entity->id(), static::STORAGE_CLIENT_FLAG_WRITE);
$original_id = $entity->id();
foreach ($aggr_clients as $client_key => $client) {
$aggr_settings = $this->getConfiguration()['storage_clients'][$client_key]['aggr'] ?? [];
// Check if it is a member object: no edit.
if (!empty($aggr_settings['merge'])
&& ('sub' == $aggr_settings['merge'])
) {
continue;
}
// Set $new_id to the orignal one to later detect changes.
$new_id = $original_id;
// Check if identifier should be adjusted.
if (!empty($aggr_settings['id_mapper']['id'])) {
// Using a join field instead of the default identifier field.
$property_mapper = $this->propertyMapperManager->createInstance(
$sc_config[$plugin_index]['aggr']['id_mapper']['id']
?? FieldMapperBase::DEFAULT_PROPERTY_MAPPER,
$sc_config[$plugin_index]['aggr']['id_mapper']['config']
?? []
);
$raw_data = $entity->toRawData();
$new_id = $property_mapper
->extractPropertyValuesFromRawData($raw_data)[0]
?? NULL;
}
elseif (!empty($aggr_settings['group_prefix_strip'])) {
// Default identifier should be stipped.
$groups = $aggr_settings['groups'] ?? [];
// Remove group prefixes.
$re = '/^\Q' . implode('\E|^\Q', $groups) . '\E/';
$new_id = preg_replace($re, '', $original_id);
}
if (!isset($new_id) || ('' == $new_id)) {
// The join identifier is not available, skip.
continue 1;
}
if ($new_id === $original_id) {
// No id change.
$client->delete($entity);
}
else {
// Id change.
// Delete using the given id.
$entity->id = $new_id;
$entity->setOriginalId($new_id);
$client->delete($entity);
// Put back previous value.
$entity->id = $original_id;
$entity->setOriginalId($original_id);
}
}
}
/**
* {@inheritdoc}
*/
public function query(
array $parameters = [],
array $sorts = [],
?int $start = NULL,
?int $length = NULL,
) :array {
if (empty($this->externalEntityType)) {
// Warn for invalid initialization.
$this->logger->warning(
'GroupAggregator::query() called without proper initialization of the external entity type member.'
);
return [];
}
// Filter out non-read clients.
$clients = $this->getRemappedStorageClients(static::STORAGE_CLIENT_FLAG_READ);
$start ??= 0;
$client_start = $start;
$remaining_length = $length ?? NULL;
$config = $this->getConfiguration();
$no_groups = empty($config['storage_clients'][0]['aggr']['groups']);
// Check if we are working with groups and set groups.
if ($no_groups) {
// If first storage client has no group, then it is the reference.
// Only its entities will be taken into account.
$clients_by_group = ['' => $clients];
}
else {
// Otherwise we work by groups.
// Get each group reference and set of clients.
$clients_by_group = [];
foreach ($clients as $client_index => $client) {
$groups =
$config['storage_clients'][$client_index]['aggr']['groups']
// No specific groups, add client to all groups.
?? array_keys($clients_by_group);
foreach ($groups as $group) {
$clients_by_group[$group][$client_index] = $client;
}
}
}
if (1 <= $this->getDebugLevel()) {
$group_details = '';
foreach ($clients_by_group as $group => $group_clients) {
$group_details .= $this->t(
"- Group '@group': @clients\n",
[
'@group' => $group,
'@clients' => implode(
', ',
array_map(
function ($c) use ($group_clients) {
return $c . '(' . $group_clients[$c]->getPluginId() . ')';
},
array_keys($group_clients)
)
),
]
);
}
$this->logger->debug(
"Start managing aggregation query with grouped clients:\n@groups\nSorting: @sorting\nStart position: @start\nLength: @length",
[
'@groups' => $group_details,
'@sorting' => empty($sorts) ? $this->t('no') : $this->t('yes'),
'@start' => $start,
'@length' => $length ?? 'NULL',
]
);
}
$entities = [];
foreach ($clients_by_group as $group => $group_clients) {
// Check if we're done.
if (isset($remaining_length) && (0 >= $remaining_length)) {
if (1 <= $this->getDebugLevel()) {
$this->logger->debug(
"All needed entities loaded. Stopping here.",
);
}
break;
}
$group_len = strlen($group);
// Get the reference storage client.
reset($group_clients);
$ref_client_number = key($group_clients);
$ref_client = current($group_clients);
// We will process the reference storage client separately.
// Unset the ref. client to keep indexes but remove it from the set.
unset($group_clients[$ref_client_number]);
$id_field_mapper = $this
->externalEntityType
->getFieldMapper('id');
if (empty($id_field_mapper)) {
// Warn for invalid configuration.
$this->logger->warning(
'GroupAggregator::query() the identifier field (id) is not mapped.'
);
return [];
}
$id_field = $id_field_mapper->getMappedSourceFieldName('value');
if (empty($id_field)) {
// Warn for invalid configuration.
$this->logger->warning(
'GroupAggregator::query() the identifier field (id) is not mapped properly as it is not possible to get its corresponding raw field name.'
);
return [];
}
// Check if the reference client uses group filtering.
$groups = $config['storage_clients'][$ref_client_number]['aggr']['groups'] ?? [];
// Get the counts for each client until we reach the first page and
// length.
// @todo We are not filtering out entities with prefix outside of
// current group. Therefore the count might be incorrect.
// Maybe add a filter for identifier but will it be efficient if is is
// not natively supported by the source? Test if the filter is supported
// and if not, remove it and tolerate the possible error?
$client_entity_count = $ref_client->countQuery($parameters);
$client_length = $remaining_length;
if ((0 <= $client_start)
&& ($client_start < $client_entity_count)
) {
$client_entities = $ref_client->query(
$parameters,
$sorts,
$client_start,
$client_length
);
if (1 <= $this->getDebugLevel()) {
$this->logger->debug(
"Current number of entities loaded: @entity_count\nClient @client_number (@client_id)\nClient start position: @client_start\nClient length: @client_length\nLoaded: @client_entity_count",
[
'@entity_count' => count($entities),
'@client_number' => $ref_client_number,
'@client_id' => $ref_client->getPluginId(),
'@client_entity_count' => count($client_entities),
'@client_start' => $client_start,
'@client_length' => $client_length ?? 'n/a',
]
);
}
$client_start = 0;
}
else {
if (1 <= $this->getDebugLevel()) {
$this->logger->debug(
"Currently out of range\nCurrent number of entities loaded: @entity_count\nClient @client_number (@client_id)\nClient start position: @client_start\nClient length: @client_length\nFound: @client_entity_count",
[
'@entity_count' => count($entities),
'@client_number' => $ref_client_number,
'@client_id' => $ref_client->getPluginId(),
'@client_entity_count' => $client_entity_count,
'@client_start' => $client_start,
'@client_length' => $client_length ?? 'n/a',
]
);
}
$client_start = max($client_start - $client_entity_count, 0);
continue;
}
// Check if need to use a mapper.
$property_mapper = NULL;
if (!empty($config['storage_clients'][$ref_client_number]['aggr']['id_mapper']['id'])) {
$property_mapper = $this->propertyMapperManager->createInstance(
$config['storage_clients'][$ref_client_number]['aggr']['id_mapper']['id']
?? FieldMapperBase::DEFAULT_PROPERTY_MAPPER,
$config['storage_clients'][$ref_client_number]['aggr']['id_mapper']['config']
?? []
);
}
// Filter ids by group and remap result by ids.
$remapped_entities = [];
$valid_ids = [];
$group_len = strlen($group);
if ($group_len
&& empty($config['storage_clients'][$ref_client_number]['aggr']['group_prefix_strip'])
) {
foreach ($client_entities as $client_entity) {
if (!empty($property_mapper)) {
$id = $property_mapper
->extractPropertyValuesFromRawData($client_entity)[0]
?? NULL;
}
else {
$id = $client_entity[$id_field];
}
if (!isset($id)) {
continue;
}
if (strncmp($id, $group, $group_len) === 0) {
$valid_ids[] = $id;
$remapped_entities[$id] = $client_entity;
}
}
}
else {
foreach ($client_entities as $client_entity) {
if (!empty($property_mapper)) {
$id = $property_mapper
->extractPropertyValuesFromRawData($client_entity)[0]
?? NULL;
}
else {
$id = $client_entity[$id_field];
}
if (!isset($id)) {
continue;
}
$valid_ids[] = $id;
$remapped_entities[$id] = $client_entity;
}
}
$client_entities = $remapped_entities;
// Update global (virtual) position and count.
// $current_pos += count($client_entities);
if (isset($remaining_length)) {
$remaining_length -= count($client_entities);
}
// Ajust id filter according to prefix settings.
if (!empty($config['storage_clients'][$ref_client_number]['aggr']['group_prefix_strip'])
&& !empty($group)
) {
// Use virtual pefix: unstrip.
$valid_ids = array_map(
function ($x) use ($group) {
return $group . $x;
},
$valid_ids
);
$remapped_entities = [];
foreach ($client_entities as $client_entity) {
if (0 < strlen($client_entity[$id_field] ?? '')) {
$client_entity[$id_field] = $group . $client_entity[$id_field];
$remapped_entities[$client_entity[$id_field]] = $client_entity;
}
}
$client_entities = $remapped_entities;
}
$valid_ids = array_combine($valid_ids, $valid_ids);
// Merge arrays but keep keys as they are, especially for integer keys as
// keys are identifiers. We should only get new keys and there should
// not be any override here (unless the client returns duplicates).
foreach ($client_entities as $client_entity_id => $client_entity) {
$entities[$client_entity_id] = $client_entity;
}
// Aggregate values from other storage clients.
$this->aggregateSubStorageClientsData(
$entities,
$group_clients,
$valid_ids,
$id_field,
'query'
);
}
// Clear id indexation.
$entities = array_values($entities);
return $entities;
}
/**
* {@inheritdoc}
*/
public function countQuery(array $parameters = []) :int {
$count = 0;
// Filter out non-read clients.
$plugins = $this->getRemappedStorageClients(static::STORAGE_CLIENT_FLAG_READ);
$config = $this->getConfiguration();
$sc_config = $config['storage_clients'];
// Count entities from plugins.
$count = 0;
if (empty($sc_config[0]['aggr']['groups'])) {
// If first storage client has no group, then it is the reference.
// Only its entity identifiers will be taken into account.
$count += $plugins[0]->countQuery($parameters);
}
else {
// Otherwise we work by groups.
// Get each group reference and count.
$counted_groups = [];
foreach ($plugins as $plugin_index => $plugin) {
$groups = $sc_config[$plugin_index]['aggr']['groups'] ?? [];
if (empty($groups)) {
// No groups, skip.
continue;
}
else {
// Count each new group.
foreach ($groups as $group) {
if (empty($counted_groups[$group])) {
// @todo We can't check if the counted entity identifiers match
// their group. Maybe we should add an id filter?
$count += $plugin->countQuery($parameters);
$counted_groups[$group] = TRUE;
}
}
}
}
}
return $count;
}
/**
* {@inheritdoc}
*/
public function buildConfigurationForm(array $form, FormStateInterface $form_state) {
// Check if external entity type has locks.
if (!empty($this->externalEntityType)
&& is_a($this->externalEntityType, ConfigurableExternalEntityTypeInterface::class)
) {
$this->locks = $this->externalEntityType->getLocks() ?? [];
}
$da_id = ($form['#attributes']['id'] ??= uniqid('da', TRUE));
$config = $this->getConfiguration();
$storage_count =
$form_state->get('storage_count')
?? count($config['storage_clients']);
// We must have at least one storage.
if (empty($storage_count)) {
$storage_count = 1;
}
$form_state->set('storage_count', $storage_count);
// Get storage client configs: try from form state first and then from
// config.
$storage_settings = $form_state->get('storage_settings');
if ((!isset($storage_settings) || empty($storage_settings))) {
$storage_settings = [];
if (!$this->externalEntityType->isNew()) {
for ($i = 0; $i < $storage_count; ++$i) {
$storage_settings[$i] = [
'id' => $this->getStorageClientId($i),
'config' => $this->getStorageClientConfig($i),
'notes' => $this->getStorageClientNotes($i),
];
}
}
else {
for ($i = 0; $i < $storage_count; ++$i) {
$storage_settings[$i] = [
'id' => '',
'config' => [],
'notes' => '',
];
}
}
$form_state->set('storage_settings', $storage_settings);
}
$form['multi_info'] = [
'#type' => 'item',
'#markup' => $this->t(
'You can specify one or more storage client. Their order is important: the first one (of a group) is loaded first and will be used as the reference (for its group). Each group reference is considered holding all the records of the group. Other records provided by the following storage clients will be discarded.
'),
];
$form['storage_clients'] = [
'#type' => 'container',
'#attributes' => [
'id' => $da_id . '_scs',
],
];
for ($client_number = 0; $client_number < $storage_count; ++$client_number) {
$form['storage_clients'][$client_number] = [
'#type' => 'details',
'#title' => $this->t(
'Storage client #@index',
['@index' => $client_number + 1]
),
'#open' => TRUE,
'#attributes' => [
'id' => $da_id . '_sc_' . $client_number,
],
'aggr' => [],
'id' => [],
'config' => [
'#attributes' => [
'id' => $da_id . '_sc_' . $client_number . '_conf',
],
],
];
// Data aggregation settings.
// Note: merging forms using NestedArray::mergeDeep() would result in
// Ajax call errors when '#ajax' are used since callback functions could
// be duplicated.
$form['storage_clients'][$client_number]['aggr'] =
[
'#type' => 'fieldset',
'#title' => $this->t('Data aggregation settings'),
'#open' => TRUE,
'#attributes' => [
'id' => $da_id . '_sc_' . $client_number . '_aggr',
],
]
+ $this->buildStorageClientAggregationForm(
$form,
$form_state,
$client_number
);
try {
$form['storage_clients'][$client_number] =
$this->buildStorageClientSelectForm(
$form,
$form_state,
$client_number
)
+ $form['storage_clients'][$client_number];
$form['storage_clients'][$client_number] =
$this->buildStorageClientConfigForm(
$form,
$form_state,
$client_number
)
+ $form['storage_clients'][$client_number];
}
catch (PluginException $e) {
$form['storage_clients'][$client_number] = [
'#type' => 'item',
'#markup' => $this->t(
'WARNING: Failed to load a client storage plugin!'
),
];
$this->logger->error(
'Failed to load a client storage plugin: '
. $e
);
}
$form['storage_clients'][$client_number]['notes'] = [
'#type' => 'textarea',
'#title' => $this->t('Storage client notes'),
'#description' => $this->t('Administrative notes for this storage client maintenance.'),
'#default_value' => $storage_settings[$client_number]['notes'] ?? '',
];
if (1 < $storage_count) {
$form['storage_clients'][$client_number]['remove_storage'] = [
'#type' => 'submit',
'#value' => $this->t(
'Remove storage client @number',
[
'@number' => '#' . ($client_number + 1),
]
),
// Match this name with self::validateConfigurationForm().
'#name' => 'remst_' . $da_id . '_sc_' . $client_number,
'#ajax' => [
'callback' => [get_class($this), 'buildAjaxParentSubForm'],
'wrapper' => ($form['storage_clients']['#attributes']['id'] ??= uniqid('sc', TRUE)),
'method' => 'replaceWith',
'effect' => 'fade',
],
];
}
}
// Append button to add storage clients.
$form['storage_clients']['add_storage'] = [
'#type' => 'submit',
'#value' => $this->t('Add a storage'),
// Match this name with self::validateConfigurationForm().
'#name' => 'addst_' . $da_id,
'#ajax' => [
'callback' => [get_class($this), 'buildAjaxParentSubForm'],
'wrapper' => ($form['storage_clients']['#attributes']['id'] ??= uniqid('sc', TRUE)),
'method' => 'replaceWith',
'effect' => 'fade',
],
];
return $form;
}
/**
* Build a storage client aggregation form for a given client number.
*
* @param array $form
* An associative array containing the initial structure of the global form.
* @param \Drupal\Core\Form\FormStateInterface $form_state
* The current state of the form. Calling code should pass on a subform
* state created through
* \Drupal\Core\Form\SubformState::createForSubform().
* @param int $client_index
* Index of the storage client.
*
* @return array
* The global form structure.
*/
public function buildStorageClientAggregationForm(
array $form,
FormStateInterface $form_state,
int $client_index,
) :array {
$aggr_selector = ($form['#attributes']['id'] ??= uniqid('aggr', TRUE));
$aggr_config = $this->getConfiguration()['storage_clients'] ?? [];
// Data aggregation settings.
$storage_client_form = [
'groups' => [
'#type' => 'textfield',
'#title' => $this->t('Group prefix(es)'),
'#description' => $this->t('Optional: you can set a group prefix for filtering or group aggregation. Multiple prefixes can be specified separated by semicolons (;).'),
'#default_value' => implode(';', $aggr_config[$client_index]['aggr']['groups'] ?? []),
'#attributes' => [
'id' => $aggr_selector . '_groups',
],
],
'group_prefix_strip' => [
'#type' => 'checkbox',
'#title' => $this->t('Virtual group prefix'),
'#description' => $this->t('"Virtual group prefix" means the group prefix is not present on the storage client side but only visible from the External Entity side.'),
'#default_value' => $aggr_config[$client_index]['aggr']['group_prefix_strip'] ?? FALSE,
'#states' => [
'invisible' => [
':input[id="' . $aggr_selector . '_groups"]' => ['value' => ''],
],
],
],
];
// The use of override or other identifier field can not work with
// the first storage client as no field has been loaded for it.
if ($client_index) {
$storage_client_form['merge_join'] = [
'#type' => 'textfield',
'#title' => $this->t(
'Previous storage clients field name that provides the join (identifier) value for this storage client (join field name)'
),
'#description' => $this->t(
'Leave empty to use the default identifier. Otherwise, any (text/numeric) field provided by previous storage clients can be used here as identifier value for this storage client as well as a JSONPath expression.'
),
'#default_value' => $aggr_config[$client_index]['aggr']['merge_join'] ?? NULL,
];
// Build id field mapper config.
$storage_client_form['id_mapper'] = $this->buildClientPropertyMappingForm(
$form,
$form_state,
$client_index,
['storage_clients', $client_index, 'aggr', 'id_mapper'],
$aggr_selector . '_pmid',
$aggr_config[$client_index]['aggr']['id_mapper'] ?? [],
);
$storage_client_form['id_mapper']['#title'] =
$this->t('Storage source field name to use as identifier');
$storage_client_form['id_mapper']['#description'] =
$this->t(
'This is used to identify each source entry (despite the join value). It must be set if the source field uses a different identifier field name than previous source or if a different join field name is used (see after). If no join field name is set, this field will be used as the join field.'
);
// Join field property mapper.
$storage_client_form['join_mapper'] = $this->buildClientPropertyMappingForm(
$form,
$form_state,
$client_index,
['storage_clients', $client_index, 'aggr', 'join_mapper'],
$aggr_selector . '_pmjn',
$aggr_config[$client_index]['aggr']['join_mapper'] ?? [],
);
$storage_client_form['join_mapper']['#title'] =
$this->t('Storage source field name to use to join data');
$storage_client_form['join_mapper']['#description'] =
$this->t('Source field name that will hold the value used to join data with the given (or default) previous storage identifier field.');
$storage_client_form['join_mapper']['#states'] = [
'invisible' => [
':input[data-mapper-id="' . $aggr_selector . '_pmid_id"]' => ['value' => ''],
],
];
$storage_client_form['merge'] = [
'#type' => 'select',
'#title' => $this->t('How to merge client data'),
'#options' => [
'keep' => $this->t('Keep existing field values (no override)'),
'over' => $this->t('Override previous field values'),
'ovem' => $this->t('Override previous field values if empty'),
'sub' => $this->t('As a sub-object'),
],
'#description' => $this->t(
'If set, entity field values provided by this storage client will override existing values with the same field name provided by previous storage clients (except for the identifier and join fields).'
),
'#default_value' => $aggr_config[$client_index]['aggr']['merge'] ?? 'keep',
'#attributes' => [
'id' => $aggr_selector . '_merge',
],
];
$storage_client_form['merge_as_member'] = [
'#type' => 'textfield',
'#title' => $this->t('Field name to store sub-object(s) in an array'),
'#description' => $this->t(
'Machine name of the field that will hold the array of sub-objects keyed by their identifiers.'
),
'#default_value' => $aggr_config[$client_index]['aggr']['merge_as_member'] ?? '',
'#states' => [
'visible' => [
':input[id="' . $aggr_selector . '_merge"]' => ['value' => 'sub'],
],
],
];
}
else {
// First storage client.
$storage_client_form['id_mapper'] = $this->buildClientPropertyMappingForm(
$form,
$form_state,
$client_index,
['storage_clients', $client_index, 'aggr', 'id_mapper'],
$aggr_selector . '_pm',
$aggr_config[$client_index]['aggr']['id_mapper'] ?? [],
);
$storage_client_form['id_mapper']['#title'] =
$this->t('Storage source field name to use as identifier');
}
$storage_client_form['mode'] = [
'#type' => 'select',
'#title' => $this->t(
'Mode'
),
'#options' => [
static::STORAGE_CLIENT_MODE_DISABLED => $this->t('Disabled'),
static::STORAGE_CLIENT_MODE_READONLY => $this->t('Read only (only used for loading)'),
static::STORAGE_CLIENT_MODE_WRITEONLY => $this->t('Write only (only used to save data)'),
static::STORAGE_CLIENT_MODE_READWRITE => $this->t('Both reading and writing'),
],
'#default_value' =>
$aggr_config[$client_index]['aggr']['mode']
?? static::STORAGE_CLIENT_MODE_READWRITE,
];
return $storage_client_form;
}
/**
* Build a property mapper form for a given client number.
*
* @param array $form
* An associative array containing the initial structure of the global form.
* @param \Drupal\Core\Form\FormStateInterface $form_state
* The current state of the form. Calling code should pass on a subform
* state created through
* \Drupal\Core\Form\SubformState::createForSubform().
* @param int $client_index
* Index of the storage client.
* @param array $parents
* The form element parents array.
* @param string $property_wrapper_id
* HTML id to use for the property mapper subform.
* @param array $mapper_config
* The current property mapper configuration.
*
* @return array
* The global form structure.
*/
public function buildClientPropertyMappingForm(
array $form,
FormStateInterface $form_state,
int $client_index,
array $parents,
string $property_wrapper_id,
array $mapper_config,
) :array {
$property_mapper_form = [
'#type' => 'fieldset',
'#title' => $this->t('Storage source field name'),
'#attributes' => [
'id' => $property_wrapper_id,
],
];
$property_mappers = $this
->propertyMapperManager
->getCompatiblePropertyMappers('string', 'value');
// Filter the list of property mappers to only 'direct', 'field' and
// 'jsonpath'.
$property_mappers = [
'direct' => $property_mappers['direct'],
'jsonpath' => $property_mappers['jsonpath'],
'simple' => $property_mappers['simple'],
];
$current_property_mapper_id =
$form_state->getValue(
[...$parents, 'id']
)
?? $mapper_config['id']
?? '';
foreach ($property_mappers as $property_mapper_id => $definition) {
$config = [
ExternalEntityTypeInterface::XNTT_TYPE_PROP => $this->externalEntityType,
'field_name' => 'id',
'property_name' => 'value',
'main_property' => TRUE,
'required_field' => FALSE,
'debug_level' => $this->getDebugLevel(),
];
if ($current_property_mapper_id == $property_mapper_id) {
$config = NestedArray::mergeDeep(
$config,
($mapper_config['config'] ?? [])
);
}
$property_mapper = $this->propertyMapperManager->createInstance($property_mapper_id, $config);
$property_mapper_options[$property_mapper_id] = $property_mapper->getLabel();
if ($current_property_mapper_id == $property_mapper_id) {
$current_property_mapper = $property_mapper;
}
}
$property_mapper_form['id'] = [
'#type' => 'select',
'#title' => $this->t('Mapping type:'),
'#default_value' => $current_property_mapper_id,
'#options' => $property_mapper_options,
'#sort_options' => TRUE,
'#empty_value' => '',
'#empty_option' => $this->t('Not mapped'),
'#required' => FALSE,
'#wrapper_attributes' => ['class' => ['xntt-inline']],
'#attributes' => [
'class' => ['xntt-field'],
'autocomplete' => 'off',
'data-mapper-id' => $property_wrapper_id . '_id',
],
'#label_attributes' => ['class' => ['xntt-label']],
'#ajax' => [
'callback' => [get_class($this), 'buildAjaxParentSubForm'],
'wrapper' => $property_wrapper_id,
'method' => 'replaceWith',
'effect' => 'fade',
],
];
$property_mapper_form['config'] = [
'#type' => 'container',
// If #parents is not set here, sub-element names will not follow the
// tree structure.
'#parents' => [
...($form['#parents'] ?? []),
...$parents,
'config',
],
'#attributes' => [
'id' => $property_wrapper_id . '_config',
],
];
$current_property_mapper_form_state = XnttSubformState::createForSubform(
[...$parents, 'config'],
$form,
$form_state
);
if (!empty($current_property_mapper)) {
$property_mapper_form['config'] =
$current_property_mapper->buildConfigurationForm(
$property_mapper_form['config'],
$current_property_mapper_form_state
);
// Remove data processor support.
unset($property_mapper_form['config']['data_processors']);
}
return $property_mapper_form;
}
/**
* Builds the storage client selection configuration.
*
* @param array $form
* The current form array.
* @param \Drupal\Core\Form\FormStateInterface $form_state
* The current form state.
* @param int $client_number
* Storage client number for this external entity.
*
* @return array
* The storage client selection form for the given client number.
*
* @throws \Drupal\Component\Plugin\Exception\PluginException
* If a storage client plug-in cannot be loaded.
*/
public function buildStorageClientSelectForm(
array $form,
FormStateInterface $form_state,
int $client_number = 0,
) :array {
$storage_clients = $this->getAvailableStorageClients();
$storage_settings = $form_state->get('storage_settings');
$storage_client_options = [];
$allowed_plugins = [];
// Manage restrictions.
if (!empty($this->locks['lock_data_aggregator']['lock_storage_clients'][$client_number]['allow_plugins'])
|| !empty($this->locks['lock_data_aggregator']['lock_storage_clients']['*']['allow_plugins'])
) {
$allowed_plugins = $this->locks['lock_data_aggregator']['lock_storage_clients'][$client_number]['allow_plugins'];
}
$group_name_lookup = [
'rest' => '' . $this->t('Rest clients'),
'files' => '' . $this->t('File clients'),
'ql' => '' . $this->t('Query Language clients'),
'others' => '' . $this->t('Other clients'),
];
foreach (array_keys($group_name_lookup) as $group) {
foreach (($storage_clients[$group] ?? []) as $storage_client_id => $storage_client) {
// Manage plugin restrictions.
if ($allowed_plugins && empty($allowed_plugins[$storage_client_id])) {
continue 1;
}
$group_name = $group_name_lookup[$group];
$storage_client_options[$group_name][$storage_client_id] =
$storage_client->getLabel();
}
}
// Sort each category.
foreach ($storage_client_options as $group_name => $group) {
$storage_client_options[$group_name]['#sort_options'] = TRUE;
}
$id_form = [];
if ($storage_client_options) {
$id_form = [
'id' => [
'#type' => 'select',
'#title' => $this->t('Storage client'),
'#description' => (1 < count($storage_client_options))
? $this->t('Choose a storage client to use, then configure it below.')
: '',
'#options' => $storage_client_options,
// We enforce the empty options because of issue #3180011.
'#empty_option' => $this->t('- Select -'),
'#default_value' => ($storage_settings[$client_number]['id'] ?? '')
?: DataAggregatorBase::DEFAULT_STORAGE_CLIENT,
'#required' => TRUE,
'#attributes' => [
'data-client' => $client_number,
'autocomplete' => 'off',
],
'#ajax' => [
'callback' => [get_class($this), 'buildAjaxParentSubForm'],
'wrapper' => ($form['storage_clients'][$client_number]['#attributes']['id'] ??= uniqid('sc', TRUE)),
'method' => 'replaceWith',
'effect' => 'fade',
],
],
];
}
return $id_form;
}
/**
* Builds the storage client-specific configuration form.
*
* @param array $form
* The current form array.
* @param \Drupal\Core\Form\FormStateInterface $form_state
* The current form state.
* @param int $client_number
* Storage client number for this external entity.
*
* @return array
* The storage client configuration form for the given client number.
*/
public function buildStorageClientConfigForm(
array $form,
FormStateInterface $form_state,
int $client_number = 0,
) :array {
$storage_settings = $form_state->get('storage_settings');
$storage_client_id =
($storage_settings[$client_number]['id'] ?? '')
?: DataAggregatorBase::DEFAULT_STORAGE_CLIENT;
$config = [];
// Check if selected client id is compatible with saved config.
$storage_clients = $this->getAvailableStorageClients();
if (($storage_client_id === $storage_settings[$client_number]['id'])
|| (($storage_clients['#group'][$storage_client_id] ?? '##') === ($storage_clients['#group'][$storage_settings[$client_number]['id']] ?? '#'))
) {
$config = $storage_settings[$client_number]['config'];
}
$config += $this->getStorageClientDefaultConfiguration();
$storage_client = $this
->storageClientManager
->createInstance($storage_client_id, $config);
if ($storage_client && $storage_client instanceof PluginFormInterface) {
// Make sure we got a default form structure.
$form['storage_clients'][$client_number]['config'] ??= [
'#type' => 'container',
'#attributes' => [
'id' => ($form['#attributes']['id'] ??= uniqid('da', TRUE))
. '_sc_'
. $client_number
. '_conf',
],
];
// Attach the storage client plugin configuration form.
$storage_client_form_state = XnttSubformState::createForSubform(
['storage_clients', $client_number, 'config'],
$form,
$form_state
);
$form['storage_clients'][$client_number]['config'] =
$storage_client->buildConfigurationForm(
$form['storage_clients'][$client_number]['config'],
$storage_client_form_state
);
// Modify the storage client plugin configuration container element.
$form['storage_clients'][$client_number]['config']['#type'] =
'fieldset';
$form['storage_clients'][$client_number]['config']['#title'] =
$this->t(
'Configure %plugin storage client',
['%plugin' => $storage_client->getLabel()]
);
$form['storage_clients'][$client_number]['config']['#open'] =
TRUE;
}
// If no plugin form was provided, use a default empty container.
$form['storage_clients'][$client_number]['config'] += [
'#type' => 'container',
];
// Check for plugin restrictions.
if (
!empty(
$this->locks['lock_data_aggregator']['lock_storage_clients'][$client_number]['lock_config']
)
) {
$form['storage_clients'][$client_number]['config']['#disabled'] = TRUE;
$form['storage_clients'][$client_number]['config']['#attributes']['class'][] = 'xntt-disabled';
$form['storage_clients'][$client_number]['config']['#description'] = $this->t('This storage client configuration cannot be edited.');
}
if (
!empty(
$this->locks['lock_data_aggregator']['lock_storage_clients'][$client_number]['hide_config']
)
) {
$form['storage_clients'][$client_number]['#type'] = 'hidden';
}
return $form['storage_clients'][$client_number];
}
/**
* {@inheritdoc}
*/
public function validateConfigurationForm(array &$form, FormStateInterface $form_state) {
// Check for Ajax events.
if ($trigger = $form_state->getTriggeringElement()) {
$da_id = $form['#attributes']['id'] ?? '';
if ("addst_$da_id" == $trigger['#name']) {
$storage_count = $form_state->get('storage_count') + 1;
$form_state->set('storage_count', $storage_count);
$storage_settings = $form_state->get('storage_settings');
$storage_settings[] = [
'id' => '',
'config' => [],
'notes' => '',
];
$form_state->set('storage_settings', $storage_settings);
$form_state->setRebuild(TRUE);
}
elseif (preg_match("#^remst_\\Q$da_id\\E_sc_(\\d+)\$#", $trigger['#name'], $matches)) {
$client_number = $matches[1];
if (isset($client_number)) {
$storage_count = $form_state->get('storage_count') - 1;
$form_state->set('storage_count', $storage_count);
// Reorder storage client configs.
$storage_settings = $form_state->get('storage_settings');
// Note: here, $storage_count = count($storage_settings) - 1.
$user_input = $form_state->getUserInput();
for ($config_number = $client_number; $config_number < $storage_count; ++$config_number) {
$storage_settings[$config_number] = $storage_settings[$config_number + 1];
// Shift form values.
$form_state->setValueForElement(
['storage_clients', $config_number],
$form_state->getValue(
['storage_clients', $config_number + 1]
)
);
$user_input['storage_clients'][$config_number] = $user_input['storage_clients'][$config_number + 1];
}
unset($storage_settings[$storage_count]);
$form_state->set('storage_settings', $storage_settings);
unset($user_input['storage_clients'][$storage_count]);
$form_state->setUserInput($user_input);
$form_state->setRebuild(TRUE);
}
}
// Storage client selection change.
$upd_client_number = $trigger['#attributes']['data-client'] ?? NULL;
if (isset($upd_client_number)) {
$storage_settings = $form_state->get('storage_settings');
$client_id = $form_state->getValue(
['storage_clients', $upd_client_number, 'id']
);
$client_config = [];
// Try to get existing settings from config if same id.
if (!$this->externalEntityType->isNew()
&& ($this->getStorageClientId($upd_client_number) == $client_id)
) {
$client_config =
$this->getStorageClientConfig($upd_client_number);
}
$storage_settings[$upd_client_number] = [
'id' => $client_id,
'config' => $client_config,
'notes' => $form_state->getValue(
['storage_clients', $upd_client_number, 'notes'],
''
),
];
$form_state->set('storage_settings', $storage_settings);
if (preg_match(
'/^(storages_tab\[config\]\[storage_clients|language_settings\[overrides\]\[\w{2}\]\[storages\]\[config\]\[storage_clients)\]\[(\w+)\]\[id\]$/',
$trigger['#name'], $matches
)
) {
// Remove current user input on storage client config since form
// content has changed. Otherwise, if left as is, we would have
// strange behaviors such as new checkboxes being checked (because
// their keys are there) while they should not (because their
// associated values are empty). It usualy comes from hidden fields
// set with an empty/FALSE value that are turned into non-hidden
// fields such as checkboxes which are testing if their key exists in
// the user input to tell if they are checked.
$ui = $form_state->getUserInput();
$parents = array_merge(preg_split('/\]?\[/', $matches[1]), [$matches[2], 'config']);
NestedArray::unsetValue($ui, $parents);
$form_state->setUserInput($ui);
}
$form_state->setRebuild(TRUE);
}
}
$storage_count = $form_state->get('storage_count');
$group_prefixes = [];
for ($i = 0; $i < $storage_count; ++$i) {
$storage_client_id = $form_state->getValue(
['storage_clients', $i, 'id']
);
if (!empty($storage_client_id)) {
// Validate storage client settings.
$storage_client_config = $this->getStorageClientDefaultConfiguration();
$storage_client = $this->storageClientManager->createInstance(
$storage_client_id,
$storage_client_config
);
if ($storage_client instanceof PluginFormInterface) {
$storage_client_form_state = XnttSubformState::createForSubform(
['storage_clients', $i, 'config'],
$form,
$form_state
);
$storage_client->validateConfigurationForm(
$form['storage_clients'][$i]['config'],
$storage_client_form_state
);
$storage_client_config = $storage_client->getConfiguration();
}
else {
// Clear config.
$storage_client_config = [];
}
// Validate storage client aggregation settings.
$this->validateStorageClientAggregationForm(
$form,
$form_state,
$i,
$group_prefixes,
);
}
}
// If rebuild needed, ignore validation.
if ($form_state->isRebuilding()) {
$form_state->clearErrors();
}
}
/**
* Aggregation form validation for each storage client.
*
* @param array $form
* An associative array containing the initial structure of the global form.
* @param \Drupal\Core\Form\FormStateInterface $form_state
* The current state of the form. Calling code should pass on a subform
* state created through
* \Drupal\Core\Form\SubformState::createForSubform().
* @param int $client_index
* Index of the storage client.
* @param array &$group_prefixes
* An array containing group prefixes already met in previous storage client
* aggregation settings.
*/
public function validateStorageClientAggregationForm(
array $form,
FormStateInterface $form_state,
int $client_index,
array &$group_prefixes = [],
) {
$aggr_settings = $form_state->getValue(['storage_clients', $client_index, 'aggr'], []);
$aggr_settings['groups'] = array_filter(
preg_split('/\s*;\s*/', trim($aggr_settings['groups'] ?? '')),
'strlen'
);
$form_state->setValue(['storage_clients', $client_index, 'aggr'], $aggr_settings);
// Check prefixes...
// Make sure a given prefix does not appear for the first time in a set.
// @todo Check the case of empty '' prefixes and see if the following code
// behaves as expected.
$new_prefixes = [];
foreach ($aggr_settings['groups'] as $prefix) {
if (!array_key_exists($prefix, $group_prefixes)) {
$new_prefixes[$prefix] = $prefix;
}
}
if ((!empty($new_prefixes))
&& (count($new_prefixes) != count($aggr_settings['groups']))
) {
$form_state->setError(
$form['storage_clients'][$client_index]['aggr']['groups'],
$this->t(
'The group prefix(es) %prefix appear(s) for the first time in a set of group prefixes where at least one appeared before. However, when multiple group prefixes are specified, they should ALL appear only for the first time or have appeared at least once before (no mix between new and already defined prefixes).',
[
'%prefix' => implode(
', ',
array_diff_key($new_prefixes, $group_prefixes)
),
]
)
);
}
$group_prefixes += $new_prefixes;
// Make sure no prefix is part of another one.
foreach ($new_prefixes as $group_prefix1) {
if ('' == $group_prefix1) {
continue 1;
}
foreach ($group_prefixes as $group_prefix2) {
if ('' == $group_prefix2) {
continue 1;
}
if (($group_prefix1 != $group_prefix2)) {
if (strncmp($group_prefix2, $group_prefix1, strlen($group_prefix1)) === 0) {
$form_state->setError(
$form['storage_clients'][$client_index]['aggr']['groups'],
$this->t(
'In external entity storage client "Multiple storages", the group prefix %prefix1 is also a prefix for group prefix %prefix2. Prefixes must be strictly distinct.',
[
'%prefix1' => $group_prefix1,
'%prefix2' => $group_prefix2,
]
)
);
}
elseif (strncmp($group_prefix1, $group_prefix2, strlen($group_prefix2)) === 0) {
$form_state->setError(
$form['storage_clients'][$client_index]['aggr']['groups'],
$this->t(
'In external entity storage client "Multiple storages", the group prefix %prefix1 is also a prefix for group prefix %prefix2. Prefixes must be strictly distinct.',
[
'%prefix1' => $group_prefix2,
'%prefix2' => $group_prefix1,
]
)
);
}
}
}
}
if (!empty($aggr_settings['id_mapper']['id'])) {
$property_mapper_id = $aggr_settings['id_mapper']['id'];
$config = [
ExternalEntityTypeInterface::XNTT_TYPE_PROP => $this->externalEntityType,
'field_name' => 'id',
'property_name' => 'value',
'main_property' => TRUE,
'required_field' => FALSE,
'debug_level' => $this->getDebugLevel(),
];
$property_mapper = $this->propertyMapperManager->createInstance($property_mapper_id, $config);
if ($property_mapper instanceof PluginFormInterface) {
$property_mapper_form_state = XnttSubformState::createForSubform(
['storage_clients', $client_index, 'aggr', 'id_mapper', 'config'],
$form,
$form_state
);
$property_mapper->validateConfigurationForm($form['storage_clients'][$client_index]['aggr']['id_mapper']['config'], $property_mapper_form_state);
}
if (!empty($aggr_settings['join_mapper']['id'])) {
$property_mapper_id = $aggr_settings['join_mapper']['id'];
$config = [
ExternalEntityTypeInterface::XNTT_TYPE_PROP => $this->externalEntityType,
'field_name' => 'title',
'property_name' => 'value',
'main_property' => TRUE,
'required_field' => FALSE,
'debug_level' => $this->getDebugLevel(),
];
$property_mapper = $this->propertyMapperManager->createInstance($property_mapper_id, $config);
if ($property_mapper instanceof PluginFormInterface) {
$property_mapper_form_state = XnttSubformState::createForSubform(
['storage_clients', $client_index, 'aggr', 'join_mapper', 'config'],
$form,
$form_state
);
$property_mapper->validateConfigurationForm($form['storage_clients'][$client_index]['aggr']['join_mapper']['config'], $property_mapper_form_state);
}
}
}
}
/**
* {@inheritdoc}
*/
public function submitConfigurationForm(
array &$form,
FormStateInterface $form_state,
) {
$storage_count = $form_state->get('storage_count');
$storage_clients = [];
for ($i = 0; $i < $storage_count; ++$i) {
$storage_client_id = $form_state->getValue(
['storage_clients', $i, 'id']
);
if (!empty($storage_client_id)) {
// Submit new storage client settings.
$storage_client_config = $this->getStorageClientDefaultConfiguration();
$storage_client = $this->storageClientManager->createInstance(
$storage_client_id,
$storage_client_config
);
if ($storage_client instanceof PluginFormInterface) {
$storage_client_form_state = XnttSubformState::createForSubform(
['storage_clients', $i, 'config'],
$form,
$form_state
);
$storage_client->submitConfigurationForm(
$form['storage_clients'][$i]['config'],
$storage_client_form_state
);
$storage_client_config = $storage_client->getConfiguration();
}
else {
// Clear config.
$storage_client_config = [];
}
$aggr_settings = $form_state->getValue(['storage_clients', $i, 'aggr']);
if (!empty($aggr_settings['id_mapper']['id'])) {
$property_mapper_id = $aggr_settings['id_mapper']['id'];
$config = [
ExternalEntityTypeInterface::XNTT_TYPE_PROP => $this->externalEntityType,
'field_name' => 'id',
'property_name' => 'value',
'main_property' => TRUE,
'required_field' => FALSE,
'debug_level' => $this->getDebugLevel(),
];
$property_mapper = $this->propertyMapperManager->createInstance(
$property_mapper_id,
$config
);
if ($property_mapper instanceof PluginFormInterface) {
$property_mapper_form_state = XnttSubformState::createForSubform(
['storage_clients', $i, 'aggr', 'id_mapper', 'config'],
$form,
$form_state
);
$property_mapper->submitConfigurationForm($form['storage_clients'][$i]['aggr']['id_mapper']['config'], $property_mapper_form_state);
$aggr_settings['id_mapper'] = [
'id' => $property_mapper_id,
'config' => $property_mapper->getConfiguration(),
];
}
if (!empty($aggr_settings['join_mapper']['id'])) {
$property_mapper_id = $aggr_settings['join_mapper']['id'];
$config = [
ExternalEntityTypeInterface::XNTT_TYPE_PROP => $this->externalEntityType,
'field_name' => 'title',
'property_name' => 'value',
'main_property' => TRUE,
'required_field' => FALSE,
'debug_level' => $this->getDebugLevel(),
];
$property_mapper = $this->propertyMapperManager->createInstance(
$property_mapper_id,
$config
);
if ($property_mapper instanceof PluginFormInterface) {
$property_mapper_form_state = XnttSubformState::createForSubform(
['storage_clients', $i, 'aggr', 'join_mapper', 'config'],
$form,
$form_state
);
$property_mapper->submitConfigurationForm($form['storage_clients'][$i]['aggr']['join_mapper']['config'], $property_mapper_form_state);
$aggr_settings['join_mapper'] = [
'id' => $property_mapper_id,
'config' => $property_mapper->getConfiguration(),
];
}
}
}
$storage_clients[$i] = [
'id' => $storage_client_id,
'config' => $storage_client_config,
'aggr' => $aggr_settings,
];
}
}
// Cleanup form values (ie. only use the specified ones).
$form_state->setValues([
'storage_clients' => $storage_clients,
]);
if ($this instanceof ConfigurableInterface) {
$this->setConfiguration($form_state->getValues());
}
}
}
