data_pipelines-1.x-dev/data_pipelines.install
data_pipelines.install
<?php
/**
* @file
* The data pipelines install file.
*/
use Drupal\Core\Config\Entity\ConfigEntityType;
use Drupal\Core\Entity\EntityDeleteForm;
use Drupal\Core\Entity\FieldableEntityInterface;
use Drupal\Core\Entity\Routing\AdminHtmlRouteProvider;
use Drupal\Core\Field\BaseFieldDefinition;
use Drupal\Core\Field\FieldStorageDefinitionInterface;
use Drupal\Core\StringTranslation\TranslatableMarkup;
use Drupal\data_pipelines\Entity\DatasetInterface;
use Drupal\data_pipelines\Entity\Destination;
use Drupal\data_pipelines\EntityHandlers\DatasetDestinationListBuilder;
use Drupal\data_pipelines\Form\DestinationForm;
/**
* Implements hook_update_N().
*
* Update the plugin Ids for sources to reflect the use of a deriver.
*/
function data_pipelines_update_8001() {
$database = \Drupal::database();
$database->query("UPDATE data_pipelines SET source = REPLACE(source, '_', ':')")->execute();
$database->query("UPDATE data_pipelines SET source = REPLACE(source, 'string', 'text')")->execute();
}
/**
* Implements hook_update_N().
*
* Rebuild the field bundle definitions.
*/
function data_pipelines_update_8002() {
$entity_bundle_info = \Drupal::service('entity_type.bundle.info');
$entity_type_manager = \Drupal::service('entity_type.manager');
$entity_field_manager = \Drupal::service('entity_field.manager');
$map = [];
foreach ($entity_type_manager->getDefinitions() as $entity_type_id => $entity_type) {
if (!$entity_type->entityClassImplements(FieldableEntityInterface::class)) {
continue;
}
foreach ($entity_bundle_info->getBundleInfo($entity_type_id) as $bundle => $bundle_info) {
foreach ($entity_field_manager->getFieldDefinitions($entity_type_id, $bundle) as $field_name => $field_definition) {
$map[$entity_type_id][$field_name]['type'] = $field_definition->getType();
$map[$entity_type_id][$field_name]['bundles'][] = $bundle;
}
}
}
$persistent_map = \Drupal::keyValue('entity.definitions.bundle_field_map');
$persistent_map->setMultiple($map);
}
/**
* Implements hook_update_N().
*
* Install the field storage definition for json_path_to_data.
*/
function data_pipelines_update_8003() {
$entity_definition_update_manager = \Drupal::entityDefinitionUpdateManager();
$field_storage_definition = BaseFieldDefinition::create('string')
->setLabel(new TranslatableMarkup('Path to data'))
->setDescription(new TranslatableMarkup('This must be provided using a <a href="@link" target="_blank">JSON path</a> query.', ['@link' => 'https://github.com/SoftCreatR/JSONPath']))
->setSetting('max_length', 255)
->setDisplayOptions('form', [
'type' => 'string_textfield',
]);
$entity_definition_update_manager->installFieldStorageDefinition('json_path_to_data', 'data_pipelines', 'data_pipelines', $field_storage_definition);
}
/**
* Implements hook_update_N().
*
* Update the database field schema to reflect changes when renaming the
* text resource source field name.
*/
function data_pipelines_update_8004() {
$database = \Drupal::database();
/** @var \Drupal\Core\Entity\EntityFieldManagerInterface $entity_field_manager */
$entity_field_manager = \Drupal::service('entity_field.manager');
$field_storage_definitions = $entity_field_manager->getFieldStorageDefinitions('data_pipelines');
$csv_text_field_storage_schema = $field_storage_definitions['csv_text']->getSchema()['columns']['value'];
$database->schema()->changeField('data_pipelines', 'csv_string', 'csv_text', $csv_text_field_storage_schema);
$json_text_field_storage_schema = $field_storage_definitions['json_text']->getSchema()['columns']['value'];
$database->schema()->changeField('data_pipelines', 'json_string', 'json_text', $json_text_field_storage_schema);
$installed_storage_schema = \Drupal::keyValue('entity.storage_schema.sql');
$csv_text_installed_storage_schema = [
'data_pipelines' => [
'fields' => [
'csv_text' => $installed_storage_schema->get('data_pipelines.field_schema_data.csv_string')['data_pipelines']['fields']['csv_string'],
],
],
];
$installed_storage_schema->delete('data_pipelines.field_schema_data.csv_string');
$installed_storage_schema->set('data_pipelines.field_schema_data.csv_text', $csv_text_installed_storage_schema);
$json_text_installed_storage_schema = [
'data_pipelines' => [
'fields' => [
'json_text' => $installed_storage_schema->get('data_pipelines.field_schema_data.json_string')['data_pipelines']['fields']['json_string'],
],
],
];
$installed_storage_schema->delete('data_pipelines.field_schema_data.json_string');
$installed_storage_schema->set('data_pipelines.field_schema_data.json_text', $json_text_installed_storage_schema);
/** @var \Drupal\Core\Entity\EntityLastInstalledSchemaRepositoryInterface $entity_last_installed_schema */
$entity_last_installed_schema = \Drupal::service('entity.last_installed_schema.repository');
$entity_last_installed_schema->setLastInstalledFieldStorageDefinitions('data_pipelines', $field_storage_definitions);
}
/**
* An update hook to set the index name prefix for data pipelines settings.
*
* Implements hook_update_N().
*/
function data_pipelines_update_8005() {
// Deprecated.
}
/**
* Install the field storage definition for machine_name.
*/
function data_pipelines_update_8006() {
$entity_definition_update_manager = \Drupal::entityDefinitionUpdateManager();
$field_storage_definition = BaseFieldDefinition::create('string')
->setLabel(t('Machine Name'))
->setDescription(new TranslatableMarkup('The machine name used for indexing.'))
->setSetting('max_length', 128)
->setRequired(TRUE)
->addConstraint('UniqueField')
->addPropertyConstraints('value', ['Regex' => ['pattern' => '/^[a-z0-9_]+$/']]);
$entity_definition_update_manager->installFieldStorageDefinition('machine_name', 'data_pipelines', 'data_pipelines', $field_storage_definition);
}
/**
* Install the field storage definition for delimiter.
*/
function data_pipelines_update_8007() {
$entity_definition_update_manager = \Drupal::entityDefinitionUpdateManager();
$field_storage_definition = BaseFieldDefinition::create('list_string')
->setLabel(t('Delimiter'))
->setDescription(new TranslatableMarkup('Specific delimiter.'))
->setDefaultValue('comma')
->setSetting('allowed_values', [
'comma' => 'Comma (,)',
'semicolon' => 'Semicolon (;)',
'tab' => 'Tab (\t)',
'pipe' => 'Pipe (|)',
'colon' => 'Colon (:)',
])
->setDisplayOptions('form', [
'type' => 'options_select',
])
->setRequired(TRUE);
$entity_definition_update_manager->installFieldStorageDefinition('csv_delimiter', 'data_pipelines', 'data_pipelines', $field_storage_definition);
}
/**
* Update 'allowed_values' for data_pipelines 'status' field.
*/
function data_pipelines_update_8008() {
$entity_definition_update_manager = \Drupal::entityDefinitionUpdateManager();
$field_storage_definition = $entity_definition_update_manager->getFieldStorageDefinition('status', 'data_pipelines');
$field_storage_definition->setSetting('allowed_values', [
DatasetInterface::STATUS_PENDING_VALIDATION => new TranslatableMarkup('Pending validation'),
DatasetInterface::STATUS_PENDING_PROCESSING => new TranslatableMarkup('Pending processing'),
DatasetInterface::STATUS_PROCESSED => new TranslatableMarkup('Processed'),
]);
$entity_definition_update_manager->updateFieldStorageDefinition($field_storage_definition);
}
/**
* Install 'Data Pipeline Destination' entity.
*/
function data_pipelines_update_8009() {
$entity_definition_update_manager = \Drupal::entityDefinitionUpdateManager();
$dataset_destination = new ConfigEntityType([
'id' => 'dataset_destination',
'label' => new TranslatableMarkup('Data Pipeline Destination'),
'label_collection' => new TranslatableMarkup('Data Pipeline Destinations'),
'label_singular' => new TranslatableMarkup('destination'),
'label_plural' => new TranslatableMarkup('destinations'),
'label_count' => [
'singular' => '@count destination',
'plural' => '@count destinations',
],
'entity_keys' => [
'id' => 'id',
'label' => 'label',
],
'handlers' => [
'list_builder' => DatasetDestinationListBuilder::class,
'route_provider' => [
'html' => AdminHtmlRouteProvider::class,
],
'form' => [
'default' => DestinationForm::class,
'add' => DestinationForm::class,
'edit' => DestinationForm::class,
'delete' => EntityDeleteForm::class,
],
],
'admin_permission' => 'administer dataset destinations',
'links' => [
'add-form' => '/admin/config/content/dataset_destinations/add',
'edit-form' => '/admin/config/content/dataset_destinations/manage/{dataset_destination}',
'delete-form' => '/admin/config/content/dataset_destinations/manage/{dataset_destination}/delete',
'collection' => '/admin/config/content/dataset_destinations',
],
'config_export' => [
'id',
'label',
'destination',
'destinationSettings',
],
'class' => Destination::class,
'provider' => 'data_pipelines',
]);
$entity_definition_update_manager->installEntityType($dataset_destination);
}
/**
* Update the database field schema to add destination.
*/
function data_pipelines_update_8010() {
$entity_definition_update_manager = \Drupal::entityDefinitionUpdateManager();
$field_storage_definition = BaseFieldDefinition::create('entity_reference')
->setLabel(t('Destination'))
->setDescription(t('The output destination.'))
->setRequired(TRUE)
->setCardinality(FieldStorageDefinitionInterface::CARDINALITY_UNLIMITED)
->setSetting('target_type', 'dataset_destination')
->setDisplayOptions('form', [
'type' => 'options_select',
]);
$entity_definition_update_manager->installFieldStorageDefinition('destinations', 'data_pipelines', 'data_pipelines', $field_storage_definition);
}
/**
* Update to using the data_pipelines_elasticsearch.
*/
function data_pipelines_update_8011() {
// Update to using the elasticsearch submodule.
\Drupal::service('module_installer')->install(['data_pipelines_elasticsearch']);
}
/**
* Update the queue names.
*/
function data_pipelines_update_8012() {
$database = \Drupal::database();
$database->query("UPDATE queue SET name = REPLACE(name, 'data_pipelines_index:', 'data_pipelines_process:') WHERE name LIKE 'data_pipelines_index:%'")->execute();
}
/**
* Remove the deprecated config.
*/
function data_pipelines_update_8013() {
$data_pipelines_settings = \Drupal::configFactory()->getEditable('data_pipelines.settings');
$data_pipelines_settings->delete();
}
/**
* Install fields for batch size.
*/
function data_pipelines_update_8014(): void {
$entity_definition_update_manager = \Drupal::entityDefinitionUpdateManager();
$field_storage_definition = BaseFieldDefinition::create('list_integer')
->setLabel(t('Batch size'))
->setDescription(new TranslatableMarkup('The number of items to process in each batch operation. Datasets with only limited data in each row can select a large value here to increase performance. Datasets with a lot of data in each row should select a small value here to prevent out of memory issues.'))
->setSetting('allowed_values', [
10 => '10',
100 => '100',
1000 => '1,000',
10000 => '10,000',
100000 => '100,000',
])
->setRequired(TRUE)
->setInitialValue(1000)
->setDefaultValue(1000)
->setDisplayConfigurable('form', TRUE)
->setDisplayOptions('form', [
'type' => 'options_select',
'label' => 'above',
'weight' => 10,
]);
$entity_definition_update_manager->installFieldStorageDefinition('batch_size', 'data_pipelines', 'data_pipelines', $field_storage_definition);
}
/**
* Install fields for invalid handling.
*/
function data_pipelines_update_8015(): void {
$entity_definition_update_manager = \Drupal::entityDefinitionUpdateManager();
$field_storage_definition = BaseFieldDefinition::create('list_string')
->setLabel(t('Invalid values handling'))
->setDescription(new TranslatableMarkup('Select the approach to be used for invalid values that were previously valid.'))
->setSetting('allowed_values', [
DatasetInterface::INVALID_RETAIN => new TranslatableMarkup('Retain records that were previously valid'),
DatasetInterface::INVALID_REMOVE => new TranslatableMarkup('Remove records that are now invalid'),
])
->setRequired(TRUE)
->setInitialValue(DatasetInterface::INVALID_RETAIN)
->setDefaultValue(DatasetInterface::INVALID_RETAIN)
->setDisplayConfigurable('form', TRUE)
->setDisplayOptions('form', [
'type' => 'options_select',
'label' => 'above',
'weight' => 10,
]);
$entity_definition_update_manager->installFieldStorageDefinition('invalid_values', 'data_pipelines', 'data_pipelines', $field_storage_definition);
}
/**
* Update fields for invalid handling.
*/
function data_pipelines_update_8016(): void {
$entity_definition_update_manager = \Drupal::entityDefinitionUpdateManager();
$field_storage_definition = \Drupal::entityDefinitionUpdateManager()->getFieldStorageDefinition('invalid_values', 'data_pipelines');
$field_storage_definition
->setSetting('allowed_values', [
DatasetInterface::INVALID_RETAIN => new TranslatableMarkup('Retain records that were previously valid'),
DatasetInterface::INVALID_REMOVE => new TranslatableMarkup('Remove records that are now invalid and outdated'),
]);
$entity_definition_update_manager->updateFieldStorageDefinition($field_storage_definition);
}
/**
* Install published field.
*/
function data_pipelines_update_8017(): void {
$entity_definition_update_manager = \Drupal::entityDefinitionUpdateManager();
// Update the entity keys.
$entity_type = $entity_definition_update_manager->getEntityType('data_pipelines');
$keys = $entity_type->getKeys();
$keys['published'] = 'published';
$entity_type->set('entity_keys', $keys);
$entity_definition_update_manager->updateEntityType($entity_type);
// Update the field storage.
$field_storage_definition = BaseFieldDefinition::create('boolean')
->setLabel(new TranslatableMarkup('Published'))
->setDescription(new TranslatableMarkup('Unpublishing the pipeline will remove all the destination data.'))
->setRevisionable(FALSE)
->setTranslatable(FALSE)
->setDefaultValue(TRUE)
->setInitialValue(TRUE)
->setDisplayOptions('view', [
'type' => 'boolean',
'format' => 'yes-no',
'weight' => -4,
])
->setDisplayOptions('form', [
'type' => 'options_select',
'weight' => 50,
]);
$entity_definition_update_manager->installFieldStorageDefinition('published', 'data_pipelines', 'data_pipelines', $field_storage_definition);
}
