pax-1.0.0-rc1/src/ShardingFileStorage.php
src/ShardingFileStorage.php
<?php
namespace Drupal\pax;
use Drupal\Component\FileCache\FileCacheFactory;
use Drupal\Component\FileSecurity\FileSecurity;
use Drupal\Component\Serialization\Exception\InvalidDataTypeException;
use Drupal\Component\Utility\NestedArray;
use Drupal\Core\Config\Entity\ConfigEntityType;
use Drupal\Core\Config\StorageException;
use Drupal\Core\Config\StorageInterface;
use Drupal\Core\Config\UnsupportedDataTypeConfigException;
use Drupal\Core\File\FileSystemInterface;
use Drupal\Core\Serialization\Yaml;
/**
* Defines a sharding file storage.
*
* It is a copy of Drupal\Core\Config\FileStorage so that it can be
* class_alias()'d early to fully replace it.
*
* For maintainers: changes to FileStorage are marked with a comment starting
* with ShardingFileStorage.
*/
class ShardingFileStorage implements StorageInterface {
const PAX_SHARDS = 'pax_shards';
/**
* @var array
*/
protected $shardList;
/**
* @var string
*/
protected $shardRegexp;
/**
* The storage collection.
*
* @var string
*/
protected $collection;
/**
* The filesystem path for configuration objects.
*
* @var string
*/
protected $directory = '';
/**
* The file cache object.
*
* @var \Drupal\Component\FileCache\FileCacheInterface
*/
protected $fileCache;
public static bool $disableSharding = FALSE;
/**
* Constructs a new FileStorage.
*
* @param string $directory
* A directory path to use for reading and writing of configuration files.
* @param string $collection
* (optional) The collection to store configuration in. Defaults to the
* default collection.
*/
public function __construct($directory, $collection = StorageInterface::DEFAULT_COLLECTION) {
$this->directory = $directory;
$this->collection = $collection;
// Use a NULL File Cache backend by default. This will ensure only the
// internal static caching of FileCache is used and thus avoids blowing up
// the APCu cache.
$this->fileCache = FileCacheFactory::get('config', ['cache_backend_class' => NULL]);
}
/**
* Returns the path to the configuration file.
*
* @return string
* The path to the configuration file.
*/
public function getFilePath($name) {
return $this->getCollectionDirectory() . '/' . $name . '.' . static::getFileExtension();
}
/**
* Returns the file extension used by the file storage for all configuration
* files.
*
* @return string
* The file extension.
*/
public static function getFileExtension() {
return 'yml';
}
/**
* Check if the directory exists and create it if not.
*/
protected function ensureStorage() {
// ShardingFileStorage factors this method out.
$this->ensureDirectory($this->getCollectionDirectory());
return $this;
}
/**
* @param $dir
*
* @return mixed
*/
protected function ensureDirectory($dir) {
// ShardingFileStorage specific factoring out from ensureStorage().
$success = $this->getFileSystem()->prepareDirectory($dir, FileSystemInterface::CREATE_DIRECTORY | FileSystemInterface::MODIFY_PERMISSIONS);
// Only create .htaccess file in root directory.
if ($dir == $this->directory) {
$success = $success && FileSecurity::writeHtaccess($this->directory);
}
if (!$success) {
throw new StorageException('Failed to create config directory ' . $dir);
}
return $dir;
}
/**
* {@inheritdoc}
*/
public function exists($name) {
return file_exists($this->getFilePath($name));
}
/**
* {@inheritdoc}
*/
public function readMultiple(array $names) {
$list = [];
foreach ($names as $name) {
if ($data = $this->read($name)) {
$list[$name] = $data;
}
}
return $list;
}
/**
* {@inheritDoc}
*/
public function read($name) {
if (!$this->exists($name)) {
return FALSE;
}
$filepath = $this->getFilePath($name);
if ($data = $this->fileCache->get($filepath)) {
return $data;
}
// ShardingFileStorage factors out actual reading and adds sharding.
$data = $this->doRead($filepath, $name);
if ($shard_dir = $this->getShardDir($filepath)) {
$fs = $this->getFileSystem();
$ext = '.' .static::getFileExtension();
$dirs = scandir($shard_dir);
foreach ($dirs as $shard_key) {
if (!empty($data[$shard_key])) {
continue;
}
$dir = "$shard_dir/$shard_key";
if ($shard_key[0] !== '.' && is_dir($dir)) {
$shard_parents = explode('.', $shard_key);
$order_file = "$dir/.order$ext";
if (file_exists($order_file)) {
$order = $this->doRead($order_file, "$name.shard.$shard_key..order");
}
else {
$order = [];
}
$files = scandir($dir);
$shards = [];
foreach ($files as $file) {
$file = "$dir/$file";
if ($file !== $order_file && is_file($file)) {
$config_key = $fs->basename($file, $ext);
$shards[$config_key] = $this->doRead($file, "$name.shard.$shard_key.$config_key");
}
}
foreach ($order as $config_key) {
if (isset($shards[$config_key])) {
$this->doSet($data, $shard_parents, $config_key, $shards[$config_key]);
unset($shards[$config_key]);
}
}
foreach ($shards as $config_key => $shard) {
$this->doSet($data, $shard_parents, $config_key, $shards[$config_key]);
}
}
}
}
$this->fileCache->set($filepath, $data);
return $data;
}
/**
* Read a file and return decoded data.
*
* @param string $filepath
* The file to read
* @param string $name
* Name of the config object, only used for error message.
*
* @return array
*/
protected function doRead($filepath, $name) {
// ShardingFileStorage factored out actual reading from doRead().
$data = file_get_contents($filepath);
try {
$data = $this->decode($data);
}
catch (InvalidDataTypeException $e) {
throw new UnsupportedDataTypeConfigException('Invalid data type in config ' . $name . ', found in file' . $filepath . ' : ' . $e->getMessage());
}
return $data;
}
public function doSet(array &$data, array $shard_parents, $config_key, $shard) {
$shard_parents[] = $config_key;
NestedArray::setValue($data, $shard_parents, $shard);;
}
/**
* {@inheritdoc}
*/
public function write($name, array $data) {
// ShardingFileStorage moved almost all of this method into doWrite() and
// adds sharding.
$target = $this->getFilePath($name);
$original_data = $data;
$this->initShardInfo();
// Make sure there are no lingering shards -- even in the rare case where
// sharding information changes, $this->read() is unaware of such and just
// blindly reads all shards. Do not remove the directory itself though to
// save rmdir/mkdir calls on the shard directory.
$this->deleteShardDir($target, FALSE);
if ($this->shardRegexp && preg_match($this->shardRegexp, $name, $matches)) {
$ext = static::getFileExtension();
foreach (($this->shardList[$matches[1]] ?? []) as $shard_key) {
$shard_parents = explode('.', $shard_key);
$shard_data = NestedArray::getValue($data, $shard_parents, $key_exists);
if ($key_exists) {
if (!isset($shard_dir)) {
$shard_dir = $this->getShardDir($target, ['ensure' => TRUE]);
}
$dir = "$shard_dir/$shard_key";
$this->ensureDirectory($dir);
$this->recursiveDeleteDir($dir, FALSE);
// Preserve the array order.
$this->doWrite("$dir/.order.$ext", array_keys($shard_data), "$name.shard.$shard_key..order");
foreach ($shard_data as $config_key => $config_data) {
$this->doWrite("$dir/$config_key.$ext", $config_data, "$name.shard.$shard_key.$config_key");
}
NestedArray::setValue($data, $shard_parents, []);
}
}
}
$this->doWrite($target, $data, $name);
$this->fileCache->set($target, $original_data);
return TRUE;
}
/**
* @param string $target
* @param array $data
* @param string $name
*/
protected function doWrite($target, array $data, $name) {
// ShardingFileStorage copy of FileStorage::write() with minimal changes.
try {
$encoded_data = $this->encode($data);
}
catch (InvalidDataTypeException $e) {
throw new StorageException("Invalid data type in config $name: {$e->getMessage()}");
}
$status = @file_put_contents($target, $encoded_data);
if ($status === FALSE) {
// Try to make sure the directory exists and try writing again.
$this->ensureStorage();
$status = @file_put_contents($target, $encoded_data);
}
if ($status === FALSE) {
throw new StorageException('Failed to write configuration file: ' . $target);
}
}
/**
* @param string $filepath
* @param array $options
* - ensure: Create the directory if it doesn't exist.
* - no_check: Return the directory name even if it doesn't exist.
*
* @return string
*/
protected function getShardDir($filepath, $options = []) {
// ShardingFileStorage helper method.
$fs = $this->getFileSystem();
$dir = $fs->dirname($filepath);
$filename = $fs->basename($filepath, '.' . static::getFileExtension());
$shard_dir = "$dir/$filename";
if (!empty($options['ensure'])) {
$this->ensureDirectory($shard_dir);
}
return (is_dir($shard_dir) ||!empty($options['no_check'])) ? $shard_dir : FALSE;
}
/**
* Initialize shard info.
*
* This calls the entity type manager so it must not be called on read() to
* avoid a mess during import.
*/
protected function initShardInfo() {
// ShardingFileStorage helper method.
if (!isset($this->shardList) && !static::$disableSharding) {
$this->shardList = [];
foreach ($this->getEntityTypeManager()->getDefinitions() as $entity_type) {
if ($entity_type instanceof ConfigEntityType && ($shards = $entity_type->get(self::PAX_SHARDS))) {
$this->shardList[$entity_type->getConfigPrefix()] = $shards;
}
}
if ($this->shardList) {
$this->shardRegexp = '/^(' . implode('|', array_keys($this->shardList)) . ')/';
}
}
}
/**
* {@inheritdoc}
*/
public function delete($name) {
if (!$this->exists($name)) {
return FALSE;
}
// ShardingFileStorage specific rewrite to remove the sharding dir.
$target = $this->getFilePath($name);
$this->fileCache->delete($target);
return $this->getFileSystem()->unlink($target) && $this->deleteShardDir($target);
}
protected function deleteShardDir($target, $rmdir = TRUE) {
if ($shardDir = $this->getShardDir($target)) {
return $this->recursiveDeleteDir($shardDir, $rmdir);
}
return TRUE;
}
protected function recursiveDeleteDir($dir, $rmdir) {
$fs = $this->getFileSystem();
$files = new \RecursiveIteratorIterator(
new \RecursiveDirectoryIterator($dir, \RecursiveDirectoryIterator::SKIP_DOTS),
\RecursiveIteratorIterator::CHILD_FIRST
);
/** @var \SplFileInfo $fileinfo */
foreach ($files as $fileinfo) {
$pathname = $fileinfo->getPathname();
if ($fileinfo->isFile()) {
$status = $fs->unlink($pathname);
if ($status === FALSE) {
return $status;
}
}
if ($fileinfo->isDir() && $rmdir) {
$status = $fs->rmdir($pathname);
if ($status === FALSE) {
return $status;
}
}
}
return TRUE;
}
/**
* {@inheritdoc}
*/
public function rename($name, $new_name) {
// ShardingFileStorage specific rewrite to remove the sharding dir.
$filePath = $this->getFilePath($name);
$newFilePath = $this->getFilePath($new_name);
$status = @rename($filePath, $newFilePath);
if ($status === FALSE) {
return FALSE;
}
$this->fileCache->delete($filePath);
$this->fileCache->delete($newFilePath);
if ($shardDir = $this->getShardDir($filePath)) {
$newShardDir = $this->getShardDir($newFilePath, ['no_check' => TRUE]);
$status = @rename($shardDir, $newShardDir);
if ($status === FALSE) {
return FALSE;
}
}
return TRUE;
}
/**
* {@inheritdoc}
*/
public function encode($data) {
return Yaml::encode($data);
}
/**
* {@inheritdoc}
*/
public function decode($raw) {
$data = Yaml::decode($raw);
// A simple string is valid YAML for any reason.
if (!is_array($data)) {
return FALSE;
}
return $data;
}
/**
* {@inheritdoc}
*/
public function listAll($prefix = '') {
$dir = $this->getCollectionDirectory();
if (!is_dir($dir)) {
return [];
}
$extension = '.' . static::getFileExtension();
// glob() directly calls into libc glob(), which is not aware of PHP stream
// wrappers. Same for \GlobIterator (which additionally requires an absolute
// realpath() on Windows).
// @see https://github.com/mikey179/vfsStream/issues/2
$files = scandir($dir);
$names = [];
$pattern = '/^' . preg_quote($prefix, '/') . '.*' . preg_quote($extension, '/') . '$/';
foreach ($files as $file) {
if ($file[0] !== '.' && preg_match($pattern, $file)) {
$names[] = basename($file, $extension);
}
}
return $names;
}
/**
* {@inheritdoc}
*/
public function deleteAll($prefix = '') {
$files = $this->listAll($prefix);
$success = !empty($files);
foreach ($files as $name) {
if (!$this->delete($name) && $success) {
$success = FALSE;
}
}
if ($success && $this->collection != StorageInterface::DEFAULT_COLLECTION) {
// Remove empty directories.
if (!(new \FilesystemIterator($this->getCollectionDirectory()))->valid()) {
$this->getFileSystem()->rmdir($this->getCollectionDirectory());
}
}
return $success;
}
/**
* {@inheritdoc}
*/
public function createCollection($collection) {
return new static(
$this->directory,
$collection
);
}
/**
* {@inheritdoc}
*/
public function getCollectionName() {
return $this->collection;
}
/**
* {@inheritdoc}
*/
public function getAllCollectionNames() {
if (!is_dir($this->directory)) {
return [];
}
$collections = $this->getAllCollectionNamesHelper($this->directory);
sort($collections);
return $collections;
}
/**
* Helper function for getAllCollectionNames().
*
* If the file storage has the following subdirectory structure:
* ./another_collection/one
* ./another_collection/two
* ./collection/sub/one
* ./collection/sub/two
* this function will return:
* @code
* array(
* 'another_collection.one',
* 'another_collection.two',
* 'collection.sub.one',
* 'collection.sub.two',
* );
* @endcode
*
* @param string $directory
* The directory to check for sub directories. This allows this
* function to be used recursively to discover all the collections in the
* storage. It is the responsibility of the caller to ensure the directory
* exists.
*
* @return array
* A list of collection names contained within the provided directory.
*/
protected function getAllCollectionNamesHelper($directory) {
$collections = [];
$pattern = '/\.' . preg_quote($this->getFileExtension(), '/') . '$/';
foreach (new \DirectoryIterator($directory) as $fileinfo) {
if ($fileinfo->isDir()) {
$collection = $fileinfo->getFilename();
// ShardingFileStorage specific change: skip directories with dots in
// them. The dots in collection names are changed to slash in this
// storage so if a directory has a dot in them, it can not be a
// collection. It can be . or .. or a sharding directory.
if (strpos($collection, '.') !== FALSE) {
continue;
}
// Recursively call getAllCollectionNamesHelper() to discover if there
// are subdirectories. Subdirectories represent a dotted collection
// name.
$sub_collections = $this->getAllCollectionNamesHelper($directory . '/' . $collection);
if (!empty($sub_collections)) {
// Build up the collection name by concatenating the subdirectory
// names with the current directory name.
foreach ($sub_collections as $sub_collection) {
$collections[] = $collection . '.' . $sub_collection;
}
}
// Check that the collection is valid by searching it for configuration
// objects. A directory without any configuration objects is not a valid
// collection.
// @see \Drupal\Core\Config\FileStorage::listAll()
foreach (scandir($directory . '/' . $collection) as $file) {
if ($file[0] !== '.' && preg_match($pattern, $file)) {
$collections[] = $collection;
break;
}
}
}
}
return $collections;
}
/**
* Gets the directory for the collection.
*
* @return string
* The directory for the collection.
*/
protected function getCollectionDirectory() {
if ($this->collection == StorageInterface::DEFAULT_COLLECTION) {
$dir = $this->directory;
}
else {
$dir = $this->directory . '/' . str_replace('.', '/', $this->collection);
}
return $dir;
}
/**
* Returns file system service.
*
* @return \Drupal\Core\File\FileSystemInterface
* The file system service.
*/
private function getFileSystem() {
return \Drupal::service('file_system');
}
/**
* Returns entity type manager service.
*
* @return \Drupal\Core\Entity\EntityTypeManagerInterface
* The entity type manager service.
*/
private function getEntityTypeManager() {
// ShardingFileStorage specific helper.
return \Drupal::service('entity_type.manager');
}
}
