l10n_server-2.x-dev/l10n_packager/src/L10nPackager.php
l10n_packager/src/L10nPackager.php
<?php
declare(strict_types=1);
namespace Drupal\l10n_packager;
use Drupal\Component\Datetime\TimeInterface;
use Drupal\Component\FileSecurity\FileSecurity;
use Drupal\Component\Render\FormattableMarkup;
use Drupal\Component\Utility\Timer;
use Drupal\Core\Cache\CacheBackendInterface;
use Drupal\Core\Config\ConfigFactoryInterface;
use Drupal\Core\Database\Connection;
use Drupal\Core\Datetime\DateFormatterInterface;
use Drupal\Core\Entity\EntityTypeManagerInterface;
use Drupal\Core\Extension\ModuleHandler;
use Drupal\Core\File\FileSystemInterface;
use Drupal\Core\File\FileUrlGeneratorInterface;
use Drupal\Core\Language\LanguageInterface;
use Drupal\Core\Language\LanguageManagerInterface;
use Drupal\Core\Logger\LoggerChannelInterface;
use Drupal\Core\Url;
use Drupal\file\Entity\File;
use Drupal\file\FileInterface;
use Drupal\l10n_packager\Entity\L10nPackagerFileInterface;
use Drupal\l10n_packager\Entity\L10nPackagerRelease;
use Drupal\l10n_packager\Entity\L10nPackagerReleaseInterface;
use Drupal\l10n_server\Entity\L10nServerProjectInterface;
use Drupal\l10n_server\Entity\L10nServerReleaseInterface;
/**
* Service description.
*/
class L10nPackager {
/**
* Cache expiration time for the l10n_packager cache.
*/
private const CACHE_EXPIRATION = 3600;
/**
* Release packager status: do not repackage anymore.
*/
const DISABLED = 0;
/**
* Release packager status: keep repackaging.
*/
const ACTIVE = 1;
/**
* Release packager status: error.
*/
const ERROR = 2;
/**
* Default path structure for generated files.
*/
const FILEPATH = '%core/%project/%project-%release.%language.po';
/**
* Packager API version.
*/
const API_VERSION = '1.1';
/**
* The database connection.
*
* @var \Drupal\Core\Database\Connection
*/
protected Connection $database;
/**
* Packager settings.
*
* @var \Drupal\Core\Config\ImmutableConfig
*/
protected $packagerSettings;
/**
* Community settings.
*
* @var \Drupal\Core\Config\ImmutableConfig
*/
protected $communitySettings;
/**
* Entity type manager service.
*
* @var \Drupal\Core\Entity\EntityTypeManagerInterface
*/
protected $entityTypeManager;
/**
* The file system service.
*
* @var \Drupal\Core\File\FileSystemInterface
*/
protected FileSystemInterface $fileSystem;
/**
* Logger service.
*
* @var \Drupal\Core\Logger\LoggerChannelInterface
*/
protected $logger;
/**
* Time (datetime.time) service.
*
* @var \Drupal\Component\Datetime\TimeInterface
*/
protected $time;
/**
* Date formatter service.
*
* @var \Drupal\Core\Datetime\DateFormatterInterface
*/
protected $dateFormatter;
/**
* File URL generator service.
*
* @var \Drupal\Core\File\FileUrlGeneratorInterface
*/
protected $fileUrlGenerator;
/**
* L10n exporter service.
*
* @var \Drupal\l10n_packager\L10nExporter
*/
protected $exporter;
/**
* Module handler.
*
* @var \Drupal\Core\Extension\ModuleHandler
*/
protected $moduleHandler;
/**
* Language manager.
*
* @var \Drupal\Core\Language\LanguageManagerInterface
*/
protected $languageManager;
/**
* Constructs a L10nPackager object.
*
* @param \Drupal\Core\Database\Connection $database
* The database connection.
* @param \Drupal\Core\Config\ConfigFactoryInterface $config_factory
* The config manager.
* @param \Drupal\Core\Entity\EntityTypeManagerInterface $entity_type_manager
* Entity type manager service.
* @param \Drupal\Core\File\FileSystemInterface $file_system
* The file system service.
* @param \Drupal\Core\Logger\LoggerChannelInterface $logger
* The logger service.
* @param \Drupal\Component\Datetime\TimeInterface $time
* Time (datetime.time) service.
* @param \Drupal\Core\Datetime\DateFormatterInterface $date_formatter
* Date formatter service.
* @param \Drupal\Core\File\FileUrlGeneratorInterface $file_url_generator
* File URL generator service.
* @param \Drupal\l10n_packager\L10nExporter $exporter
* L10n exporter service.
* @param \Drupal\Core\Extension\ModuleHandler $module_handler
* Module handler service.
* @param \Drupal\Core\Language\LanguageManagerInterface $language_manager
* Language manager service.
* @param \Drupal\Core\Cache\CacheBackendInterface $cacheBackend
* The cache backend.
*/
public function __construct(
Connection $database,
ConfigFactoryInterface $config_factory,
EntityTypeManagerInterface $entity_type_manager,
FileSystemInterface $file_system,
LoggerChannelInterface $logger,
TimeInterface $time,
DateFormatterInterface $date_formatter,
FileUrlGeneratorInterface $file_url_generator,
L10nExporter $exporter,
ModuleHandler $module_handler,
LanguageManagerInterface $language_manager,
private readonly CacheBackendInterface $cacheBackend,
) {
$this->database = $database;
$this->packagerSettings = $config_factory->get('l10n_packager.settings');
$this->communitySettings = $config_factory->get('l10n_community.settings');
$this->entityTypeManager = $entity_type_manager;
$this->fileSystem = $file_system;
$this->logger = $logger;
$this->time = $time;
$this->dateFormatter = $date_formatter;
$this->fileUrlGenerator = $file_url_generator;
$this->exporter = $exporter;
$this->moduleHandler = $module_handler;
$this->languageManager = $language_manager;
}
/**
* Get files for a release, indexed by language.
*
* @param int $rid
* A L10nServerRelease id.
*
* @return \Drupal\l10n_packager\Entity\L10nPackagerFileInterface[]
* An associative array, or an empty array if there is no result set.
*/
public function getFiles($rid): array {
$files = $this->entityTypeManager
->getStorage('l10n_packager_file')
->loadByProperties(['rid' => $rid]);
$files_per_language = [];
foreach ($files as $file) {
$files_per_language[$file->language->value] = $file;
}
return $files_per_language;
}
/**
* Get timestamp of the last updated string for a release, for each language.
*
* @param int $rid
* A L10nServerRelease id.
*
* @return array
* A keyed query result array.
*/
public function translationLastUpdated($rid) {
$query = $this->database
->select('l10n_server_translation', 't');
$query
->innerJoin('l10n_server_line', 'l', 't.sid = l.sid');
$query
->addExpression('MAX(t.changed)', 'latest_time');
$query
->fields('t', ['language']);
$query
->condition('t.status', 1)
->condition('t.suggestion', 0)
->condition('l.rid', $rid);
$query
->groupBy('t.language');
return $query->execute()
->fetchAllKeyed();
}
/**
* Get release name.
*
* @param int $rid
* A l10n_server_release id.
*
* @return string
* A normalized release name.
*/
public function releaseName($rid) {
if ($release = $this->entityTypeManager->getStorage('l10n_server_release')->load($rid)) {
return $release->getProject()->getUri() . '-' . $release->getTitle();
}
else {
return '';
}
}
/**
* Build target filepath from release object based on the set pattern.
*
* @param \Drupal\l10n_server\Entity\L10nServerReleaseInterface $release
* A l10n_server_release entity.
* @param \Drupal\Core\Language\LanguageInterface|null $language
* A language object.
* @param string|null $pattern
* A pattern string.
*
* @return string
* The filepath string.
*/
public function getFilepath(L10nServerReleaseInterface $release, ?LanguageInterface $language = NULL, ?string $pattern = NULL) {
$project = $release->getProject();
$replace = [
'%project' => $project->getUri(),
'%release' => $release->getVersion(),
'%core' => $release->getCoreFromVersion(),
'%version' => $release->getVersion(),
'%branch' => $release->getBranchFromVersion(),
'%extra' => !empty($extra) ? '-' . $extra : '',
'%language' => isset($language) ? $language->getId() : '',
];
if (!isset($pattern)) {
$pattern = $this->packagerSettings->get('filepath');
}
return strtr($pattern, $replace);
}
/**
* Create a symlink to the latest version of this locale for the project.
*
* The symlink name has the pattern [project]-[branch].[langcode].po and will
* be placed is the same directory as the translation file.
*
* @param string $uri
* A stream wrapper URI or a filepath.
* @param \Drupal\l10n_packager\Entity\L10nPackagerReleaseInterface $release
* An object containing the file's release information.
* @param \Drupal\Core\Language\LanguageInterface $language
* A language object.
*
* @return bool
* Returns TRUE if a symlink was created.
*/
public function createLatestSymlink(string $uri, L10nPackagerReleaseInterface $release, LanguageInterface $language): bool {
// If there is a minor version number, remove it. “Branch” is only
// '{major}.x' or '{compatibility}-{major}.x'. So a new dev branch can fall
// back to the latest translation for the major branch. For example, 9.1.x,
// when there are no tagged 9.1.* releases, can get the 9.0.0-beta1
// translations.
$abbreviated_release = $this->entityTypeManager
->getStorage('l10n_server_release')
->load($release->id());
$target = $this->fileSystem->realpath($uri);
$latest_file = dirname($target) . '/' . $this->getFilepath($abbreviated_release, $language, '%project-%branch.%language.po');
if (file_exists($latest_file)) {
unlink($latest_file);
$latest_file_object = new \stdClass();
$latest_file_object->uri = $this->packagerSettings->get('directory') . '/' . $this->getFilepath($abbreviated_release, $language, '%core/%project/%project-%branch.%language.po');
// Allow modules to react to the symlink, such as purge a CDN.
$this->moduleHandler->invokeAll('l10n_packager_done', [$latest_file_object]);
}
return symlink(basename($target), $latest_file);
}
/**
* Generate a new file for a given release or update an existing one.
*
* @param \Drupal\l10n_packager\Entity\L10nPackagerReleaseInterface $packager_release
* Release object with uri and rid properties.
* @param \Drupal\Core\Language\LanguageInterface $language
* Language object.
* @param \Drupal\l10n_packager\Entity\L10nPackagerFileInterface|null $packager_file
* Release file object to be updated.
* @param int|null $timestamp
* Timestamp to mark the files, for it to be consistent across tables.
*
* @return \Drupal\file\Entity\File|false
* Drupal file object or FALSE on error.
*
* @throws \Drupal\Core\TypedData\Exception\MissingDataException
* @throws \Drupal\Core\Entity\EntityStorageException
*/
public function package(L10nPackagerReleaseInterface $packager_release, LanguageInterface $language, ?L10nPackagerFileInterface $packager_file = NULL, ?int $timestamp = NULL): bool|File {
$release = $this->entityTypeManager
->getStorage('l10n_server_release')
->load($packager_release->id());
$timestamp = $timestamp ?: $this->time->getRequestTime();
$variables = [
'%release' => $this->releaseName($packager_release->id()),
'%language' => $language->getId(),
];
if (!$packager_file) {
$packager_file = $this->entityTypeManager
->getStorage('l10n_packager_file')
->create([
'rid' => $packager_release->id(),
'language' => $language->getId(),
]);
}
// Generate PO file. Always export in compact form.
$export_result = $this->exporter->export($release->getProject()->getUri(), $release->id(), $language, FALSE, TRUE);
if (!empty($export_result) && is_array($export_result)) {
// If we got an array back from the export build, tear that into pieces.
[$mime_type, $export_name, $serve_name, $sid_count] = $export_result;
// Get the destination file path.
$file_path = $this->getFilepath($release, $language);
// Build the full path and make sure the directory exits.
$file_path = $this->createPath($file_path);
// Now build the Drupal file object or update the old one.
if (!$packager_file->fid->isEmpty()) {
$file = $packager_file->fid->entity;
$this->fileSystem->delete($file->getFileUri());
}
else {
$file = $this->entityTypeManager
->getStorage('file')
->create(['created' => $timestamp]);
}
// Check / update / create all file data.
$file
->set('status', FileInterface::STATUS_PERMANENT)
->set('changed', $timestamp)
->set('filename', basename($file_path))
->set('filemime', $mime_type)
->set('uri', $file_path);
$this->fileSystem->move($export_name, $file->getFileUri(), FileSystemInterface::EXISTS_REPLACE);
$file
->set('filesize', filesize($this->fileSystem->realpath($file->getFileUri())));
$file->save();
// Create actual symlink to the latest release.
$this->createLatestSymlink($file_path, $packager_release, $language);
$packager_file
->set('rid', $packager_release->id())
->set('language', $language->getId())
->set('fid', $file->id())
->set('sid_count', $sid_count)
->set('checked', $timestamp)
->save();
$this->moduleHandler->invokeAll('l10n_packager_done', [$file]);
return $file;
}
else {
$this->logger->error('Failed packaging release %release for language %language.', $variables);
return FALSE;
}
}
/**
* Check releases that need repackaging.
*
* @param string $project
* The project to scan.
*
* @return array
* Number of projets checked, number of files, elapsed time.
*/
public function checkUpdates(string $project = '') : array {
$count_check = $count_files = $time = 0;
$updates = [];
if ($interval = $this->packagerSettings->get('update')) {
Timer::start('l10n_packager');
$timestamp = $this->time->getRequestTime() - $interval;
$file_limit = $this->packagerSettings->get('file_limit');
$count_files = $count_check = 0;
// Go for it: check releases for repackaging. We need project_uri for
// later.
$query = $this->database->select('l10n_server_release', 'r');
$query->innerJoin('l10n_server_project', 'p', 'r.pid = p.pid');
$query->leftJoin('l10n_packager_release', 'pr', 'pr.rid = r.rid');
$query->addField('r', 'rid');
$query->addField('pr', 'status');
if ($project) {
$query->condition('p.uri', $project);
}
else {
$query->condition($query->orConditionGroup()
->isNull('pr.status')
->condition($query->andConditionGroup()
->condition('pr.status', L10nPackagerRelease::ACTIVE)
->condition($query->orConditionGroup()
->condition('pr.checked', $timestamp, '<')
->condition('pr.changed', $timestamp, '<')
)
)
);
}
$query->range(0, $this->packagerSettings->get('release_limit'));
$query->orderBy('pr.checked');
$results = $query->execute();
$storage = $this->entityTypeManager->getStorage('l10n_packager_release');
while ((!$file_limit || $file_limit > $count_files) && ($result = $results->fetchObject())) {
$release = is_null($result->status) ? $storage->create(['rid' => $result->rid]) : $storage->load($result->rid);
$updates = $this->check($release, FALSE, $file_limit - $count_files, NULL, TRUE);
$count_files += count($updates);
$count_check++;
}
$time = Timer::stop('l10n_packager')['time'];
$this->logger->notice('@ms ms for %checked releases/%repack files.', [
'%checked' => $count_check,
'%repack' => $count_files,
'@ms' => $time,
]);
}
return [$count_check, $count_files, $time];
}
/**
* Check release translations and repackage if needed.
*
* For each release we store packaging data in {l10n_packager_release}
* - 'checked' is the last time all languages for this release were checked.
* - 'updated' is the last time a file was updated for this release.
*
* We don't update the 'checked' field until we've checked all the languages
* for this release, so we can keep track of releases and files and package a
* few languages on every cron.
*
* @param \Drupal\l10n_packager\Entity\L10nPackagerReleaseInterface $packager_release
* Release object.
* @param bool $force
* Force repackage even when strings not updated.
* @param int $limit
* Maximum number of files to create.
* @param \Drupal\Core\Language\LanguageInterface|null $language
* Optional language object to check only this one.
* @param bool $cron
* In a cron run, a release may be packaged partially, for some languages.
*
* @return array
* An associative array of File entities indexed by langcode.
*
* @throws \Drupal\Core\TypedData\Exception\MissingDataException
*/
public function check(L10nPackagerReleaseInterface $packager_release, bool $force = FALSE, int $limit = 0, ?LanguageInterface $language = NULL, bool $cron = FALSE) {
$check_languages = isset($language) ? [$language->getId() => $language] : $this->languageManager->getLanguages();
$updated = [];
// We get update time before creating files so the release checked time
// is <= file timestamp.
$timestamp = $this->time->getRequestTime();
$files = $this->getFiles($packager_release->id());
$last_updated = $this->translationLastUpdated($packager_release->id());
// Get only the languages we have translations for, that need updating.
$languages = [];
foreach ($check_languages as $langcode => $language) {
if (!empty($last_updated[$langcode]) && ($force || empty($files[$langcode]) || ($last_updated[$langcode] > $files[$langcode]->getCheckedTime()))) {
$languages[$langcode] = $language;
}
}
// For this special case we check we didn't stop before in the middle of a
// release. Otherwise it could stick on a release forever when forcing.
if ($force && $cron && $packager_release->getCheckedTime() < $packager_release->getChangedTime()) {
foreach ($files as $lang => $file) {
if (!$file->checked->isEmpty() && ($file->isChecked() > $packager_release->getCheckedTime())) {
unset($languages[$lang]);
}
}
}
// Repackage this release for the remaining language list.
while ((!$limit || $limit > count($updated)) && ($language = array_shift($languages))) {
$langcode = $language->getId();
// Warning: this may upload release data with or without file.
$existing = !empty($files[$langcode]) ? $files[$langcode] : NULL;
$updated[$langcode] = $this->package($packager_release, $language, $existing, $timestamp);
}
// Update the release data.
if (!count($languages)) {
// We only mark the release checked if there are no languages left.
$packager_release->set('checked', $timestamp);
}
if ($updated) {
$packager_release->set('changed', $timestamp);
}
$packager_release->save();
return $updated;
}
/**
* Ensure that directories on the $path exist in our packager directory.
*
* @param string $path
* A path string.
*
* @return string
* A path string.
*/
public function createPath(string $path) {
$directory = dirname($path);
$basepath = $currentpath = $this->packagerSettings->get('directory');
$finalpath = $basepath . '/' . $directory;
$parts = explode('/', $directory);
$htaccess_path = $this->fileSystem->realpath($basepath) . '/.htaccess';
if (!is_dir($finalpath)) {
$this->fileSystem->prepareDirectory($finalpath, FileSystemInterface::CREATE_DIRECTORY);
}
if (!file_exists($htaccess_path)) {
$htaccess_lines = "\n\n<FilesMatch \"\.(po)$\">\n\tForceType \"text/plain; charset=utf-8\"\n\tAllow from ALL\n</FilesMatch>\n";
FileSecurity::writeHtaccess($basepath, FALSE);
$this->fileSystem->chmod($htaccess_path, 0744);
file_put_contents($htaccess_path, $htaccess_lines, FILE_APPEND);
$this->fileSystem->chmod($htaccess_path, 0444);
}
return $basepath . '/' . $path;
}
/**
* Load and return the highlighted project if set and found.
*/
public function getHighlightedProject() {
if ($highlight_project = $this->communitySettings->get('highlighted_project')) {
$storage = $this->entityTypeManager->getStorage('l10n_server_project');
$entity_ids = $storage
->getQuery()
->accessCheck(TRUE)
->condition('title', $highlight_project)
->execute();
$project = $storage->load(reset($entity_ids));
if ($project) {
return $project;
}
}
return NULL;
}
/**
* Build download table rows for the language view pages.
*
* @param \Drupal\l10n_server\Entity\L10nServerProjectInterface $project
* A l10n_server_project entity.
* @param \Drupal\Core\Language\LanguageInterface $language
* A language object.
* @param bool $summary
* Whether to do a summary.
* @param array $headers
* An array of table headers.
*
* @return array|array[]
* A renderable array.
*/
public function getProjectDownloads(L10nServerProjectInterface $project, LanguageInterface $language, bool $summary = FALSE, array $headers = []): array {
$summary = ($summary) ? 'summary' : 'detail';
$cid = 'l10n_packager:' . $project->id() . ':' . $language->getId() . ':' . $summary;
if ($cache = $this->cacheBackend->get($cid)) {
return $cache->data;
}
$files = [];
$branches = [];
$result = $this->database->query("SELECT r.rid, r.version, pr.checked as release_checked, lf.checked, f.filename, f.filesize, f.uri, f.created as timestamp FROM {l10n_server_release} r INNER JOIN {l10n_packager_release} pr ON r.rid = pr.rid INNER JOIN {l10n_packager_file} lf ON r.rid = lf.rid INNER JOIN {file_managed} f ON lf.fid = f.fid WHERE r.pid = :pid AND lf.language = :language;", [
':pid' => $project->id(),
':language' => $language->getId(),
]);
foreach ($result as $item) {
// phpcs:ignore
// @todo: Fix bad, bad harcoding of Drupal versioning schemes.
$branch = preg_replace(($project->getUri() == 'drupal' ? '!^(\d\.)(.+)$!' : '!^(\d\.x-\d\.)(.+)$!'), '\1', $item->version) . 'x';
$branches[$branch] = TRUE;
$files[$branch][$item->rid] = $item;
}
if (empty($branches)) {
return [
[
$project->label(),
[
'data' => t('Not available for download yet.'),
'colspan' => 4,
],
],
];
}
ksort($branches);
$rows = [];
$summary_list = [];
$uri_class = str_replace('_', '-', $project->getUri());
foreach (array_keys($branches) as $branch) {
// Grab the latest item and compute its up-to-date stats.
krsort($files[$branch]);
$latest_item = array_shift($files[$branch]);
$up_to_date = max($latest_item->checked, $latest_item->release_checked);
$rows[] = [
'class' => ['l10n-packager-detail l10n-packager-detail-' . $uri_class],
'data' => [
[
'data' => new FormattableMarkup('<a href=":link">@name</a>', [
':link' => Url::fromUri('internal:/translate/projects/' . $project->id())->toString(),
'@name' => $project->label(),
]),
],
$latest_item->version,
[
'data' => new FormattableMarkup('<a href=":link">@name <span class="filesize">(@filesize)</span></a>', [
':link' => $this->getDownloadUrl($project, $branch, $latest_item),
'@name' => t('Download'),
'@filesize' => format_size($latest_item->filesize),
]),
],
$this->dateFormatter->format($latest_item->timestamp, 'custom', 'Y-M-d'),
$this->dateFormatter->format($up_to_date, 'custom', 'Y-M-d'),
],
];
$summary_list[] = $this->getDownloadUrl($project, $branch, $latest_item);
}
if (count($summary_list) && $summary) {
$summary_row = [
'class' => ['l10n-packager-summary'],
'id' => 'l10n-packager-summary-' . $uri_class,
'data' => [
[
'data' => [
'#type' => 'link',
'#title' => '► ' . $project->label(),
'#href' => 'translate/projects/' . $project->getUri(),
'#options' => [
'attributes' => ['class' => ['expand']],
'html' => TRUE,
],
],
],
[
'data' => implode(', ', $summary_list),
'colspan' => 4,
],
],
];
$headers = [
'class' => ['l10n-packager-detail l10n-packager-detail-' . $uri_class],
'data' => $headers,
];
array_unshift($rows, $headers);
array_unshift($rows, $summary_row);
}
$this->cacheBackend->set($cid, $rows, $this->time->getRequestTime() + self::CACHE_EXPIRATION);
return $rows;
}
/**
* Generate a download URL for a PO file.
*
* @param \Drupal\l10n_server\Entity\L10nServerProjectInterface $project
* A l10n_server_project entity.
* @param string $branch
* A branch string.
* @param object $file
* A file object.
*
* @return string
* A URL string for download.
*/
public function getDownloadUrl(L10nServerProjectInterface $project, string $branch, $file) {
$download_url = '';
$update_url = $this->packagerSettings->get('update_url');
if ($update_url) {
if (!empty($file->rid)) {
$release = $this->entityTypeManager->getStorage('l10n_server_release')->load($file->rid);
if ($release) {
$download_url = $update_url . '/' . $release->getCoreFromVersion() . '/' . $project->getUri() . '/' . $file->filename;
}
}
}
else {
$download_url = $this->fileUrlGenerator->generateAbsoluteString($file->uri);
}
return $download_url;
}
}
