lionbridge_translation_provider-8.x-2.4/tmgmt_contentapi/src/Services/JobUploadManagerService.php
tmgmt_contentapi/src/Services/JobUploadManagerService.php
<?php
namespace Drupal\tmgmt_contentapi\Services;
use Drupal\Core\Database\Connection;
use Drupal\Core\Logger\LoggerChannelFactoryInterface;
use Drupal\Core\Queue\QueueFactory;
use Drupal\Core\State\StateInterface;
use Drupal\Core\StringTranslation\StringTranslationTrait;
use Drupal\tmgmt\Entity\Job;
use Drupal\tmgmt\JobInterface;
use Drupal\tmgmt_contentapi\Services\CapiDataProcessor;
use Drupal\tmgmt_contentapi\Services\JobHelper;
use Drupal\tmgmt_contentapi\Services\QueueOperations;
use Drupal\tmgmt_contentapi\Swagger\Client\Api\JobApi;
use Symfony\Component\DependencyInjection\ContainerInterface;
/**
* Service for managing job upload operations and lifecycle.
*
* This service centralizes all upload management logic including:
* - Upload status tracking and completion checking
* - Upload timeout and retry limit management
* - Failed upload detection and recovery operations
* - Smart cleanup (partial vs complete) strategies
* - Automatic retry and requeue operations
* - Error message storage and retrieval
* - Upload scenario determination and optimization
*/
class JobUploadManagerService {
use StringTranslationTrait;
/**
* The database connection.
*
* @var \Drupal\Core\Database\Connection
*/
protected $database;
/**
* Logger service.
*
* @var \Drupal\Core\Logger\LoggerChannelFactoryInterface
*/
protected $logger;
/**
* The queue factory.
*
* @var \Drupal\Core\Queue\QueueFactory
*/
protected $queueFactory;
/**
* The state service.
*
* @var \Drupal\Core\State\StateInterface
*/
protected $state;
/**
* The capi data processor service.
*
* @var \Drupal\tmgmt_contentapi\Services\CapiDataProcessor
*/
protected $capiDataProcessor;
/**
* Job helper service.
*
* @var \Drupal\tmgmt_contentapi\Services\JobHelper
*/
protected $jobHelper;
/**
* Queue operations service.
*
* @var \Drupal\tmgmt_contentapi\Services\QueueOperations
*/
protected $queueOperations;
/**
* Job API for CAPI operations.
*
* @var \Drupal\tmgmt_contentapi\Swagger\Client\Api\JobApi
*/
protected $jobApi;
/**
* Constructor.
*
* @param \Drupal\Core\Database\Connection $database
* The database connection.
* @param \Drupal\Core\Logger\LoggerChannelFactoryInterface $logger
* The logger service.
* @param \Drupal\Core\Queue\QueueFactory $queueFactory
* The queue factory.
* @param \Drupal\Core\State\StateInterface $state
* The state service.
* @param \Drupal\tmgmt_contentapi\Services\CapiDataProcessor $capiDataProcessor
* The CAPI data processor service.
* @param \Drupal\tmgmt_contentapi\Services\JobHelper $jobHelper
* The job helper service.
* @param \Drupal\tmgmt_contentapi\Services\QueueOperations $queueOperations
* The queue operations service.
*/
public function __construct(
Connection $database,
LoggerChannelFactoryInterface $logger,
QueueFactory $queueFactory,
StateInterface $state,
CapiDataProcessor $capiDataProcessor,
JobHelper $jobHelper,
QueueOperations $queueOperations
) {
$this->database = $database;
$this->logger = $logger;
$this->queueFactory = $queueFactory;
$this->state = $state;
$this->capiDataProcessor = $capiDataProcessor;
$this->jobHelper = $jobHelper;
$this->queueOperations = $queueOperations;
$this->jobApi = new JobApi();
}
/**
* Factory method for service instantiation.
*
* @param \Symfony\Component\DependencyInjection\ContainerInterface $container
* The service container.
*
* @return static
* The instantiated service.
*/
public static function create(ContainerInterface $container) {
return new static(
$container->get('database'),
$container->get('logger.factory'),
$container->get('queue'),
$container->get('state'),
$container->get('tmgmt_contentapi.capi_data_processor'),
$container->get('tmgmt_contentapi.job_helper'),
$container->get('tmgmt_contentapi.queue_operations')
);
}
/**
* Check retry limits for a specific item.
*
* @param int $item_id
* The item ID.
* @param int $jobid
* The job ID.
* @param string $capi_job_id
* The CAPI job ID.
*
* @return string
* Status: CONTINUE or RETRY_EXCEEDED.
*/
public function checkUploadRetryLimits($item_id, $jobid, $capi_job_id): string {
// Load job only when needed (when $item_id === 'all')
if ($item_id === 'all') {
$job = Job::load($jobid);
$job_items = $job->getItems();
$item_id = array_key_first($job_items); // Convert 'all' to actual item ID
}
try {
$result = $this->database->select('tmgmt_capi_request_processor', 'p')
->fields('p', ['file_upload_status', 'file_upload_attempts', 'lastupdated'])
->condition('tjiid', $item_id)
->condition('tjid', $jobid)
->condition('statuscode', 'SENDING')
->execute()
->fetchAssoc();
if (!$result) {
return 'CONTINUE'; // No record found, proceed normally
}
$attempts = (int) $result['file_upload_attempts'];
$status = $result['file_upload_status'];
$last_updated = $result['lastupdated'];
// Check if max attempts exceeded
if ($attempts >= 3) {
$this->logger->get("TMGMT_CONTENTAPI_RETRY")->error(
'Upload retry limit exceeded for item @item_id in job @jobid after @attempts attempts',
['@item_id' => $item_id, '@jobid' => $jobid, '@attempts' => $attempts]
);
$this->updateFileUploadStatus($item_id, $jobid, 'FAILED', NULL, FALSE, 'Upload retry limit exceeded after ' . $attempts . ' attempts');
return 'RETRY_EXCEEDED';
}
return 'CONTINUE';
}
catch (\Exception $e) {
$this->logger->get("TMGMT_CONTENTAPI_RETRY")->error(
'Error checking retry status: @message',
['@message' => $e->getMessage()]
);
return 'CONTINUE';
}
}
/**
* ROBUSTNESS IMPROVEMENT: Update file upload status in database.
* Only applies to SENDING statuscode rows.
*
* @param string $item_id
* The item ID.
* @param int $jobid
* The job ID (local Drupal job ID).
* @param string $status
* Upload status: FILE_GENERATING, READY_FOR_UPLOAD, UPLOADING, UPLOADED, FAILED.
* @param mixed $capi_response
* The CAPI response object (optional).
* @param bool $increment_attempts
* Whether to increment the attempt counter.
* @param string $error_message
* Error message to store when status is FAILED (optional).
*/
public function updateFileUploadStatus($item_id, $jobid, $status, $capi_response = NULL, $increment_attempts = FALSE, $error_message = '') {
try {
$job = Job::load($jobid);
$job_items = $job->getItems();
$is_single_response = $this->isSingleResponseScenario($job);
// OPTIMIZATION: Get first item ID for single-response scenario storage
$first_item_id = array_key_first($job_items);
// COMPOSITE ITEM ID APPROACH: Handle both individual items and 'all' scenario
$item_ids_to_update = [];
if ($item_id == 'all') {
// For single-response scenarios, update ALL individual items with the same status
foreach ($job_items as $job_item_id => $value) {
$item_ids_to_update[] = $job_item_id;
}
$this->logger->get("TMGMT_CONTENTAPI_FILE_UPLOAD_STATUS")->info(
'Processing single-response scenario - updating @count individual items for job @jobid',
['@count' => count($item_ids_to_update), '@jobid' => $jobid]
);
} else {
// For separate-file scenarios, update only the specific item
$item_ids_to_update[] = $item_id;
}
$fields = [
'file_upload_status' => $status,
'lastupdated' => \Drupal::database()->query('SELECT UTC_TIMESTAMP()')->fetchField(),
];
// Store error message for failed uploads
if ($status === 'FAILED' && !empty($error_message)) {
$fields['errormessage'] = substr($error_message, 0, 255); // Truncate to fit varchar(255)
$fields['haserror'] = 1; // Set error flag
} elseif ($status === 'UPLOADED') {
// Clear error message and flag on successful upload
$fields['errormessage'] = '';
$fields['haserror'] = 0;
// Check if this is a retry success that needs propagation
if ($item_id !== 'all') {
$this->checkAndHandleRetrySuccess($item_id, $jobid, $job);
}
}
// PERFORMANCE OPTIMIZATION: Use bulk operations instead of individual loops
if ($is_single_response && count($item_ids_to_update) > 1) {
// BULK UPDATE for single-response jobs (dramatically faster for 500 items)
$bulk_fields = $fields;
// Handle attempt incrementing with bulk operation
if ($increment_attempts) {
$bulk_fields['file_upload_attempts'] = \Drupal::database()->query(
'SELECT COALESCE(MAX(file_upload_attempts), 0) + 1 FROM {tmgmt_capi_request_processor} WHERE tjid = :jobid AND statuscode = :statuscode',
[':jobid' => $jobid, ':statuscode' => 'SENDING']
)->fetchField();
}
// Store CAPI response only in first item for single-response scenario
if (!empty($capi_response)) {
// First update: Store response in first item only
$first_item_fields = $bulk_fields;
$first_item_fields['capi_response'] = serialize($capi_response);
$first_updated_rows = \Drupal::database()->update('tmgmt_capi_request_processor')
->fields($first_item_fields)
->condition('tjiid', $first_item_id)
->condition('tjid', $jobid)
->condition('statuscode', 'SENDING')
->execute();
// Second update: Bulk update remaining items without response
if (count($item_ids_to_update) > 1) {
$remaining_items = array_diff($item_ids_to_update, [$first_item_id]);
$remaining_updated_rows = \Drupal::database()->update('tmgmt_capi_request_processor')
->fields($bulk_fields)
->condition('tjiid', $remaining_items, 'IN')
->condition('tjid', $jobid)
->condition('statuscode', 'SENDING')
->execute();
$total_updated_rows = $first_updated_rows + $remaining_updated_rows;
} else {
$total_updated_rows = $first_updated_rows;
}
} else {
// No response to store: Single bulk update for all items
$total_updated_rows = \Drupal::database()->update('tmgmt_capi_request_processor')
->fields($bulk_fields)
->condition('tjiid', $item_ids_to_update, 'IN')
->condition('tjid', $jobid)
->condition('statuscode', 'SENDING')
->execute();
}
$this->logger->get("TMGMT_CONTENTAPI_FILE_UPLOAD_STATUS")->info(
'BULK UPDATE: Updated file upload status to @status for @count items in job @jobid (rows affected: @rows) @response_info',
[
'@status' => $status,
'@count' => count($item_ids_to_update),
'@jobid' => $jobid,
'@rows' => $total_updated_rows,
'@response_info' => !empty($capi_response) ? '[Response stored in first item]' : ''
]
);
} else {
// INDIVIDUAL UPDATES for separate-file jobs or single items
foreach ($item_ids_to_update as $current_item_id) {
// Handle attempt incrementing per individual item
if ($increment_attempts) {
$current_attempts = \Drupal::database()->query(
'SELECT COALESCE(file_upload_attempts, 0) FROM {tmgmt_capi_request_processor} WHERE tjiid = :item_id AND tjid = :jobid AND statuscode = :statuscode LIMIT 1',
[':item_id' => $current_item_id, ':jobid' => $jobid, ':statuscode' => 'SENDING']
)->fetchField();
$fields['file_upload_attempts'] = ($current_attempts ?: 0) + 1;
}
// Store CAPI response logic for separate-file jobs
$current_fields = $fields;
if (!empty($capi_response) && !$is_single_response) {
// Separate-file: Store response in each item (only for non-single-response scenarios)
$current_fields['capi_response'] = serialize($capi_response);
}
// Update file_upload_status only for SENDING statuscode rows
$updated_rows = \Drupal::database()->update('tmgmt_capi_request_processor')
->fields($current_fields)
->condition('tjiid', $current_item_id)
->condition('tjid', $jobid)
->condition('statuscode', 'SENDING')
->execute();
$this->logger->get("TMGMT_CONTENTAPI_FILE_UPLOAD_STATUS")->info(
'Updated file upload status to @status for item @item_id in job @jobid (rows affected: @rows) @response_stored',
[
'@status' => $status,
'@item_id' => $current_item_id,
'@jobid' => $jobid,
'@rows' => $updated_rows,
'@response_stored' => ($capi_response && isset($current_fields['capi_response'])) ? '[Response stored]' : ''
]
);
}
}
}
catch (\Exception $e) {
$this->logger->get("TMGMT_CONTENTAPI_FILE_UPLOAD_STATUS")->error(
'Failed to update file upload status: @message',
['@message' => $e->getMessage()]
);
}
}
/**
* Smart cleanup handler - determines and executes appropriate cleanup strategy.
*
* @param int $jobid
* The job ID.
* @param string $capi_job_id
* The CAPI job ID.
* @param string $allFilesPath
* All files path.
* @param string $zipPath
* Zip path.
* @param \Throwable $ex
* Exception object.
* @param array $specific_failed_items
* Optional specific failed item IDs.
* @param bool $force_complete_cleanup
* Force complete cleanup even if some items succeeded.
*
* @return array
* Cleanup result with strategy used and details.
*/
public function handleJobFailure(int $jobid, string $capi_job_id, string $allFilesPath, string $zipPath, \Throwable $ex, array $specific_failed_items = [], bool $force_complete_cleanup = FALSE): array {
// Analyze job status
$upload_status = $this->queueOperations->getJobUploadStatus($jobid, $capi_job_id);
$this->logger->get('TMGMT_CONTENTAPI_FAILURE_HANDLING')->info(
'Smart failure handling started for job: @jobids - Upload status: @uploaded uploaded, @failed failed, @total total. Error: @errormessage',
[
'@jobids' => $jobid . '_' . $capi_job_id,
'@uploaded' => $upload_status['uploaded'],
'@failed' => $upload_status['failed'],
'@total' => $upload_status['total'],
'@errormessage' => $ex->getMessage(),
]
);
// Determine cleanup strategy
$has_successful_uploads = $upload_status['uploaded'] > 0;
$complete_failure = $upload_status['failed'] == $upload_status['total'] || $upload_status['total'] == 0;
if ($force_complete_cleanup || $complete_failure || !$has_successful_uploads) {
return $this->performCompleteCleanup($jobid, $capi_job_id, $allFilesPath, $zipPath, $ex);
} else {
return $this->performPartialCleanup($jobid, $capi_job_id, $allFilesPath, $zipPath, $ex, $specific_failed_items);
}
}
/**
* Perform complete cleanup (enhanced with comprehensive file cleanup).
*/
public function performCompleteCleanup(int $jobid, string $capi_job_id, string $allFilesPath, string $zipPath, \Throwable $ex): array {
$this->logger->get('TMGMT_CONTENTAPI_FAILURE_HANDLING')->info(
'Performing COMPLETE cleanup for job @jobids - will reset job to unprocessed state',
['@jobids' => $jobid . '_' . $capi_job_id]
);
try {
$job = Job::load($jobid);
// Step 1: Cancel CAPI job
if (isset($capi_job_id)) {
$capiToken = \Drupal::service('tmgmt_contentapi.capi_details')->getCapiToken($job->getTranslator());
$this->jobApi->jobsJobIdDelete($capiToken, $capi_job_id);
// Mark records as failed instead of deleting - preserves failure tracking for retry
$this->markRecordsAsFailedForRetry($capi_job_id, $jobid, $ex);
$this->capiDataProcessor->deleteMessageRecords($jobid);
// Reset job state
$job->setState(JobInterface::STATE_UNPROCESSED);
$job->save();
}
// Step 2: Delete zip files only (directories handled by cleanupLioxDirectories)
$this->cleanupZipFiles($zipPath, $jobid, $capi_job_id);
// Step 3: Comprehensive cleanup for all LioxSentFiles and LioxRefFiles directories
$this->cleanupLioxDirectories($jobid, $capi_job_id, $job);
// Step 4: Delete all queue items
$this->cleanupQueueItems($jobid);
// Step 5: Clean up state variables only (no file operations)
$this->queueOperations->deleteStateVariable($capi_job_id . '_' . $jobid . '_transfer_files');
$msg = substr($ex->getMessage(), 0, 200);
\Drupal::messenger()->addMessage($job->label() . ' - Complete job reset due to failure: ' . $msg, 'error');
return [
'success' => TRUE,
'strategy' => 'COMPLETE_CLEANUP',
'message' => 'Complete job cleanup performed - job reset to unprocessed state',
'details' => ['file_cleanup' => 'directory-based', 'state_cleanup' => 'variables-only']
];
}
catch (\Exception $e) {
return [
'success' => FALSE,
'strategy' => 'COMPLETE_CLEANUP',
'message' => 'Complete cleanup failed: ' . $e->getMessage(),
'details' => []
];
}
}
/**
* Mark records as failed instead of deleting to preserve failure tracking for retry.
* Uses smart retry strategy: single-response scenarios only mark first item for retry.
*
* @param string $capi_job_id
* The CAPI job ID.
* @param int $jobid
* The local Drupal job ID.
* @param \Throwable $ex
* The exception that caused the failure.
*/
private function markRecordsAsFailedForRetry(string $capi_job_id, int $jobid, \Throwable $ex): void {
try {
$job = Job::load($jobid);
if (!$job) {
throw new \Exception("Job {$jobid} not found");
}
$is_single_response = $this->isSingleResponseScenario($job);
$failure_message = substr($ex->getMessage(), 0, 150); // Leave room for prefixes
$failure_time = \Drupal::database()->query('SELECT UTC_TIMESTAMP()')->fetchField();
if ($is_single_response) {
// SMART RETRY: For single-response scenarios, only mark first item for retry
// Get first item (the one with actual failure data)
$first_item = $this->database->select('tmgmt_capi_request_processor', 't')
->fields('t', ['tjiid'])
->condition('jobid', $capi_job_id)
->condition('tjid', $jobid)
->orderBy('tjiid', 'ASC')
->range(0, 1)
->execute()
->fetchField();
if ($first_item) {
// Mark ONLY first item for retry (preserves original capi_response)
$first_updated = $this->database->update('tmgmt_capi_request_processor')
->fields([
'statuscode' => 'COMPLETE_CLEANUP_FAILED',
'haserror' => 1,
'errormessage' => 'Single-response retry required: ' . $failure_message,
'file_upload_status' => 'FAILED',
'file_upload_attempts' => 0, // Reset attempts for retry
'lastupdated' => $failure_time,
// Preserve capi_response - contains actual failure details
])
->condition('tjiid', $first_item)
->condition('jobid', $capi_job_id)
->condition('tjid', $jobid)
->execute();
// Mark all OTHER items as waiting for first item retry
$others_updated = $this->database->update('tmgmt_capi_request_processor')
->fields([
'statuscode' => 'WAITING_FOR_RETRY',
'haserror' => 1,
'errormessage' => 'Waiting for first item retry: ' . $failure_message,
'file_upload_status' => 'WAITING',
'lastupdated' => $failure_time,
// Keep capi_response as NULL - these items depend on first item
])
->condition('tjid', $jobid)
->condition('jobid', $capi_job_id)
->condition('tjiid', $first_item, '!=')
->execute();
$this->logger->get('TMGMT_CONTENTAPI_FAILURE_HANDLING')->info(
'SMART RETRY: Single-response scenario - marked first item (@first_item) for retry, @others_count items waiting (job @job_id)',
[
'@first_item' => $first_item,
'@others_count' => $others_updated,
'@job_id' => $jobid
]
);
}
} else {
// SEPARATE FILES: Mark all items for individual retry (current approach)
$updated_count = $this->database->update('tmgmt_capi_request_processor')
->fields([
'statuscode' => 'COMPLETE_CLEANUP_FAILED',
'haserror' => 1,
'errormessage' => 'Separate files retry required: ' . $failure_message,
'file_upload_status' => 'FAILED',
'file_upload_attempts' => 0, // Reset attempts for retry
'lastupdated' => $failure_time,
// Preserve capi_response - each item has individual failure data
])
->condition('jobid', $capi_job_id)
->condition('tjid', $jobid)
->execute();
$this->logger->get('TMGMT_CONTENTAPI_FAILURE_HANDLING')->info(
'SEPARATE FILES: Marked @count items for individual retry (job @job_id)',
['@count' => $updated_count, '@job_id' => $jobid]
);
}
} catch (\Exception $e) {
$this->logger->get('TMGMT_CONTENTAPI_FAILURE_HANDLING')->error(
'Failed to mark records as failed for job @job_id: @error',
['@job_id' => $jobid, '@error' => $e->getMessage()]
);
// Fallback to original deletion if marking fails
$this->capiDataProcessor->deleteProcessorRecords($capi_job_id, $jobid);
}
}
/**
* Check if a successful upload is a retry that needs success propagation.
*
* @param int $item_id
* The item ID that succeeded.
* @param int $jobid
* The local Drupal job ID.
* @param \Drupal\tmgmt\JobInterface $job
* The job object.
*/
private function checkAndHandleRetrySuccess($item_id, $jobid, $job): void {
try {
// Get the item's current status to check if it was marked for retry
$item_record = $this->database->select('tmgmt_capi_request_processor', 't')
->fields('t', ['statuscode', 'jobid'])
->condition('tjiid', $item_id)
->condition('tjid', $jobid)
->execute()
->fetchAssoc();
if ($item_record && $item_record['statuscode'] === 'COMPLETE_CLEANUP_FAILED') {
// This was a retry of the first item in a single-response scenario
$is_single_response = $this->isSingleResponseScenario($job);
if ($is_single_response) {
// Propagate success to all waiting items
$this->handleSingleResponseSuccess($item_id, $jobid, $item_record['jobid']);
}
}
} catch (\Exception $e) {
$this->logger->get('TMGMT_CONTENTAPI_UPLOAD_SUCCESS')->error(
'Error checking retry success for item @item_id: @error',
['@item_id' => $item_id, '@error' => $e->getMessage()]
);
}
}
/**
* Handle success propagation for single-response retry scenarios.
* Called when first item upload succeeds and needs to update all waiting items.
*
* @param int $first_item_id
* The first item ID that succeeded.
* @param int $jobid
* The local Drupal job ID.
* @param string $capi_job_id
* The CAPI job ID.
*
* @return array
* Result of the propagation operation.
*/
public function handleSingleResponseSuccess($first_item_id, $jobid, $capi_job_id): array {
try {
// Check if this is a single-response scenario with waiting items
$waiting_count = $this->database->select('tmgmt_capi_request_processor', 't')
->condition('tjid', $jobid)
->condition('jobid', $capi_job_id)
->condition('statuscode', 'WAITING_FOR_RETRY')
->countQuery()
->execute()
->fetchField();
if ($waiting_count > 0) {
// Propagate success to all waiting items
$result = $this->queueOperations->propagateSuccessToWaitingItems($jobid, $capi_job_id, $first_item_id);
$this->logger->get('TMGMT_CONTENTAPI_UPLOAD_SUCCESS')->info(
'Single-response success: First item @first_item succeeded, propagated to @count waiting items (job @job_id)',
[
'@first_item' => $first_item_id,
'@count' => $result['updated_count'] ?? 0,
'@job_id' => $jobid
]
);
return $result;
}
return [
'success' => TRUE,
'message' => 'No waiting items found - normal upload completion',
'updated_count' => 0
];
} catch (\Exception $e) {
$this->logger->get('TMGMT_CONTENTAPI_UPLOAD_SUCCESS')->error(
'Error handling single-response success for job @job_id: @error',
['@job_id' => $jobid, '@error' => $e->getMessage()]
);
return [
'success' => FALSE,
'message' => 'Error handling success propagation: ' . $e->getMessage()
];
}
}
/**
* Perform partial cleanup (preserve successful uploads).
*/
private function performPartialCleanup(int $jobid, string $capi_job_id, string $allFilesPath, string $zipPath, \Throwable $ex, array $specific_failed_items = []): array {
$this->logger->get('TMGMT_CONTENTAPI_FAILURE_HANDLING')->info(
'Performing PARTIAL cleanup for job @jobids - preserving successful uploads',
['@jobids' => $jobid . '_' . $capi_job_id]
);
try {
$job = Job::load($jobid);
// Use QueueOperations service for intelligent partial cleanup
$cleanup_result = $this->queueOperations->cleanupPartialJobFailure($jobid, $capi_job_id);
if (!$cleanup_result['success']) {
// Fallback to complete cleanup
return $this->performCompleteCleanup($jobid, $capi_job_id, $allFilesPath, $zipPath, $ex);
}
// Auto-requeue logic
$requeue_result = ['success' => FALSE, 'requeued_count' => 0];
if ($this->shouldAutoRequeue($ex)) {
$requeue_result = $this->queueOperations->requeueFailedItems($jobid, $capi_job_id, $specific_failed_items);
}
// Update job state and messaging
$upload_status = $this->queueOperations->getJobUploadStatus($jobid, $capi_job_id);
if ($upload_status['uploaded'] > 0) {
$job->addMessage('Partial upload completed: @uploaded successful, @failed failed', [
'@uploaded' => $upload_status['uploaded'],
'@failed' => $upload_status['failed']
]);
$job->save();
}
// User messaging
if ($requeue_result['success']) {
\Drupal::messenger()->addMessage(
$job->label() . ' - ' . $requeue_result['requeued_count'] . ' failed items automatically requeued for retry. Successful uploads preserved.',
'warning'
);
} else {
$failed_count = count($this->queueOperations->getFailedUploadItems($jobid, $capi_job_id));
\Drupal::messenger()->addMessage(
$job->label() . ' - Partial failure: ' . $failed_count . ' items failed, successful uploads preserved. Use admin interface to retry failed items.',
'warning'
);
}
return [
'success' => TRUE,
'strategy' => 'PARTIAL_CLEANUP',
'message' => 'Partial cleanup performed - successful uploads preserved',
'details' => [
'cleanup' => $cleanup_result,
'requeue' => $requeue_result,
'status' => $upload_status
]
];
}
catch (\Exception $e) {
return [
'success' => FALSE,
'strategy' => 'PARTIAL_CLEANUP',
'message' => 'Partial cleanup failed: ' . $e->getMessage(),
'details' => []
];
}
}
/**
* Determine if failed items should be automatically requeued.
*/
private function shouldAutoRequeue(\Throwable $ex): bool {
$recoverable_errors = [
'timeout',
'connection',
'network',
'temporary',
'rate limit',
'service unavailable'
];
$error_message = strtolower($ex->getMessage());
foreach ($recoverable_errors as $recoverable) {
if (strpos($error_message, $recoverable) !== FALSE) {
return TRUE;
}
}
return FALSE;
}
/**
* Check if all files are uploaded using database state.
*
* @param int $jobid
* The job ID.
* @param string $capi_job_id
* The CAPI job ID.
*
* @return bool
* TRUE if all files are uploaded, FALSE otherwise.
*/
public function areAllFilesUploaded($jobid, $capi_job_id): bool {
try {
$pending_count = $this->database->select('tmgmt_capi_request_processor', 'p')
->condition('tjid', $jobid)
->condition('statuscode', 'SENDING')
->condition('file_upload_status', ['FILE_GENERATING', 'FILE_GENERATED', 'READY_FOR_UPLOAD', 'UPLOADING'], 'IN')
->countQuery()
->execute()
->fetchField();
$uploaded_count = $this->database->select('tmgmt_capi_request_processor', 'p')
->condition('tjid', $jobid)
->condition('statuscode', 'SENDING')
->condition('file_upload_status', 'UPLOADED')
->countQuery()
->execute()
->fetchField();
$this->logger->get("TMGMT_CONTENTAPI_FAILURE_HANDLING")->info(
'Job @capi_job_id upload status: @uploaded uploaded, @pending pending',
[
'@capi_job_id' => $capi_job_id,
'@uploaded' => $uploaded_count,
'@pending' => $pending_count
]
);
return $pending_count == 0 && $uploaded_count > 0;
}
catch (\Exception $e) {
$this->logger->get("TMGMT_CONTENTAPI_FAILURE_HANDLING")->error(
'Error checking file upload status: @message',
['@message' => $e->getMessage()]
);
return FALSE;
}
}
/**
* Get comprehensive failure report for a job.
*
* @param int $jobid
* The job ID.
* @param string $capi_job_id
* The CAPI job ID.
*
* @return array
* Comprehensive failure report.
*/
public function getJobFailureReport($jobid, $capi_job_id): array {
return [
'status' => $this->queueOperations->getJobUploadStatus($jobid, $capi_job_id),
'failed_details' => $this->queueOperations->getFailedUploadItems($jobid, $capi_job_id),
'can_requeue' => TRUE,
'recommendations' => $this->getFailureRecommendations($jobid, $capi_job_id)
];
}
/**
* Get failure recommendations based on error patterns.
*/
private function getFailureRecommendations($jobid, $capi_job_id): array {
$failed_items = $this->queueOperations->getFailedUploadItems($jobid, $capi_job_id);
$recommendations = [];
foreach ($failed_items as $item) {
$error_message = strtolower($item['errormessage'] ?? '');
if (strpos($error_message, 'timeout') !== FALSE) {
$recommendations[] = 'Consider increasing timeout settings or retrying during off-peak hours';
} elseif (strpos($error_message, 'size') !== FALSE) {
$recommendations[] = 'File size may be too large - consider splitting large content';
} elseif (strpos($error_message, 'network') !== FALSE) {
$recommendations[] = 'Network connectivity issues - check internet connection and retry';
} else {
$recommendations[] = 'Unknown error - manual investigation recommended';
}
}
return array_unique($recommendations);
}
/**
* HELPER: Determine if job uses single-response scenario (one API response for all items).
*
* Covers four main upload scenarios:
* 1. SEPARATE FILES: one_export_file=false, transfer-settings=false → Multiple files, multiple responses
* 2. SINGLE FILE: one_export_file=true, transfer-settings=false → One file, one response
* 3. ZIP + SINGLE FILE: one_export_file=true, transfer-settings=true → One ZIP (1 XLF inside), one response
* 4. ZIP + MULTIPLE FILES: one_export_file=false, transfer-settings=true → One ZIP (multiple XLFs inside), one response
*
* KEY INSIGHT: ZIP transfer ALWAYS results in single response regardless of one_export_file setting!
*
* @param \Drupal\tmgmt\JobInterface $job
* The job to check.
*
* @return bool
* TRUE if single-response scenario, FALSE for separate files (only scenario 1).
*/
public function isSingleResponseScenario(JobInterface $job) {
$one_export_file = $job->getTranslator()->getSetting('one_export_file');
$zip_transfer = $job->getTranslator()->getSetting('transfer-settings');
// Single response scenarios:
// 1. Single export file (one_export_file = true) - All items in one XLF file
// 2. ZIP transfer (transfer-settings = true) - All files packaged in one ZIP
// - This includes BOTH: ZIP+single XLF AND ZIP+multiple XLFs
// - CAPI sees only one ZIP upload, so only one API response regardless of contents
$is_single_response = $one_export_file || $zip_transfer;
// Determine specific scenario for logging
$scenario = '';
if (!$one_export_file && !$zip_transfer) {
$scenario = 'SEPARATE_FILES';
} elseif ($one_export_file && !$zip_transfer) {
$scenario = 'SINGLE_FILE';
} elseif ($one_export_file && $zip_transfer) {
$scenario = 'ZIP_SINGLE_FILE';
} elseif (!$one_export_file && $zip_transfer) {
$scenario = 'ZIP_MULTIPLE_FILES';
}
$this->logger->get("TMGMT_CONTENTAPI_SCENARIO_DETECTION")->info(
'Job @jobid scenario: @scenario (one_export_file=@single, transfer-settings=@zip) → single_response=@result',
[
'@jobid' => $job->id(),
'@scenario' => $scenario,
'@single' => $one_export_file ? 'true' : 'false',
'@zip' => $zip_transfer ? 'true' : 'false',
'@result' => $is_single_response ? 'true' : 'false'
]
);
return $is_single_response;
}
/**
* ROBUSTNESS IMPROVEMENT: Check if job submission is a duplicate.
*
* Performs comprehensive duplicate detection using multiple criteria:
* - Queue existence check
* - Job processing state check
* - Active upload tracking check
*
* @param \Drupal\tmgmt\JobInterface $job
* The job to check for duplicates.
* @param string $export_queue_name
* The export queue name to check.
*
* @return bool
* TRUE if job is a duplicate and should be blocked, FALSE if job can proceed.
*/
public function isDuplicateJobSubmission(JobInterface $job, string $export_queue_name): bool {
// Check if job is already in queue
$queue_items = $this->queueOperations->getQueueItems($export_queue_name, 'item_id');
// Check if job already has active upload tracking records using service method
$active_upload_count = $this->capiDataProcessor->getActiveUploadCount($job->id());
// Check if job already exists in queue, already processed, or has active uploads
if (in_array($job->id(), $queue_items) || !$job->isUnprocessed() || $active_upload_count > 0) {
$reason = '';
if (in_array($job->id(), $queue_items)) {
$reason = 'already in export queue';
} elseif (!$job->isUnprocessed()) {
$reason = 'already processed (state: ' . $job->getState() . ')';
} elseif ($active_upload_count > 0) {
$reason = 'has active file uploads in progress (' . $active_upload_count . ' items)';
}
$this->logger->get("TMGMT_CONTENTAPI_JOB_POST")->notice(
'Job submission blocked - @reason for job ID: @jobids',
[
'@reason' => $reason,
'@jobids' => $job->id(),
]
);
return TRUE; // Job is duplicate, should be blocked
}
return FALSE; // Job is not duplicate, can proceed
}
/**
* ROBUSTNESS IMPROVEMENT: Create upload tracking record for a single item.
* Creates CREATED statuscode record when queue item is created - one at a time.
*
* @param \Drupal\tmgmt\JobInterface $job
* The job being processed.
* @param string $capi_job_id
* The CAPI job ID.
* @param int $item_id
* The specific item ID being processed.
* @param string $initial_status
* Initial file upload status (default: FILE_GENERATING).
*/
public function createUploadTrackingRecord(JobInterface $job, $capi_job_id, $item_id, $initial_status = 'FILE_GENERATING') {
try {
// SMART REQUEST_ID PATTERN: Use consistent request_id for single-response scenarios
$is_single_response = $this->isSingleResponseScenario($job);
if ($is_single_response) {
// For single-response scenarios: All items share request_id based on first item
// This includes: single file (one_export_file) AND ZIP transfer (transfer-settings)
$job_items = $job->getItems();
$first_item_id = array_key_first($job_items);
$request_id = "temp_$first_item_id"; // All items use first item's ID in request_id
} else {
// For separate-file jobs: Each item gets its own request_id
$request_id = "temp_$item_id";
}
// Create database record for upload tracking with initial status
$job_details = [
'tjid' => $job->id(),
'tjiid' => $item_id,
'job_id' => $capi_job_id,
'provider_id' => '',
'request_id' => $request_id, // Use consistent request_id for single-file
'job_status' => 'SENDING',
'update_time' => NULL,
'file_upload_status' => $initial_status, // Set initial tracking status
];
$this->capiDataProcessor->storeDataWhenJobCreated($job_details);
$this->logger->get("TMGMT_CONTENTAPI_UPLOAD_TRACKING")->info(
'Created upload tracking record for item @item_id in job @job_id (CAPI: @capi_job_id)',
['@item_id' => $item_id, '@job_id' => $job->id(), '@capi_job_id' => $capi_job_id]
);
}
catch (\Exception $e) {
$this->logger->get("TMGMT_CONTENTAPI_UPLOAD_TRACKING")->error(
'Failed to create upload tracking record for item @item_id: @message',
['@item_id' => $item_id, '@message' => $e->getMessage()]
);
// Don't throw exception - let upload proceed even if tracking fails
}
}
/**
* ROBUSTNESS IMPROVEMENT: Update existing records with correct provider details.
* Updates the records created during initialization with final provider information.
*
* @param \Drupal\tmgmt\JobInterface $job
* The job being processed.
* @param string $provider_id
* The actual provider ID from job submission.
* @param string $capi_job_id
* The CAPI job ID.
*/
public function updateExistingRecordsWithProviderDetails(JobInterface $job, $provider_id, $capi_job_id) {
try {
// Use the existing method to get API responses - much cleaner!
$is_single_response = $this->isSingleResponseScenario($job);
$contentApiBundle = $this->getApiResponsesFromProcessorTable($job->id(), $capi_job_id, $is_single_response);
foreach ($contentApiBundle as $item_id => $response_data) {
$request_id = $response_data[0]->getRequestId();
// Update the existing SENDING record with correct provider and request IDs
// Add file_upload_status condition to ensure we only update successfully uploaded files
$updated_rows = $this->database->update('tmgmt_capi_request_processor')
->fields([
'providerid' => $provider_id,
'requestid' => $request_id,
'statuscode' => 'CREATED', // Change statuscode to CREATED after submission
'lastupdated' => $this->database->query('SELECT UTC_TIMESTAMP()')->fetchField(),
])
->condition('tjid', $job->id())
->condition('tjiid', $item_id)
->condition('jobid', $capi_job_id)
->condition('statuscode', 'SENDING')
->condition('file_upload_status', 'UPLOADED') // Ensure only uploaded files are updated
->execute();
// Log warning if unexpected number of rows updated
if ($updated_rows !== 1) {
$this->logger->get("TMGMT_CONTENTAPI_UPLOAD_TRACKING")->warning(
'Unexpected number of rows updated (@count) for item @item_id in job @job_id',
['@count' => $updated_rows, '@item_id' => $item_id, '@job_id' => $job->id()]
);
}
$this->logger->get("TMGMT_CONTENTAPI_UPLOAD_TRACKING")->info(
'Updated tracking record for item @item_id with provider @provider_id and request @request_id',
[
'@item_id' => $item_id,
'@provider_id' => $provider_id,
'@request_id' => $request_id
]
);
}
}
catch (\Exception $e) {
$this->logger->get("TMGMT_CONTENTAPI_UPLOAD_TRACKING")->error(
'Failed to update records with provider details: @message',
['@message' => $e->getMessage()]
);
}
}
/**
* Clean up zip files only (directories handled by cleanupLioxDirectories).
*
* @param string $zipPath
* Zip file path.
* @param int $jobid
* Job ID.
* @param string $capi_job_id
* CAPI job ID.
*/
public function cleanupZipFiles(string $zipPath, int $jobid, string $capi_job_id) {
// Delete zip files
if (strlen($zipPath) < 250) {
$zipfileobj = $this->jobHelper->createFileObject($zipPath);
if ($zipfileobj->getFileUri() != NULL) {
\Drupal::service('file_system')->delete($zipfileobj->getFileUri());
}
$this->logger->get('TMGMT_CONTENTAPI_JOB_POST_CLEANUP')->info(
'Zip file deleted at path: @path for job: @jobids',
['@path' => $zipPath, '@jobids' => $jobid . '_' . $capi_job_id]
);
}
}
/**
* Comprehensive cleanup for all LioxSentFiles and LioxRefFiles directories.
* Handles all directory patterns created by ExportJobFiles service including:
* - Regular translation directories: <capi_job_id>_<job_id>_<src_lang>_<trg_lang>
* - Translation memory directories: <capi_job_id>_tmupdate_<job_id>_<src_lang>_<trg_lang>_tm
* - Covers both LioxSentFiles and LioxRefFiles locations
*
* @param int $jobid
* Job ID.
* @param string $capi_job_id
* CAPI job ID.
* @param \Drupal\tmgmt\JobInterface $job
* The job object for language information.
*/
private function cleanupLioxDirectories(int $jobid, string $capi_job_id, JobInterface $job) {
try {
$fileSystem = \Drupal::service('file_system');
$scheme = $job->getSetting('scheme') ?: 'public';
// Get source and target languages
$src_lang = $job->getRemoteSourceLanguage();
$trg_lang = $job->getRemoteTargetLanguage();
// Pattern: <capi_job_id>_<job_id>_<src_lang>_<trg_lang>
$base_pattern = $capi_job_id . '_' . $jobid . '_' . $src_lang . '_' . $trg_lang;
// Directory patterns to clean up
$directories_to_clean = [
// Regular translation directories
$scheme . '://tmgmt_contentapi/LioxSentFiles/' . $base_pattern,
$scheme . '://tmgmt_contentapi/LioxRefFiles/' . $base_pattern,
// Translation memory directories (with _tmupdate_ and _tm suffix)
$scheme . '://tmgmt_contentapi/LioxSentFiles/' . $capi_job_id . '_tmupdate_' . $jobid . '_' . $src_lang . '_' . $trg_lang . '_tm',
$scheme . '://tmgmt_contentapi/LioxRefFiles/' . $capi_job_id . '_tmupdate_' . $jobid . '_' . $src_lang . '_' . $trg_lang . '_tm',
];
$cleaned_directories = 0;
foreach ($directories_to_clean as $directory_uri) {
$real_path = $fileSystem->realpath($directory_uri);
if ($real_path && is_dir($real_path)) {
if ($fileSystem->deleteRecursive($directory_uri)) {
$cleaned_directories++;
$this->logger->get('TMGMT_CONTENTAPI_JOB_POST_CLEANUP')->info(
'LioxFiles directory deleted: @path for job: @jobids',
['@path' => $directory_uri, '@jobids' => $jobid . '_' . $capi_job_id]
);
} else {
$this->logger->get('TMGMT_CONTENTAPI_JOB_POST_CLEANUP')->warning(
'Failed to delete LioxFiles directory: @path for job: @jobids',
['@path' => $directory_uri, '@jobids' => $jobid . '_' . $capi_job_id]
);
}
}
}
$this->logger->get('TMGMT_CONTENTAPI_JOB_POST_CLEANUP')->info(
'Comprehensive LioxFiles cleanup completed: @count directories processed for job: @jobids (includes all variants: regular, translation memory)',
['@count' => $cleaned_directories, '@jobids' => $jobid . '_' . $capi_job_id]
);
} catch (\Exception $e) {
$this->logger->get('TMGMT_CONTENTAPI_JOB_POST_CLEANUP')->error(
'Error during LioxFiles cleanup for job @jobids: @message',
['@jobids' => $jobid . '_' . $capi_job_id, '@message' => $e->getMessage()]
);
}
}
/**
* Recursively delete a directory and its contents.
*
* @param string $dir
* Directory path to delete.
*/
private function recursiveDirectoryDelete(string $dir): void {
if (is_dir($dir)) {
$files = array_diff(scandir($dir), ['.', '..']);
foreach ($files as $file) {
$path = $dir . '/' . $file;
is_dir($path) ? $this->recursiveDirectoryDelete($path) : unlink($path);
}
rmdir($dir);
}
}
/**
* Clean up all queue items for complete cleanup.
*
* @param int $jobid
* Job ID.
*/
public function cleanupQueueItems(int $jobid) {
// Queue names from CreateConnectorJob constants
$send_files_queue = 'send_file_for_translation_to_capi';
$generate_files_queue = 'generate_file_for_translation_to_capi';
// Delete all queue items for send files
$queue_items = $this->queueOperations->getQueueItems($send_files_queue, 'jobid');
$current_job_id = [$jobid];
$specific_job_array = array_intersect($queue_items, $current_job_id);
foreach ($specific_job_array as $item_id => $item) {
$this->queueOperations->deleteItemFromQueue($item_id);
}
// Delete all queue items for generate files
$queue_items_generate_files = $this->queueOperations->getQueueItems($generate_files_queue, 'jobid');
$specific_job_array_generate_files = array_intersect($queue_items_generate_files, $current_job_id);
foreach ($specific_job_array_generate_files as $item_id => $item) {
$this->queueOperations->deleteItemFromQueue($item_id);
}
$this->logger->get('TMGMT_CONTENTAPI_JOB_POST_CLEANUP')->info(
'All queue items deleted for job: @jobids',
['@jobids' => $jobid]
);
}
/**
* Get API responses from the tmgmt_capi_request_processor table.
*
* @param int $jobid
* The local Drupal job ID.
* @param string $capi_job_id
* The CAPI job ID.
* @param bool $is_single_file
* TRUE if this is a single-file job, FALSE for separate files.
*
* @return array
* An array of API responses indexed by item_id.
*/
public function getApiResponsesFromProcessorTable($jobid, $capi_job_id, $is_single_response = FALSE): array {
// UNIFIED QUERY: Use your proven MIN(rid) + GROUP BY requestid pattern for both scenarios
$query = $this->database->select('tmgmt_capi_request_processor', 't1');
$query->fields('t1', ['tjiid', 'capi_response']);
// Create subquery to find first rid for each requestid (your existing proven pattern)
$subquery = $this->database->select('tmgmt_capi_request_processor', 't2');
$subquery->addField('t2', 'requestid');
$subquery->addExpression('MIN(rid)', 'min_rid');
$subquery->condition('t2.tjid', $jobid);
$subquery->condition('t2.jobid', $capi_job_id);
$subquery->condition('t2.statuscode', 'SENDING');
$subquery->condition('t2.file_upload_status', 'UPLOADED');
$subquery->isNotNull('t2.capi_response');
$subquery->groupBy('t2.requestid');
// Join to get only one record per requestid (your proven approach)
$query->join($subquery, 'subq', 't1.rid = subq.min_rid');
$query->condition('t1.tjid', $jobid);
$query->condition('t1.jobid', $capi_job_id);
$query->condition('t1.statuscode', 'SENDING');
$query->condition('t1.file_upload_status', 'UPLOADED');
$query->isNotNull('t1.capi_response');
$representative_results = $query->execute()->fetchAllKeyed();
if (empty($representative_results)) {
return [];
}
if ($is_single_response) {
// SINGLE-RESPONSE: Response stored only in first item to avoid duplicates (includes single-file AND ZIP scenarios)
// Get all item IDs for this job to replicate response
$all_item_ids = $this->database->select('tmgmt_capi_request_processor', 'p')
->fields('p', ['tjiid'])
->condition('tjid', $jobid)
->condition('jobid', $capi_job_id)
->condition('statuscode', 'SENDING')
->condition('file_upload_status', 'UPLOADED')
->execute()
->fetchCol();
// Get the response from first item (where it's actually stored)
$first_item_id = min($all_item_ids); // Get smallest item ID (first item)
$single_response = $representative_results[$first_item_id] ?? reset($representative_results);
$results = array_fill_keys($all_item_ids, $single_response);
} else {
// SEPARATE-FILE: Each item has unique request_id, so we get individual responses
$results = $representative_results;
}
// Apply same transformation for both scenarios
return array_map(fn($response) => unserialize($response), $results);
}
}