ai_agents_test-1.0.0-alpha1/src/Drush/Commands/AiAgentTestCommands.php
src/Drush/Commands/AiAgentTestCommands.php
<?php
declare(strict_types=1);
namespace Drupal\ai_agents_test\Drush\Commands;
use Consolidation\OutputFormatters\StructuredData\RowsOfFields;
use Drupal\Core\Entity\EntityTypeManagerInterface;
use Drupal\Core\Session\AccountProxyInterface;
use Drupal\ai\AiProviderPluginManager;
use Drupal\ai_agents_test\Service\TestRunner;
use Drush\Attributes as CLI;
use Drush\Boot\DrupalBootLevels;
use Drush\Commands\DrushCommands;
use Symfony\Component\DependencyInjection\ContainerInterface;
/**
* Provides Drush commands that integrate with the AI agents tests.
*/
class AiAgentTestCommands extends DrushCommands {
/**
* The constructor.
*
* @param \Drupal\ai\AiProviderPluginManager $aiProviderPluginManager
* The AI provider plugin manager.
* @param \Drupal\ai_agents_test\Service\TestRunner $testRunner
* The test runner service.
* @param \Drupal\Core\Session\AccountProxyInterface $currentUser
* The current user service.
* @param \Drupal\Core\Entity\EntityTypeManagerInterface $entityTypeManager
* The entity type manager service.
*/
public function __construct(
protected AiProviderPluginManager $aiProviderPluginManager,
protected TestRunner $testRunner,
protected AccountProxyInterface $currentUser,
protected EntityTypeManagerInterface $entityTypeManager,
) {
parent::__construct();
}
/**
* Return an instance of these Drush commands.
*
* @param \Symfony\Component\DependencyInjection\ContainerInterface $container
* The container.
*
* @return \Drupal\ai\Drush\Commands\AiAgentTestCommands
* The instance of Drush commands.
*/
public static function create(ContainerInterface $container): AiAgentTestCommands {
return new AiAgentTestCommands(
$container->get('ai.provider'),
$container->get('ai_agents_test.test_runner'),
$container->get('current_user'),
$container->get('entity_type.manager'),
);
}
/**
* Run one or multiple tests.
*/
#[CLI\Command(name: 'agents:test-agents', aliases: ['agetes'])]
#[CLI\Option(name: 'test_id', description: 'The test id to run test for.')]
#[CLI\Option(name: 'group_id', description: 'The group id to run test for.')]
#[CLI\Option(name: 'uid', description: 'The potential user you want to run as.')]
#[CLI\Option(name: 'detailed', description: 'If you want a detailed output.')]
#[CLI\Option(name: 'provider', description: 'Indicates a provider to use other than the default. If left empty, the default provider will be used.')]
#[CLI\Option(name: 'model', description: 'Indicates which model to use, other than the default. If left empty, the default model will be used.')]
#[CLI\Usage(name: 'drush agents:test-agents --test_id=1', description: 'Tests the test 1.')]
#[CLI\Bootstrap(level: DrupalBootLevels::FULL)]
#[CLI\Option(name: 'eval_provider', description: 'Provider for LLM evaluation of test results.')]
#[CLI\Option(name: 'eval_model', description: 'Model for LLM evaluation of test results.')]
public function runTests(
array $options = [
'test_id' => self::OPT,
'group_id' => self::OPT,
'uid' => self::OPT,
'detailed' => self::OPT,
'provider' => self::OPT,
'model' => self::OPT,
'system' => self::OPT,
'eval_provider' => self::OPT,
'eval_model' => self::OPT,
'format' => 'table',
],
): RowsOfFields {
// Make sure that one of test_id, group_id is provided.
if (empty($options['test_id']) && empty($options['group_id'])) {
$this->logger()->error(dt('You must provide at least one of the following options: test_id or group_id.'));
exit;
}
// If the provider is set, the model must also be set and vice versa.
if (!empty($options['provider']) && empty($options['model'])) {
$this->logger()->error(dt('If you set a provider, you must also set a model.'));
exit;
}
if (empty($options['provider']) && !empty($options['model'])) {
$this->logger()->error(dt('If you set a model, you must also set a provider.'));
exit;
}
// If provider and model are set, we try to load the provider.
if (!empty($options['provider']) && !empty($options['model'])) {
$provider = $this->aiProviderPluginManager->createInstance($options['provider']);
if (!$provider) {
$this->logger()->error(dt('The provider @provider does not exist.', ['@provider' => $options['provider']]));
exit;
}
$provider_id = $options['provider'];
$model_id = $options['model'];
}
else {
// Retrieve the AI Provider and the options.
$defaults = $this->aiProviderPluginManager->getDefaultProviderForOperationType('chat_with_tools');
if (empty($defaults)) {
$this->logger()->error(dt('No default AI provider set for chat with tools'));
exit;
}
$provider_id = empty($options['provider']) ? $defaults['provider_id'] : $options['provider'];
$model_id = empty($options['model']) ? $defaults['model_id'] : $options['model'];
}
// Validate the provider ID and model ID.
if (empty($provider_id) || empty($model_id)) {
$this->logger()->error(dt('No default AI provider or model set'));
exit;
}
if (!empty($options['uid'])) {
// Upcast the current user to a user ID.
$uid = (int) $options['uid'];
/** @var \Drupal\user\Entity\User $user */
$user = $this->entityTypeManager->getStorage('user')->load($uid);
if (!$user) {
$this->logger()->error(dt('The user with ID @uid does not exist.', ['@uid' => $uid]));
exit;
}
$this->currentUser->setAccount($user);
$this->logger()->notice(dt('Running tests as user: @user', ['@user' => $user->getDisplayName()]));
}
// Run Individual Test.
if (!empty($options['test_id'])) {
$test = $this->testRunner->runTest((int) $options['test_id'], '', $provider_id, $model_id);
$total_success = TRUE;
$rows = [];
foreach ($test['results'] as $key => $result) {
if (!$result['result']) {
$total_success = FALSE;
}
if (!empty($options['detailed'])) {
$rows[] = [
'id' => $key,
'result' => $result['result'] ? 'Success' : 'Failure',
'label' => $result['label'],
'message' => $result['message'],
];
}
}
$rows[] = [
'id' => 'total',
'result' => $total_success ? 'Success' : 'Failure',
'label' => $test['entity']->label(),
'message' => '',
];
return new RowsOfFields($rows);
}
// Run Test Group.
if (!empty($options['group_id'])) {
$group_id = (int) $options['group_id'];
// Load the test group entity.
$test_group = $this->entityTypeManager->getStorage('ai_agents_test_group')->load($group_id);
if (!$test_group) {
$this->logger()->error(dt('Test group with ID @id not found.', ['@id' => $group_id]));
return new RowsOfFields([]);
}
$this->logger()->notice(dt('Running test group: @name', ['@name' => $test_group->label()]));
// Check if we should use a different model for evaluation.
$eval_provider_id = NULL;
$eval_model_id = NULL;
if (!empty($options['eval_provider']) && !empty($options['eval_model'])) {
$eval_provider_id = $options['eval_provider'];
$eval_model_id = $options['eval_model'];
$this->logger()->notice(dt('Overriding LLM evaluation model: @provider/@model', [
'@provider' => $eval_provider_id,
'@model' => $eval_model_id,
]));
}
else {
$this->logger()->notice(dt('Using LLM evaluation default model (@provider/@model)', [
'@provider' => $provider_id,
'@model' => $model_id,
]));
}
// Create a group result entity (mimicking what the UI does).
$group_result_data = [
'label' => $test_group->label() . ' - ' . date('Y-m-d H:i:s'),
'ai_agents_test_group' => $group_id,
'chat_provider' => $provider_id,
'model' => $model_id,
'process_status' => 'running',
'config_reset' => $test_group->get('config_reset')->value ?? FALSE,
'ai_agents_test_results' => [],
];
// Add evaluation model if specified.
if ($eval_provider_id && $eval_model_id) {
$group_result_data['eval_provider'] = $eval_provider_id;
$group_result_data['eval_model'] = $eval_model_id;
}
$group_result = $this->entityTypeManager->getStorage('ai_agents_test_group_result')->create($group_result_data);
$group_result->save();
// Get all test IDs from the group.
$test_refs = $test_group->get('tests')->getValue();
if (empty($test_refs)) {
$this->logger()->warning(dt('Test group "@name" has no tests.', ['@name' => $test_group->label()]));
return new RowsOfFields([]);
}
$rows = [];
$total_tests = count($test_refs);
$successful_tests = 0;
$tests_run = 0;
$breaking_exit = FALSE;
// Minimum percentage of success required to avoid a breaking exit.
$approval_percentage = $test_group->get('approval_percentage')->value ?? NULL;
// Track timing for performance monitoring.
$group_start_time = microtime(TRUE);
// Run each test in the group (mimicking the AJAX sequential execution).
foreach ($test_refs as $index => $test_ref) {
$test_id = $test_ref['target_id'];
$tests_run++;
$test_start_time = microtime(TRUE);
$this->logger()->notice(dt('Running test @num of @total: Test ID @id (@model)', [
'@num' => $index + 1,
'@total' => $total_tests,
'@id' => $test_id,
'@model' => $model_id,
]));
// Run the test with all parameters including evaluation model.
$test = $this->testRunner->runTest(
(int) $test_id,
(string) $group_result->id(),
$provider_id,
$model_id,
$eval_provider_id,
$eval_model_id
);
// Process results.
$test_success = TRUE;
$test_label = '';
if (isset($test['error'])) {
$test_success = FALSE;
$test_label = isset($test['entity']) ? $test['entity']->label() : "Test {$test_id}";
$rows[] = [
'id' => "test_{$test_id}",
'result' => 'Error',
'label' => $test_label,
'message' => $test['error'],
];
}
else {
$test_label = $test['entity']->label();
foreach ($test['results'] as $key => $result) {
if (!$result['result']) {
$test_success = FALSE;
}
if (!empty($options['detailed'])) {
$rows[] = [
'id' => "test_{$test_id}_rule_{$key}",
'result' => $result['result'] ? 'Success' : 'Failure',
'label' => $result['label'],
'message' => $result['message'],
];
}
}
if ($test_success) {
$successful_tests++;
}
// Calculate test execution time.
$test_duration = microtime(TRUE) - $test_start_time;
// Check if we should continue based on approval percentage.
if ($approval_percentage !== NULL && !$test_success) {
// Calculate max possible success if all remaining tests pass.
$remaining_tests = $total_tests - $tests_run;
$max_possible_successful = $successful_tests + $remaining_tests;
$max_possible_percentage = ($max_possible_successful / $total_tests) * 100;
// If it's impossible to reach the approval percentage, break early.
if ($max_possible_percentage < $approval_percentage) {
$breaking_exit = TRUE;
$this->logger()->error(dt('Breaking exit: Maximum possible success rate (@max%) is below approval threshold (@approval%). Stopping test execution.', [
'@max' => round($max_possible_percentage, 2),
'@approval' => $approval_percentage,
]));
// Add test summary if not in detailed mode.
if (empty($options['detailed'])) {
$rows[] = [
'id' => "test_{$test_id}",
'result' => $test_success ? 'Success' : 'Failure',
'label' => $test_label,
'message' => sprintf('%.2fs', $test_duration),
];
}
// Exit the loop early.
break;
}
}
// Add test summary if not in detailed mode.
if (empty($options['detailed'])) {
$rows[] = [
'id' => "test_{$test_id}",
'result' => $test_success ? 'Success' : 'Failure',
'label' => $test_label,
'message' => sprintf('%.2fs', $test_duration),
];
}
else {
// Add timing info in detailed mode after all rule results.
$rows[] = [
'id' => "test_{$test_id}_timing",
'result' => '---',
'label' => sprintf('Test %d timing', $test_id),
'message' => sprintf('Completed in %.2fs', $test_duration),
];
}
}
}
// Update group result status if breaking exit occurred.
if ($breaking_exit) {
$group_result->set('process_status', 'failed');
$group_result->save();
}
// Calculate total execution time.
$group_duration = microtime(TRUE) - $group_start_time;
// AgentTestGroupResult::preSave() handles the test group calculations.
// Reload the entity to get the latest calculated values.
$group_result = $this->entityTypeManager->getStorage('ai_agents_test_group_result')->load($group_result->id());
$success_percentage = $group_result->get('success_percentage')->value ?? 0;
$final_result = $group_result->get('result')->value ?? 'failure';
// Add group summary.
$message = sprintf(
'%d/%d tests passed | Duration: %ds | Group Result ID: %d | Success rate: %.2f%%',
$successful_tests,
$tests_run,
(int) round($group_duration),
$group_result->id(),
$success_percentage
);
if ($breaking_exit) {
$message .= ' | BREAKING EXIT: Approval threshold not achievable';
}
$rows[] = [
'id' => 'group_total',
'result' => $final_result === 'success' ? 'Success' : 'Failure',
'label' => sprintf('Group: %s', $test_group->label()),
'message' => $message,
];
$this->logger()->notice(dt('Test group completed. Results available at: /admin/content/ai-agents-test/group/result/@id', [
'@id' => $group_result->id(),
]));
return new RowsOfFields($rows);
}
return new RowsOfFields([]);
}
}
