feat(ai): add AI plugins for A/B test generation and analysis
Implement AI-powered A/B testing capabilities for landing pages: - ABTestGeneratePlugin: creates experiment variants with config overrides - ABTestAnalyzePlugin: evaluates results and determines statistical significance - Generate 1-3 test variants focusing on high-impact changes (headlines, CTAs, value props) - Analyze conversion rates with 95% confidence threshold - Require minimum 100 impressions per variant before declaring winner - Return structured recommendations for next actions
This commit is contained in:
79
src/GitCaddy.AI.Service/Plugins/ABTestAnalyzePlugin.cs
Normal file
79
src/GitCaddy.AI.Service/Plugins/ABTestAnalyzePlugin.cs
Normal file
@@ -0,0 +1,79 @@
|
||||
// Copyright 2026 MarketAlly. All rights reserved.
|
||||
// SPDX-License-Identifier: BSL-1.1
|
||||
|
||||
using System.Text.Json.Serialization;
|
||||
using MarketAlly.AIPlugin;
|
||||
|
||||
namespace GitCaddy.AI.Service.Plugins;
|
||||
|
||||
/// <summary>
|
||||
/// Result DTO for A/B test analysis.
|
||||
/// </summary>
|
||||
public class ABTestAnalyzeResult
|
||||
{
|
||||
[JsonPropertyName("status")]
|
||||
public string Status { get; set; }
|
||||
|
||||
[JsonPropertyName("winner_variant_id")]
|
||||
public long WinnerVariantId { get; set; }
|
||||
|
||||
[JsonPropertyName("confidence")]
|
||||
public double Confidence { get; set; }
|
||||
|
||||
[JsonPropertyName("summary")]
|
||||
public string Summary { get; set; }
|
||||
|
||||
[JsonPropertyName("recommendation")]
|
||||
public string Recommendation { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// AI plugin that analyzes A/B test experiment results.
|
||||
/// Evaluates conversion rates, impression counts, and event distributions
|
||||
/// to determine if there is a statistically significant winner.
|
||||
/// </summary>
|
||||
[AIPlugin("AnalyzeABTestResults",
|
||||
"Analyze A/B test experiment results and determine the outcome. " +
|
||||
"Evaluate conversion rates, impression counts, and statistical significance. " +
|
||||
"Require at least 100 impressions per variant before declaring a winner. " +
|
||||
"Use a minimum 95% confidence threshold.")]
|
||||
public class ABTestAnalyzePlugin : IAIPlugin
|
||||
{
|
||||
[AIParameter("Analysis status: must be exactly one of 'winner', 'needs_more_data', or 'no_difference'", required: true)]
|
||||
public string Status { get; set; }
|
||||
|
||||
[AIParameter("ID of the winning variant (use 0 if no winner)", required: true)]
|
||||
public long WinnerVariantId { get; set; }
|
||||
|
||||
[AIParameter("Statistical confidence level from 0.0 to 1.0", required: true)]
|
||||
public double Confidence { get; set; }
|
||||
|
||||
[AIParameter("Brief human-readable summary of the results", required: true)]
|
||||
public string Summary { get; set; }
|
||||
|
||||
[AIParameter("Actionable recommendation for what to do next", required: true)]
|
||||
public string Recommendation { get; set; }
|
||||
|
||||
public IReadOnlyDictionary<string, Type> SupportedParameters => new Dictionary<string, Type>
|
||||
{
|
||||
["status"] = typeof(string),
|
||||
["winnervariantid"] = typeof(long),
|
||||
["confidence"] = typeof(double),
|
||||
["summary"] = typeof(string),
|
||||
["recommendation"] = typeof(string)
|
||||
};
|
||||
|
||||
public Task<AIPluginResult> ExecuteAsync(IReadOnlyDictionary<string, object> parameters)
|
||||
{
|
||||
var result = new ABTestAnalyzeResult
|
||||
{
|
||||
Status = Status,
|
||||
WinnerVariantId = WinnerVariantId,
|
||||
Confidence = Confidence,
|
||||
Summary = Summary,
|
||||
Recommendation = Recommendation
|
||||
};
|
||||
|
||||
return Task.FromResult(new AIPluginResult(result, "A/B test analysis complete"));
|
||||
}
|
||||
}
|
||||
74
src/GitCaddy.AI.Service/Plugins/ABTestGeneratePlugin.cs
Normal file
74
src/GitCaddy.AI.Service/Plugins/ABTestGeneratePlugin.cs
Normal file
@@ -0,0 +1,74 @@
|
||||
// Copyright 2026 MarketAlly. All rights reserved.
|
||||
// SPDX-License-Identifier: BSL-1.1
|
||||
|
||||
using System.Text.Json.Serialization;
|
||||
using MarketAlly.AIPlugin;
|
||||
|
||||
namespace GitCaddy.AI.Service.Plugins;
|
||||
|
||||
/// <summary>
|
||||
/// A/B test variant generated by AI, with a partial config override.
|
||||
/// </summary>
|
||||
public class ABTestVariant
|
||||
{
|
||||
[JsonPropertyName("name")]
|
||||
public string Name { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Partial LandingConfig fields to override for this variant.
|
||||
/// Stored as object to preserve arbitrary nested JSON structure.
|
||||
/// </summary>
|
||||
[JsonPropertyName("config_override")]
|
||||
public object ConfigOverride { get; set; }
|
||||
|
||||
[JsonPropertyName("weight")]
|
||||
public int Weight { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Result DTO for A/B test experiment generation.
|
||||
/// </summary>
|
||||
public class ABTestGenerateResult
|
||||
{
|
||||
[JsonPropertyName("name")]
|
||||
public string Name { get; set; }
|
||||
|
||||
[JsonPropertyName("variants")]
|
||||
public List<ABTestVariant> Variants { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// AI plugin that generates A/B test experiment variants for a landing page.
|
||||
/// The AI analyzes the current landing page config and produces meaningful
|
||||
/// variant overrides to test (headlines, CTAs, value props, etc.).
|
||||
/// </summary>
|
||||
[AIPlugin("GenerateABTestExperiment",
|
||||
"Generate an A/B test experiment for a landing page. " +
|
||||
"Analyze the current landing page config and create meaningful test variants " +
|
||||
"that focus on high-impact changes like headlines, CTAs, and value propositions. " +
|
||||
"Do NOT include a control variant — it is added automatically.")]
|
||||
public class ABTestGeneratePlugin : IAIPlugin
|
||||
{
|
||||
[AIParameter("Short descriptive name for the experiment (e.g. 'Headline Impact Test')", required: true)]
|
||||
public string ExperimentName { get; set; }
|
||||
|
||||
[AIParameter("Test variants to create. Each must have 'name' (string), 'config_override' (object with partial landing config fields to override), and 'weight' (integer traffic percentage, should sum to ~50 since control gets 50%). Generate 1-3 variants with meaningfully different but plausible changes.", required: true)]
|
||||
public List<ABTestVariant> Variants { get; set; }
|
||||
|
||||
public IReadOnlyDictionary<string, Type> SupportedParameters => new Dictionary<string, Type>
|
||||
{
|
||||
["experimentname"] = typeof(string),
|
||||
["variants"] = typeof(List<ABTestVariant>)
|
||||
};
|
||||
|
||||
public Task<AIPluginResult> ExecuteAsync(IReadOnlyDictionary<string, object> parameters)
|
||||
{
|
||||
var result = new ABTestGenerateResult
|
||||
{
|
||||
Name = ExperimentName,
|
||||
Variants = Variants ?? new List<ABTestVariant>()
|
||||
};
|
||||
|
||||
return Task.FromResult(new AIPluginResult(result, "A/B test experiment generated"));
|
||||
}
|
||||
}
|
||||
@@ -124,6 +124,8 @@ public class WorkflowService : IWorkflowService
|
||||
{
|
||||
"landing_page_generate" => await ExecuteLandingPageGenerateAsync(request, cancellationToken),
|
||||
"landing_page_translate" => await ExecuteLandingPageTranslateAsync(request, cancellationToken),
|
||||
"ab_test_generate" => await ExecuteABTestGenerateAsync(request, cancellationToken),
|
||||
"ab_test_analyze" => await ExecuteABTestAnalyzeAsync(request, cancellationToken),
|
||||
_ => await ExecuteGenericTaskAsync(request, cancellationToken)
|
||||
};
|
||||
}
|
||||
@@ -231,6 +233,95 @@ public class WorkflowService : IWorkflowService
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Generates A/B test experiment variants using structured output via the ABTestGeneratePlugin.
|
||||
/// </summary>
|
||||
private async Task<ExecuteTaskResponse> ExecuteABTestGenerateAsync(
|
||||
ExecuteTaskRequest request, CancellationToken cancellationToken)
|
||||
{
|
||||
_logger.LogInformation("Generating A/B test experiment via structured output plugin");
|
||||
|
||||
var conversation = _providerFactory.CreateConversation()
|
||||
.WithSystemPrompt("""
|
||||
You are a conversion rate optimization expert. Analyze landing page configurations
|
||||
and create meaningful A/B test experiments that focus on high-impact changes.
|
||||
Generate variants that test different headlines, CTAs, value propositions, or layouts.
|
||||
Each variant should be meaningfully different but plausible.
|
||||
The control variant (original config) is added automatically — do NOT include it.
|
||||
""")
|
||||
.RegisterPlugin<ABTestGeneratePlugin>()
|
||||
.Build();
|
||||
|
||||
var ctx = request.Context;
|
||||
var prompt = $"""
|
||||
Analyze this landing page config and create an A/B test experiment.
|
||||
Focus on high-impact changes: headlines, CTAs, value propositions.
|
||||
Generate 1-3 test variants. Each variant's config_override should be a partial
|
||||
LandingConfig with only the fields that differ from the control.
|
||||
|
||||
Repository: {GetContextValue(ctx, "repo_name")}
|
||||
Description: {GetContextValue(ctx, "repo_description")}
|
||||
|
||||
Current landing page config:
|
||||
{GetContextValue(ctx, "landing_config")}
|
||||
""";
|
||||
|
||||
var response = await conversation.SendForStructuredOutputAsync<ABTestGenerateResult>(
|
||||
prompt, "GenerateABTestExperiment", cancellationToken);
|
||||
|
||||
var json = SerializeStructuredOutput(response.StructuredOutput);
|
||||
|
||||
return new ExecuteTaskResponse
|
||||
{
|
||||
Success = true,
|
||||
Result = json
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Analyzes A/B test experiment results using structured output via the ABTestAnalyzePlugin.
|
||||
/// </summary>
|
||||
private async Task<ExecuteTaskResponse> ExecuteABTestAnalyzeAsync(
|
||||
ExecuteTaskRequest request, CancellationToken cancellationToken)
|
||||
{
|
||||
_logger.LogInformation("Analyzing A/B test results via structured output plugin");
|
||||
|
||||
var conversation = _providerFactory.CreateConversation()
|
||||
.WithSystemPrompt("""
|
||||
You are a data analyst specializing in A/B test evaluation.
|
||||
Analyze experiment results using statistical methods to determine significance.
|
||||
Require at least 100 impressions per variant before declaring a winner.
|
||||
Use a minimum 95% confidence threshold for statistical significance.
|
||||
Be conservative — only declare a winner when the data clearly supports it.
|
||||
""")
|
||||
.RegisterPlugin<ABTestAnalyzePlugin>()
|
||||
.Build();
|
||||
|
||||
var ctx = request.Context;
|
||||
var prompt = $"""
|
||||
Analyze these A/B test results. Look at conversion rates, impression counts,
|
||||
and event distributions across variants. Determine if there is a statistically
|
||||
significant winner.
|
||||
|
||||
Experiment:
|
||||
{GetContextValue(ctx, "experiment")}
|
||||
|
||||
Variants with metrics:
|
||||
{GetContextValue(ctx, "variants")}
|
||||
""";
|
||||
|
||||
var response = await conversation.SendForStructuredOutputAsync<ABTestAnalyzeResult>(
|
||||
prompt, "AnalyzeABTestResults", cancellationToken);
|
||||
|
||||
var json = SerializeStructuredOutput(response.StructuredOutput);
|
||||
|
||||
return new ExecuteTaskResponse
|
||||
{
|
||||
Success = true,
|
||||
Result = json
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Fallback for tasks that don't have a dedicated plugin.
|
||||
/// Uses generic SendAsync with free-form text response.
|
||||
|
||||
Reference in New Issue
Block a user