From 1385cbafa99ec5cba2dfa3b81c4d6755d0406be0 Mon Sep 17 00:00:00 2001 From: logikonline Date: Sat, 7 Mar 2026 15:57:28 -0500 Subject: [PATCH] feat(ai): add AI plugins for A/B test generation and analysis Implement AI-powered A/B testing capabilities for landing pages: - ABTestGeneratePlugin: creates experiment variants with config overrides - ABTestAnalyzePlugin: evaluates results and determines statistical significance - Generate 1-3 test variants focusing on high-impact changes (headlines, CTAs, value props) - Analyze conversion rates with 95% confidence threshold - Require minimum 100 impressions per variant before declaring winner - Return structured recommendations for next actions --- .../Plugins/ABTestAnalyzePlugin.cs | 79 ++++++++++++++++ .../Plugins/ABTestGeneratePlugin.cs | 74 +++++++++++++++ .../Services/WorkflowService.cs | 91 +++++++++++++++++++ 3 files changed, 244 insertions(+) create mode 100644 src/GitCaddy.AI.Service/Plugins/ABTestAnalyzePlugin.cs create mode 100644 src/GitCaddy.AI.Service/Plugins/ABTestGeneratePlugin.cs diff --git a/src/GitCaddy.AI.Service/Plugins/ABTestAnalyzePlugin.cs b/src/GitCaddy.AI.Service/Plugins/ABTestAnalyzePlugin.cs new file mode 100644 index 0000000..415a1c8 --- /dev/null +++ b/src/GitCaddy.AI.Service/Plugins/ABTestAnalyzePlugin.cs @@ -0,0 +1,79 @@ +// Copyright 2026 MarketAlly. All rights reserved. +// SPDX-License-Identifier: BSL-1.1 + +using System.Text.Json.Serialization; +using MarketAlly.AIPlugin; + +namespace GitCaddy.AI.Service.Plugins; + +/// +/// Result DTO for A/B test analysis. +/// +public class ABTestAnalyzeResult +{ + [JsonPropertyName("status")] + public string Status { get; set; } + + [JsonPropertyName("winner_variant_id")] + public long WinnerVariantId { get; set; } + + [JsonPropertyName("confidence")] + public double Confidence { get; set; } + + [JsonPropertyName("summary")] + public string Summary { get; set; } + + [JsonPropertyName("recommendation")] + public string Recommendation { get; set; } +} + +/// +/// AI plugin that analyzes A/B test experiment results. +/// Evaluates conversion rates, impression counts, and event distributions +/// to determine if there is a statistically significant winner. +/// +[AIPlugin("AnalyzeABTestResults", + "Analyze A/B test experiment results and determine the outcome. " + + "Evaluate conversion rates, impression counts, and statistical significance. " + + "Require at least 100 impressions per variant before declaring a winner. " + + "Use a minimum 95% confidence threshold.")] +public class ABTestAnalyzePlugin : IAIPlugin +{ + [AIParameter("Analysis status: must be exactly one of 'winner', 'needs_more_data', or 'no_difference'", required: true)] + public string Status { get; set; } + + [AIParameter("ID of the winning variant (use 0 if no winner)", required: true)] + public long WinnerVariantId { get; set; } + + [AIParameter("Statistical confidence level from 0.0 to 1.0", required: true)] + public double Confidence { get; set; } + + [AIParameter("Brief human-readable summary of the results", required: true)] + public string Summary { get; set; } + + [AIParameter("Actionable recommendation for what to do next", required: true)] + public string Recommendation { get; set; } + + public IReadOnlyDictionary SupportedParameters => new Dictionary + { + ["status"] = typeof(string), + ["winnervariantid"] = typeof(long), + ["confidence"] = typeof(double), + ["summary"] = typeof(string), + ["recommendation"] = typeof(string) + }; + + public Task ExecuteAsync(IReadOnlyDictionary parameters) + { + var result = new ABTestAnalyzeResult + { + Status = Status, + WinnerVariantId = WinnerVariantId, + Confidence = Confidence, + Summary = Summary, + Recommendation = Recommendation + }; + + return Task.FromResult(new AIPluginResult(result, "A/B test analysis complete")); + } +} diff --git a/src/GitCaddy.AI.Service/Plugins/ABTestGeneratePlugin.cs b/src/GitCaddy.AI.Service/Plugins/ABTestGeneratePlugin.cs new file mode 100644 index 0000000..0528acb --- /dev/null +++ b/src/GitCaddy.AI.Service/Plugins/ABTestGeneratePlugin.cs @@ -0,0 +1,74 @@ +// Copyright 2026 MarketAlly. All rights reserved. +// SPDX-License-Identifier: BSL-1.1 + +using System.Text.Json.Serialization; +using MarketAlly.AIPlugin; + +namespace GitCaddy.AI.Service.Plugins; + +/// +/// A/B test variant generated by AI, with a partial config override. +/// +public class ABTestVariant +{ + [JsonPropertyName("name")] + public string Name { get; set; } + + /// + /// Partial LandingConfig fields to override for this variant. + /// Stored as object to preserve arbitrary nested JSON structure. + /// + [JsonPropertyName("config_override")] + public object ConfigOverride { get; set; } + + [JsonPropertyName("weight")] + public int Weight { get; set; } +} + +/// +/// Result DTO for A/B test experiment generation. +/// +public class ABTestGenerateResult +{ + [JsonPropertyName("name")] + public string Name { get; set; } + + [JsonPropertyName("variants")] + public List Variants { get; set; } +} + +/// +/// AI plugin that generates A/B test experiment variants for a landing page. +/// The AI analyzes the current landing page config and produces meaningful +/// variant overrides to test (headlines, CTAs, value props, etc.). +/// +[AIPlugin("GenerateABTestExperiment", + "Generate an A/B test experiment for a landing page. " + + "Analyze the current landing page config and create meaningful test variants " + + "that focus on high-impact changes like headlines, CTAs, and value propositions. " + + "Do NOT include a control variant — it is added automatically.")] +public class ABTestGeneratePlugin : IAIPlugin +{ + [AIParameter("Short descriptive name for the experiment (e.g. 'Headline Impact Test')", required: true)] + public string ExperimentName { get; set; } + + [AIParameter("Test variants to create. Each must have 'name' (string), 'config_override' (object with partial landing config fields to override), and 'weight' (integer traffic percentage, should sum to ~50 since control gets 50%). Generate 1-3 variants with meaningfully different but plausible changes.", required: true)] + public List Variants { get; set; } + + public IReadOnlyDictionary SupportedParameters => new Dictionary + { + ["experimentname"] = typeof(string), + ["variants"] = typeof(List) + }; + + public Task ExecuteAsync(IReadOnlyDictionary parameters) + { + var result = new ABTestGenerateResult + { + Name = ExperimentName, + Variants = Variants ?? new List() + }; + + return Task.FromResult(new AIPluginResult(result, "A/B test experiment generated")); + } +} diff --git a/src/GitCaddy.AI.Service/Services/WorkflowService.cs b/src/GitCaddy.AI.Service/Services/WorkflowService.cs index ed97c80..720a09f 100644 --- a/src/GitCaddy.AI.Service/Services/WorkflowService.cs +++ b/src/GitCaddy.AI.Service/Services/WorkflowService.cs @@ -124,6 +124,8 @@ public class WorkflowService : IWorkflowService { "landing_page_generate" => await ExecuteLandingPageGenerateAsync(request, cancellationToken), "landing_page_translate" => await ExecuteLandingPageTranslateAsync(request, cancellationToken), + "ab_test_generate" => await ExecuteABTestGenerateAsync(request, cancellationToken), + "ab_test_analyze" => await ExecuteABTestAnalyzeAsync(request, cancellationToken), _ => await ExecuteGenericTaskAsync(request, cancellationToken) }; } @@ -231,6 +233,95 @@ public class WorkflowService : IWorkflowService }; } + /// + /// Generates A/B test experiment variants using structured output via the ABTestGeneratePlugin. + /// + private async Task ExecuteABTestGenerateAsync( + ExecuteTaskRequest request, CancellationToken cancellationToken) + { + _logger.LogInformation("Generating A/B test experiment via structured output plugin"); + + var conversation = _providerFactory.CreateConversation() + .WithSystemPrompt(""" + You are a conversion rate optimization expert. Analyze landing page configurations + and create meaningful A/B test experiments that focus on high-impact changes. + Generate variants that test different headlines, CTAs, value propositions, or layouts. + Each variant should be meaningfully different but plausible. + The control variant (original config) is added automatically — do NOT include it. + """) + .RegisterPlugin() + .Build(); + + var ctx = request.Context; + var prompt = $""" + Analyze this landing page config and create an A/B test experiment. + Focus on high-impact changes: headlines, CTAs, value propositions. + Generate 1-3 test variants. Each variant's config_override should be a partial + LandingConfig with only the fields that differ from the control. + + Repository: {GetContextValue(ctx, "repo_name")} + Description: {GetContextValue(ctx, "repo_description")} + + Current landing page config: + {GetContextValue(ctx, "landing_config")} + """; + + var response = await conversation.SendForStructuredOutputAsync( + prompt, "GenerateABTestExperiment", cancellationToken); + + var json = SerializeStructuredOutput(response.StructuredOutput); + + return new ExecuteTaskResponse + { + Success = true, + Result = json + }; + } + + /// + /// Analyzes A/B test experiment results using structured output via the ABTestAnalyzePlugin. + /// + private async Task ExecuteABTestAnalyzeAsync( + ExecuteTaskRequest request, CancellationToken cancellationToken) + { + _logger.LogInformation("Analyzing A/B test results via structured output plugin"); + + var conversation = _providerFactory.CreateConversation() + .WithSystemPrompt(""" + You are a data analyst specializing in A/B test evaluation. + Analyze experiment results using statistical methods to determine significance. + Require at least 100 impressions per variant before declaring a winner. + Use a minimum 95% confidence threshold for statistical significance. + Be conservative — only declare a winner when the data clearly supports it. + """) + .RegisterPlugin() + .Build(); + + var ctx = request.Context; + var prompt = $""" + Analyze these A/B test results. Look at conversion rates, impression counts, + and event distributions across variants. Determine if there is a statistically + significant winner. + + Experiment: + {GetContextValue(ctx, "experiment")} + + Variants with metrics: + {GetContextValue(ctx, "variants")} + """; + + var response = await conversation.SendForStructuredOutputAsync( + prompt, "AnalyzeABTestResults", cancellationToken); + + var json = SerializeStructuredOutput(response.StructuredOutput); + + return new ExecuteTaskResponse + { + Success = true, + Result = json + }; + } + /// /// Fallback for tasks that don't have a dedicated plugin. /// Uses generic SendAsync with free-form text response.