From 98a47f75aa25ba5fc0fb26611ebd89b1faabb4dc Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Fri, 19 Jun 2026 14:25:28 -0700 Subject: [PATCH 1/4] Add modify command for targeted documentation changes Co-authored-by: Jonathan Molinatto Co-authored-by: Quan Nguyen Co-authored-by: Cursor --- README.md | 9 +- cmd/update.go | 1 + cmd/updatedocumentation.go | 97 ++++- internal/llmagent/docagent/docagent.go | 380 +++++++++++++++++- internal/llmagent/docagent/interactive.go | 38 +- .../llmagent/docagent/modificationanalyzer.go | 200 +++++++++ internal/llmagent/docagent/prompts/loader.go | 6 + .../prompts/modification_analysis_prompt.txt | 59 +++ .../docagent/prompts/modification_prompt.txt | 53 +++ .../llmagent/docagent/prompts/resources.go | 6 + internal/llmagent/docagent/promptsbuilder.go | 45 +++ tools/readme/readme.md.tmpl | 2 +- 12 files changed, 863 insertions(+), 33 deletions(-) create mode 100644 internal/llmagent/docagent/modificationanalyzer.go create mode 100644 internal/llmagent/docagent/prompts/modification_analysis_prompt.txt create mode 100644 internal/llmagent/docagent/prompts/modification_prompt.txt diff --git a/README.md b/README.md index 13a1318677..bac47a7645 100644 --- a/README.md +++ b/README.md @@ -746,7 +746,12 @@ _Context: package_ Use this command to update package documentation using an AI agent or to get manual instructions for update. -The AI agent analyzes your package structure, data streams, and configuration, and generates a new documentation file based on the template and the package context. +The AI agent supports two modes: +1. Rewrite mode (default): Full documentation regeneration + - Analyzes your package structure, data streams, and configuration, and generates a new documentation file based on the template and the package context. +2. Modify mode: Targeted documentation changes + - The LLM will perform a targeted change to the documentation, based on user-provided instructions. + - Use --modify-prompt flag to provide instructions for non-interactive modifications For packages with multiple documentation files, the user can specify which file to update in interactive mode, or use the --doc-file flag to specify the file to update in non-interactive mode. @@ -852,7 +857,7 @@ The following settings are available per profile: ### AI-powered Documentation Configuration -The `elastic-package update documentation` command can generate or update package documentation using an LLM. +The `elastic-package update documentation` command can generate or update package documentation using an LLM. It supports full rewrites and targeted modifications. For full details on configuration, LLM provider setup, and the optional service knowledge base, see [docs/howto/ai_documentation.md](docs/howto/ai_documentation.md). diff --git a/cmd/update.go b/cmd/update.go index 0a1e363110..e9ea1695f3 100644 --- a/cmd/update.go +++ b/cmd/update.go @@ -26,6 +26,7 @@ func setupUpdateCommand() *cobraext.Command { RunE: updateDocumentationCommandAction, } updateDocumentationCmd.Flags().Bool("non-interactive", false, "run in non-interactive mode, accepting the first result from the LLM") + updateDocumentationCmd.Flags().String("modify-prompt", "", "modification instructions for targeted documentation changes (skips full rewrite)") updateDocumentationCmd.Flags().String("doc-file", "", "specify which markdown file to update (e.g., README.md, vpc.md). Defaults to README.md") cmd := &cobra.Command{ diff --git a/cmd/updatedocumentation.go b/cmd/updatedocumentation.go index 5c8fb33f3e..53253ba22e 100644 --- a/cmd/updatedocumentation.go +++ b/cmd/updatedocumentation.go @@ -24,7 +24,12 @@ import ( const updateDocumentationLongDescription = `Use this command to update package documentation using an AI agent or to get manual instructions for update. -The AI agent analyzes your package structure, data streams, and configuration, and generates a new documentation file based on the template and the package context. +The AI agent supports two modes: +1. Rewrite mode (default): Full documentation regeneration + - Analyzes your package structure, data streams, and configuration, and generates a new documentation file based on the template and the package context. +2. Modify mode: Targeted documentation changes + - The LLM will perform a targeted change to the documentation, based on user-provided instructions. + - Use --modify-prompt flag to provide instructions for non-interactive modifications For packages with multiple documentation files, the user can specify which file to update in interactive mode, or use the --doc-file flag to specify the file to update in non-interactive mode. @@ -34,6 +39,12 @@ Configuration options for LLM providers (environment variables or profile config - ELASTIC_PACKAGE_LLM_PROVIDER / llm.provider: Provider name (only Gemini provider currently supported). - Gemini: GOOGLE_API_KEY / llm.gemini.api_key, GEMINI_MODEL / llm.gemini.model, GEMINI_THINKING_BUDGET / llm.gemini.thinking_budget` +const ( + modePromptRewrite = "Rewrite (full regeneration)" + modePromptModify = "Modify (targeted changes)" +) + + // discoverDocumentationTemplates finds all .md files in _dev/build/docs/ func discoverDocumentationTemplates(packageRoot string) ([]string, error) { docsDir := filepath.Join(packageRoot, "_dev", "build", "docs") @@ -53,10 +64,12 @@ func discoverDocumentationTemplates(packageRoot string) ([]string, error) { } } + // If no files found, return README.md as default if len(mdFiles) == 0 { return []string{"README.md"}, nil } + // Sort with README.md first, others alphabetically sort.Slice(mdFiles, func(i, j int) bool { if mdFiles[i] == "README.md" { return true @@ -72,11 +85,13 @@ func discoverDocumentationTemplates(packageRoot string) ([]string, error) { // selectDocumentationFile determines which documentation file to update func selectDocumentationFile(cmd *cobra.Command, packageRoot string, nonInteractive bool) (string, error) { + // Check if --doc-file flag was provided docFile, err := cmd.Flags().GetString("doc-file") if err != nil { return "", fmt.Errorf("failed to get doc-file flag: %w", err) } + // If flag is provided, validate its an .md file and use it if docFile != "" { if filepath.Ext(docFile) != ".md" { return "", fmt.Errorf("doc-file must be a .md file, got: %s", docFile) @@ -87,11 +102,13 @@ func selectDocumentationFile(cmd *cobra.Command, packageRoot string, nonInteract return docFile, nil } + // Discover available markdown files mdFiles, err := discoverDocumentationTemplates(packageRoot) if err != nil { return "", err } + // If only one file, use it (no prompt) if len(mdFiles) == 1 { return mdFiles[0], nil } @@ -130,6 +147,66 @@ func printNoProviderInstructions(cmd *cobra.Command) { cmd.Println(tui.Info(" - For Gemini: set GOOGLE_API_KEY or llm.gemini.api_key in your elastic-package profile config")) } +// standardModeFlags holds CLI flags for standard (non-evaluation) mode +type standardModeFlags struct { + nonInteractive bool + modifyPrompt string +} + +// getStandardModeFlags extracts CLI flags needed for standard mode +func getStandardModeFlags(cmd *cobra.Command) (standardModeFlags, error) { + var flags standardModeFlags + var err error + + flags.nonInteractive, err = cmd.Flags().GetBool("non-interactive") + if err != nil { + return flags, fmt.Errorf("failed to get non-interactive flag: %w", err) + } + + flags.modifyPrompt, err = cmd.Flags().GetString("modify-prompt") + if err != nil { + return flags, fmt.Errorf("failed to get modify-prompt flag: %w", err) + } + + return flags, nil +} + +// selectUpdateMode prompts the user to select rewrite or modify mode, or determines it automatically +func selectUpdateMode(cmd *cobra.Command, nonInteractive bool, modifyPrompt string) (useModifyMode bool, err error) { + cmd.Println("Updating documentation using AI agent...") + + if nonInteractive || modifyPrompt != "" { + return modifyPrompt != "", nil + } + + modePrompt := tui.NewSelect("Do you want to rewrite or modify the documentation?", []string{ + modePromptRewrite, + modePromptModify, + }, modePromptRewrite) + + var mode string + if err := tui.AskOne(modePrompt, &mode); err != nil { + return false, fmt.Errorf("prompt failed: %w", err) + } + + return mode == modePromptModify, nil +} + +// runDocumentationUpdate executes the documentation update using the agent +func runDocumentationUpdate(cmd *cobra.Command, docAgent *docagent.DocumentationAgent, useModifyMode, nonInteractive bool, modifyPrompt string) error { + if useModifyMode { + if err := docAgent.ModifyDocumentation(cmd.Context(), nonInteractive, modifyPrompt); err != nil { + return fmt.Errorf("documentation modification failed: %w", err) + } + } else { + if err := docAgent.UpdateDocumentation(cmd.Context(), nonInteractive); err != nil { + return fmt.Errorf("documentation update failed: %w", err) + } + } + return nil +} + + func updateDocumentationCommandAction(cmd *cobra.Command, _ []string) error { p, err := cobraext.GetProfileFlag(cmd) if err != nil { @@ -147,9 +224,9 @@ func handleStandardMode(cmd *cobra.Command, p *profile.Profile, cfg llmconfig.LL return fmt.Errorf("locating package root failed: %w", err) } - nonInteractive, err := cmd.Flags().GetBool("non-interactive") + flags, err := getStandardModeFlags(cmd) if err != nil { - return fmt.Errorf("failed to get non-interactive flag: %w", err) + return err } if cfg.APIKey == "" { @@ -163,11 +240,11 @@ func handleStandardMode(cmd *cobra.Command, p *profile.Profile, cfg llmconfig.LL cmd.Printf("Using LLM model \"%s\"\n", cfg.ModelID) } - targetDocFile, err := selectDocumentationFile(cmd, packageRoot, nonInteractive) + targetDocFile, err := selectDocumentationFile(cmd, packageRoot, flags.nonInteractive) if err != nil { return fmt.Errorf("failed to select documentation file: %w", err) } - if !nonInteractive && targetDocFile != "README.md" { + if !flags.nonInteractive && targetDocFile != "README.md" { cmd.Printf("Selected documentation file: %s\n", targetDocFile) } @@ -194,9 +271,11 @@ func handleStandardMode(cmd *cobra.Command, p *profile.Profile, cfg llmconfig.LL return fmt.Errorf("failed to create documentation agent: %w", err) } - cmd.Println("Updating documentation using AI agent...") - if err := docAgent.UpdateDocumentation(cmd.Context(), nonInteractive); err != nil { - return fmt.Errorf("documentation update failed: %w", err) + useModifyMode, err := selectUpdateMode(cmd, flags.nonInteractive, flags.modifyPrompt) + if err != nil { + return err } - return nil + + return runDocumentationUpdate(cmd, docAgent, useModifyMode, flags.nonInteractive, flags.modifyPrompt) } + diff --git a/internal/llmagent/docagent/docagent.go b/internal/llmagent/docagent/docagent.go index 5569dc240c..7be903c435 100644 --- a/internal/llmagent/docagent/docagent.go +++ b/internal/llmagent/docagent/docagent.go @@ -6,6 +6,7 @@ package docagent import ( "context" + "errors" "fmt" "os" "path/filepath" @@ -24,6 +25,7 @@ import ( "github.com/elastic/elastic-package/internal/packages" "github.com/elastic/elastic-package/internal/packages/archetype" "github.com/elastic/elastic-package/internal/profile" + "github.com/elastic/elastic-package/internal/tui" ) const ( @@ -70,8 +72,10 @@ type TaskResult = executor.TaskResult type PromptType = prompts.Type const ( - PromptTypeRevision = prompts.TypeRevision - PromptTypeSectionGeneration = prompts.TypeSectionGeneration + PromptTypeRevision = prompts.TypeRevision + PromptTypeSectionGeneration = prompts.TypeSectionGeneration + PromptTypeModificationAnalysis = prompts.TypeModificationAnalysis + PromptTypeModification = prompts.TypeModification ) // DocumentationAgent handles documentation updates for packages @@ -110,7 +114,7 @@ type SectionGenerationContext struct { // AgentConfig holds configuration for creating a DocumentationAgent type AgentConfig struct { - Provider string // LLM provider (e.g. gemini); default gemini + Provider string // LLM provider (e.g. gemini); default gemini APIKey string ModelID string PackageRoot string @@ -286,35 +290,137 @@ func (d *DocumentationAgent) UpdateDocumentation(ctx context.Context, nonInterac // In interactive mode, allow review if !nonInteractive { - return d.runInteractiveSectionReview(ctx) + return d.runInteractiveSectionReview(ctx, sections) } return nil } -// runInteractiveSectionReview allows user to review generated documentation in interactive mode -func (d *DocumentationAgent) runInteractiveSectionReview(ctx context.Context) error { - _ = ctx // reserved for future use +// ModifyDocumentation runs the documentation modification process for targeted changes using section-based approach +func (d *DocumentationAgent) ModifyDocumentation(ctx context.Context, nonInteractive bool, modifyPrompt string) error { + ctx, sessionSpan := tracing.StartSessionSpan(ctx, "doc:modify", d.executor.ModelID(), d.executor.Provider()) + var sessionOutput string + defer func() { + tracing.EndSessionSpan(ctx, sessionSpan, sessionOutput) + }() + + d.printTracingSessionID(ctx) + + // Check if documentation file exists + if _, err := os.Stat(d.docPath()); err != nil { + if os.IsNotExist(err) { + return fmt.Errorf("cannot modify documentation: %s does not exist at _dev/build/docs/%s", d.targetDocFile, d.targetDocFile) + } + return fmt.Errorf("failed to check %s: %w", d.targetDocFile, err) + } + + var instructions string + if modifyPrompt != "" { + instructions = modifyPrompt + } else if !nonInteractive { + var err error + instructions, err = tui.AskTextArea("What changes would you like to make to the documentation?") + if err != nil { + if errors.Is(err, tui.ErrCancelled) { + fmt.Println("āš ļø Modification cancelled.") + return nil + } + return fmt.Errorf("prompt failed: %w", err) + } + if strings.TrimSpace(instructions) == "" { + return fmt.Errorf("no modification instructions provided") + } + } else { + return fmt.Errorf("--modify-prompt flag is required in non-interactive mode") + } + + // Record initial input for session + inputDesc := fmt.Sprintf("Modify documentation for package: %s (file: %s) - Request: %s", d.manifest.Name, d.targetDocFile, instructions) + tracing.RecordSessionInput(sessionSpan, inputDesc) + + // Confirm LLM understands the documentation guidelines before proceeding + if err := d.ConfirmInstructionsUnderstood(ctx); err != nil { + return fmt.Errorf("instruction confirmation failed: %w", err) + } + + // Backup original README content before making any changes + d.backupOriginalReadme() + + fmt.Println("šŸ“ Analyzing modification request...") + + // Parse existing documentation into sections + existingContent, err := d.readCurrentReadme() + if err != nil { + return fmt.Errorf("failed to read current documentation: %w", err) + } + existingSections := parsing.ParseSections(existingContent) + + if len(existingSections) == 0 { + return fmt.Errorf("no sections found in existing documentation") + } + + // Get template sections for reference (structure) + templateContent := archetype.GetPackageDocsReadmeTemplate() + templateSections := parsing.ParseSections(templateContent) + + // Analyze modification scope + scope, err := d.analyzeModificationScope(ctx, instructions, templateSections) + if err != nil { + logger.Debugf("Scope analysis failed, defaulting to global: %v", err) + scope = defaultModificationScope("Scope analysis failed, defaulting to global") + } - // Display the generated documentation - if err := d.displayReadme(); err != nil { - logger.Debugf("could not display readme: %v", err) + // Report scope to user + fmt.Printf("āœ“ Scope: %s", scope.Type) + if scope.Type == ScopeSpecific { + fmt.Printf(" (sections: %s)", strings.Join(scope.AffectedSections, ", ")) + } + if scope.Confidence < 0.7 { + fmt.Printf(" [confidence: %.0f%%]", scope.Confidence*100) + } + fmt.Println() + if scope.Reasoning != "" { + logger.Debugf("Scope reasoning: %s", scope.Reasoning) } - // Get user action - action, err := d.getUserAction() + // Apply modifications based on scope + finalSections, err := d.applyModificationScope(ctx, existingSections, scope, instructions) if err != nil { - return err + return fmt.Errorf("failed to modify sections: %w", err) } - actionResult := d.handleUserAction(action, true) - if actionResult.Err != nil { - return actionResult.Err + // Combine and write + finalContent := parsing.CombineSections(finalSections) + sessionOutput = fmt.Sprintf("Modified %d sections, %d characters for %s", len(finalSections), len(finalContent), d.targetDocFile) + + if err := d.writeDocumentation(d.docPath(), finalContent); err != nil { + return fmt.Errorf("failed to write documentation: %w", err) + } + + fmt.Printf("\nāœ… Documentation modified successfully! (%d characters)\n", len(finalContent)) + fmt.Printf("šŸ“„ Written to: _dev/build/docs/%s\n", d.targetDocFile) + + // In interactive mode, allow review + if !nonInteractive { + return d.runInteractiveSectionReview(ctx, finalSections) } return nil } +// logAgentResponse logs debug information about the agent response +func (d *DocumentationAgent) logAgentResponse(result *TaskResult) { + logger.Debugf("DEBUG: Full agent task response follows (may contain sensitive content)") + logger.Debugf("Agent task response - Success: %t", result.Success) + logger.Debugf("Agent task response - FinalContent: %s", result.FinalContent) + logger.Debugf("Agent task response - Conversation entries: %d", len(result.Conversation)) + for i, entry := range result.Conversation { + logger.Debugf("Agent task response - Conversation[%d]: type=%s, content_length=%d", + i, entry.Type, len(entry.Content)) + logger.Tracef("Agent task response - Conversation[%d]: content=%s", i, entry.Content) + } +} + // NewResponseAnalyzer creates a new ResponseAnalyzer with default patterns // // These responses should be chosen to represent LLM responses to states, but are unlikely to appear in generated @@ -452,11 +558,253 @@ func (d *DocumentationAgent) writeDocumentation(path, content string) error { return nil } +// defaultModificationScope returns a default global scope when analysis fails. +func defaultModificationScope(reason string) *ModificationScope { + s := &ModificationScope{Type: ScopeGlobal, Confidence: 0.5} + if reason != "" { + s.Reasoning = reason + } + return s +} + +// applyModificationScope applies modifications to sections based on scope. +func (d *DocumentationAgent) applyModificationScope(ctx context.Context, existingSections []Section, scope *ModificationScope, instructions string) ([]Section, error) { + switch scope.Type { + case ScopeGlobal, ScopeAmbiguous: + if scope.Type == ScopeAmbiguous { + fmt.Println("āš ļø Scope is ambiguous, modifying all sections to be safe") + } + fmt.Printf("šŸ“ Modifying all %d sections...\n", len(existingSections)) + return d.modifyAllSections(ctx, existingSections, instructions) + case ScopeSpecific: + fmt.Printf("šŸ“ Modifying %d of %d sections...\n", len(scope.AffectedSections), len(existingSections)) + return d.modifySpecificSections(ctx, existingSections, scope.AffectedSections, instructions) + default: + return nil, fmt.Errorf("unexpected scope type: %v", scope.Type) + } +} + +// runInteractiveSectionReview allows user to review and request changes in interactive mode +func (d *DocumentationAgent) runInteractiveSectionReview(ctx context.Context, sections []Section) error { + for { + // Display the generated documentation + if err := d.displayReadme(); err != nil { + logger.Debugf("could not display readme: %v", err) + } + + // Get user action + action, err := d.getUserAction() + if err != nil { + return err + } + + actionResult := d.handleUserAction(action, true) + if actionResult.Err != nil { + return actionResult.Err + } + + // If user doesn't want to continue, exit + if !actionResult.ShouldContinue { + return nil + } + + // User requested changes - run modification workflow + if actionResult.Changes == "" { + continue // No changes provided, loop again + } + + fmt.Println("šŸ“ Applying requested changes...") + + // Re-read current sections from file + existingContent, err := d.readCurrentReadme() + if err != nil { + return fmt.Errorf("failed to read current documentation: %w", err) + } + sectionsFromFile := parsing.ParseSections(existingContent) + + // Analyze modification scope + templateContent := archetype.GetPackageDocsReadmeTemplate() + templateSections := parsing.ParseSections(templateContent) + + scope, err := d.analyzeModificationScope(ctx, actionResult.Changes, templateSections) + if err != nil { + logger.Debugf("Scope analysis failed, defaulting to global: %v", err) + scope = defaultModificationScope("") + } + + // Apply modifications + modifiedSections, err := d.applyModificationScope(ctx, sectionsFromFile, scope, actionResult.Changes) + if err != nil { + return fmt.Errorf("failed to modify sections: %w", err) + } + + // Write updated documentation + finalContent := parsing.CombineSections(modifiedSections) + if err := d.writeDocumentation(d.docPath(), finalContent); err != nil { + return fmt.Errorf("failed to write documentation: %w", err) + } + + fmt.Printf("\nāœ… Changes applied! (%d characters)\n", len(finalContent)) + } +} + +// modifyAllSections regenerates all sections with modification context +func (d *DocumentationAgent) modifyAllSections(ctx context.Context, existingSections []Section, modificationPrompt string) ([]Section, error) { + var modifiedSections []Section + + for i, section := range existingSections { + fmt.Printf("šŸ“ Modifying section %d/%d: %s\n", i+1, len(existingSections), section.Title) + + // Build modification prompt for this section + promptCtx := PromptContext{ + Manifest: d.manifest, + TargetDocFile: d.targetDocFile, + Changes: modificationPrompt, + SectionTitle: section.Title, + SectionLevel: section.Level, + TemplateSection: section.Content, + } + + prompt := d.buildPrompt(PromptTypeModification, promptCtx) + + // Generate modified section + modifiedSection, err := d.generateModifiedSection(ctx, section, prompt) + if err != nil { + logger.Debugf("Failed to modify section %s: %v", section.Title, err) + // On error, keep the original section + modifiedSections = append(modifiedSections, section) + continue + } + + modifiedSections = append(modifiedSections, modifiedSection) + } + + return modifiedSections, nil +} + +// modifySpecificSections regenerates only affected sections +// For hierarchical sections, if a subsection is affected, the entire parent section is regenerated +func (d *DocumentationAgent) modifySpecificSections(ctx context.Context, existingSections []Section, affectedSectionTitles []string, modificationPrompt string) ([]Section, error) { + var finalSections []Section + modifiedCount := 0 + + for _, section := range existingSections { + // Check if this section or any of its subsections are affected + isAffected := isSectionAffected(section.Title, affectedSectionTitles) + + // Check subsections - if any subsection is affected, modify the parent + if !isAffected { + for _, subsection := range section.Subsections { + if isSectionAffected(subsection.Title, affectedSectionTitles) { + isAffected = true + logger.Debugf("Subsection %s is affected, will regenerate parent section %s", subsection.Title, section.Title) + break + } + } + } + + if isAffected { + modifiedCount++ + fmt.Printf("šŸ“ Modifying section %d/%d: %s", modifiedCount, len(affectedSectionTitles), section.Title) + if section.HasSubsections() { + fmt.Printf(" (with %d subsections)", len(section.Subsections)) + } + fmt.Println() + + // Build modification prompt for this section (use FullContent for hierarchical context) + promptCtx := PromptContext{ + Manifest: d.manifest, + TargetDocFile: d.targetDocFile, + Changes: modificationPrompt, + SectionTitle: section.Title, + SectionLevel: section.Level, + TemplateSection: section.GetAllContent(), // Include subsections in context + } + + prompt := d.buildPrompt(PromptTypeModification, promptCtx) + + // Generate modified section (includes subsections) + modifiedSection, err := d.generateModifiedSection(ctx, section, prompt) + if err != nil { + logger.Debugf("Failed to modify section %s: %v", section.Title, err) + // On error, keep the original section + finalSections = append(finalSections, section) + continue + } + + // Parse the generated content to extract hierarchical structure + parsedModified := parsing.ParseSections(modifiedSection.Content) + if len(parsedModified) > 0 { + modifiedSection = parsedModified[0] // Take the full hierarchical section + } + + finalSections = append(finalSections, modifiedSection) + } else { + // Preserve entire section unchanged (including subsections) + finalSections = append(finalSections, section) + } + } + + preservedCount := len(existingSections) - modifiedCount + fmt.Printf("āœ“ Modified: %d sections, Preserved: %d sections\n", modifiedCount, preservedCount) + + return finalSections, nil +} + // placeholderSectionContent returns markdown for a section that couldn't be populated. func placeholderSectionContent(sectionTitle string) string { return fmt.Sprintf("## %s\n\n%s", sectionTitle, emptySectionPlaceholder) } +// extractGeneratedSectionContent extracts the generated section content from the LLM response +func (d *DocumentationAgent) extractGeneratedSectionContent(result *TaskResult, sectionTitle string) string { + content := result.FinalContent + if strings.TrimSpace(content) == "" { + logger.Warnf("LLM returned empty response for section: %s", sectionTitle) + } + return parsing.ExtractSectionFromLLMResponse(content, sectionTitle, emptySectionPlaceholder) +} + +// generateModifiedSection generates a modified version of a section using the LLM +func (d *DocumentationAgent) generateModifiedSection(ctx context.Context, originalSection Section, prompt string) (Section, error) { + // Execute the task + result, err := d.executor.ExecuteTask(ctx, prompt) + if err != nil { + return Section{}, fmt.Errorf("agent task failed: %w", err) + } + + // Log the result + d.logAgentResponse(result) + + // Analyze the response + analysis := d.responseAnalyzer.AnalyzeResponse(result.FinalContent, result.Conversation) + if analysis.Status == responseError { + return Section{}, fmt.Errorf("LLM reported an error: %s", analysis.Message) + } + + // Extract the generated content + generatedContent := d.extractGeneratedSectionContent(result, originalSection.Title) + + // Create the modified section + modifiedSection := Section{ + Title: originalSection.Title, + Level: originalSection.Level, + Content: generatedContent, + StartLine: originalSection.StartLine, + EndLine: originalSection.EndLine, + } + + return modifiedSection, nil +} + +// sectionResult holds the result of generating a single section +type sectionResult struct { + index int + section Section + err error +} + + // loadTemplateExampleExistingSections loads template, example, and existing doc sections and returns top-level template sections. func (d *DocumentationAgent) loadTemplateExampleExistingSections() (topLevelSections, exampleSections, existingSections []Section, err error) { templateSections := parsing.ParseSections(archetype.GetPackageDocsReadmeTemplate()) diff --git a/internal/llmagent/docagent/interactive.go b/internal/llmagent/docagent/interactive.go index ea71119bc4..c35ff9a04f 100644 --- a/internal/llmagent/docagent/interactive.go +++ b/internal/llmagent/docagent/interactive.go @@ -5,6 +5,7 @@ package docagent import ( + "errors" "fmt" "strings" @@ -15,8 +16,9 @@ import ( ) const ( - ActionAccept = "Accept and finalize" - ActionCancel = "Cancel" + ActionAccept = "Accept and finalize" + ActionRequest = "Request changes" + ActionCancel = "Cancel" ActionTryAgain = "Try again" ActionExit = "Exit" @@ -25,7 +27,8 @@ const ( // ActionResult holds the result of a user action type ActionResult struct { NewPrompt string - ShouldContinue bool // true = continue loop, false = exit + Changes string // Raw user feedback for modifications + ShouldContinue bool // true = continue loop, false = exit Err error } @@ -33,6 +36,7 @@ type ActionResult struct { func (d *DocumentationAgent) getUserAction() (string, error) { selectPrompt := tui.NewSelect("What would you like to do?", []string{ ActionAccept, + ActionRequest, ActionCancel, }, ActionAccept) @@ -57,7 +61,7 @@ func (d *DocumentationAgent) displayReadme() error { renderedContent, shouldBeRendered, err := docs.GenerateReadme(d.repositoryRoot, d.targetDocFile, "", d.packageRoot, fields.NewSchemaURLs()) if err != nil || !shouldBeRendered { fmt.Printf("\nāš ļø The generated %s could not be rendered.\n", d.targetDocFile) - fmt.Println("It's recommended that you do not accept this version (cancel and try again).") + fmt.Println("It's recommended that you do not accept this version (ask for revisions or cancel).") return fmt.Errorf("could not render readme: %w", err) } @@ -68,7 +72,7 @@ func (d *DocumentationAgent) displayReadme() error { // Try to open in browser first if ui.TryBrowserPreview(processedContentStr) { fmt.Println("🌐 Opening documentation preview in your web browser...") - fmt.Println("šŸ’” Return here to accept or cancel.") + fmt.Println("šŸ’” Return here to accept or request changes.") } else { // Fallback to terminal display if browser preview fails title := fmt.Sprintf("šŸ“„ Processed %s (as generated by elastic-package build)", d.targetDocFile) @@ -111,6 +115,8 @@ func (d *DocumentationAgent) handleUserAction(action string, readmeUpdated bool) switch action { case ActionAccept: return d.handleAcceptAction(readmeUpdated) + case ActionRequest: + return d.handleRequestChanges() case ActionCancel: fmt.Println("āŒ Documentation update cancelled.") if err := d.restoreOriginalReadme(); err != nil { @@ -154,3 +160,25 @@ func (d *DocumentationAgent) handleAcceptAction(readmeUpdated bool) ActionResult newPrompt := d.buildPrompt(PromptTypeRevision, promptCtx) return ActionResult{NewPrompt: newPrompt, ShouldContinue: true} } + +// handleRequestChanges handles the "Request changes" action +func (d *DocumentationAgent) handleRequestChanges() ActionResult { + changes, err := tui.AskTextArea("What changes would you like to make to the documentation?") + if err != nil { + // Check if user cancelled + if errors.Is(err, tui.ErrCancelled) { + fmt.Println("āš ļø Changes request cancelled.") + return ActionResult{ShouldContinue: false} + } + return ActionResult{Err: fmt.Errorf("prompt failed: %w", err)} + } + + // Check if no changes were provided + if strings.TrimSpace(changes) == "" { + fmt.Println("āš ļø No changes specified.") + return ActionResult{ShouldContinue: false} + } + promptCtx := d.createPromptContext(d.manifest, changes) + newPrompt := d.buildPrompt(PromptTypeRevision, promptCtx) + return ActionResult{NewPrompt: newPrompt, Changes: changes, ShouldContinue: true} +} diff --git a/internal/llmagent/docagent/modificationanalyzer.go b/internal/llmagent/docagent/modificationanalyzer.go new file mode 100644 index 0000000000..7084591b5f --- /dev/null +++ b/internal/llmagent/docagent/modificationanalyzer.go @@ -0,0 +1,200 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package docagent + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/elastic/elastic-package/internal/logger" +) + +// ScopeType indicates the scope of a modification request +type ScopeType int + +const ( + // ScopeGlobal indicates the modification affects the entire document + ScopeGlobal ScopeType = iota + // ScopeSpecific indicates the modification affects specific sections + ScopeSpecific + // ScopeAmbiguous indicates the scope is unclear + ScopeAmbiguous +) + +// String returns the string representation of ScopeType +func (s ScopeType) String() string { + switch s { + case ScopeGlobal: + return "global" + case ScopeSpecific: + return "specific" + case ScopeAmbiguous: + return "ambiguous" + default: + return "unknown" + } +} + +// ModificationScope represents the analyzed scope of a modification request +type ModificationScope struct { + Type ScopeType // global, specific, or ambiguous + AffectedSections []string // Section titles to modify + Confidence float64 // 0.0 to 1.0 + Reasoning string // Explanation of classification +} + +// scopeAnalysisResponse is the expected JSON response from the LLM +type scopeAnalysisResponse struct { + Type string `json:"type"` + Sections []string `json:"sections"` + Confidence float64 `json:"confidence"` + Reasoning string `json:"reasoning"` +} + +// analyzeModificationScope determines which sections a modification request affects +func (d *DocumentationAgent) analyzeModificationScope(ctx context.Context, modificationPrompt string, availableSections []Section) (*ModificationScope, error) { + // Build list of section titles (including subsections) + sectionTitles := make([]string, 0) + for _, section := range availableSections { + sectionTitles = append(sectionTitles, section.Title) + for _, subsection := range section.Subsections { + sectionTitles = append(sectionTitles, subsection.Title) + } + } + + // Build the analysis prompt with hierarchical structure + prompt := d.buildScopeAnalysisPromptHierarchical(modificationPrompt, availableSections) + + // Execute the analysis + logger.Debugf("Analyzing modification scope for prompt: %s", modificationPrompt) + result, err := d.executor.ExecuteTask(ctx, prompt) + if err != nil { + logger.Debugf("Scope analysis failed, defaulting to global: %v", err) + return &ModificationScope{ + Type: ScopeGlobal, + AffectedSections: sectionTitles, + Confidence: 0.5, + Reasoning: "Analysis failed, defaulting to global scope", + }, nil + } + + // Parse the response + scope, err := d.parseScopeAnalysisResponse(result.FinalContent, sectionTitles) + if err != nil { + logger.Debugf("Failed to parse scope analysis, defaulting to global: %v", err) + return &ModificationScope{ + Type: ScopeGlobal, + AffectedSections: sectionTitles, + Confidence: 0.5, + Reasoning: "Failed to parse analysis, defaulting to global scope", + }, nil + } + + logger.Debugf("Scope analysis complete: type=%s, sections=%v, confidence=%.2f", scope.Type, scope.AffectedSections, scope.Confidence) + return scope, nil +} + +// buildScopeAnalysisPromptHierarchical creates the prompt for scope analysis with hierarchical structure +func (d *DocumentationAgent) buildScopeAnalysisPromptHierarchical(modificationPrompt string, sections []Section) string { + // Build numbered list of sections with subsections indented + var sectionsBuilder strings.Builder + counter := 1 + + for _, section := range sections { + sectionsBuilder.WriteString(fmt.Sprintf("%d. %s\n", counter, section.Title)) + counter++ + + // List subsections with indentation + for _, subsection := range section.Subsections { + sectionsBuilder.WriteString(fmt.Sprintf(" %d. %s (subsection of %s)\n", counter, subsection.Title, section.Title)) + counter++ + } + } + + promptCtx := PromptContext{ + Manifest: d.manifest, + TargetDocFile: d.targetDocFile, + Changes: modificationPrompt, + } + + // Store section list in a temporary field for the prompt + promptCtx.SectionTitle = sectionsBuilder.String() + + return d.buildPrompt(PromptTypeModificationAnalysis, promptCtx) +} + +// parseScopeAnalysisResponse parses the LLM's scope analysis response +func (d *DocumentationAgent) parseScopeAnalysisResponse(content string, availableSections []string) (*ModificationScope, error) { + // Try to find JSON in the response + jsonStart := strings.Index(content, "{") + jsonEnd := strings.LastIndex(content, "}") + + if jsonStart == -1 || jsonEnd == -1 { + return nil, fmt.Errorf("no JSON found in response") + } + + jsonContent := content[jsonStart : jsonEnd+1] + + // Parse JSON + var response scopeAnalysisResponse + if err := json.Unmarshal([]byte(jsonContent), &response); err != nil { + return nil, fmt.Errorf("failed to parse JSON: %w", err) + } + + // Convert string type to ScopeType + var scopeType ScopeType + switch strings.ToLower(response.Type) { + case "global": + scopeType = ScopeGlobal + case "specific": + scopeType = ScopeSpecific + case "ambiguous": + scopeType = ScopeAmbiguous + default: + scopeType = ScopeGlobal + } + + // If specific scope but no sections listed, treat as global + if scopeType == ScopeSpecific && len(response.Sections) == 0 { + scopeType = ScopeGlobal + response.Sections = availableSections + } + + // If ambiguous or low confidence, default to global + if scopeType == ScopeAmbiguous || response.Confidence < 0.6 { + scopeType = ScopeGlobal + response.Sections = availableSections + } + + return &ModificationScope{ + Type: scopeType, + AffectedSections: response.Sections, + Confidence: response.Confidence, + Reasoning: response.Reasoning, + }, nil +} + +// isSectionAffected checks if a section title matches any of the affected section titles +func isSectionAffected(sectionTitle string, affectedTitles []string) bool { + titleLower := strings.ToLower(strings.TrimSpace(sectionTitle)) + + for _, affected := range affectedTitles { + affectedLower := strings.ToLower(strings.TrimSpace(affected)) + + // Exact match + if titleLower == affectedLower { + return true + } + + // Fuzzy match: check if one contains the other + if strings.Contains(titleLower, affectedLower) || strings.Contains(affectedLower, titleLower) { + return true + } + } + + return false +} diff --git a/internal/llmagent/docagent/prompts/loader.go b/internal/llmagent/docagent/prompts/loader.go index dc6f30d86f..35875f8419 100644 --- a/internal/llmagent/docagent/prompts/loader.go +++ b/internal/llmagent/docagent/prompts/loader.go @@ -12,6 +12,8 @@ type Type int const ( TypeRevision Type = iota TypeSectionGeneration + TypeModificationAnalysis + TypeModification ) // Load returns the embedded prompt content for the given type @@ -21,6 +23,10 @@ func Load(promptType Type) string { return RevisionPrompt case TypeSectionGeneration: return SectionGenerationPrompt + case TypeModificationAnalysis: + return ModificationAnalysisPrompt + case TypeModification: + return ModificationPrompt default: return "" } diff --git a/internal/llmagent/docagent/prompts/modification_analysis_prompt.txt b/internal/llmagent/docagent/prompts/modification_analysis_prompt.txt new file mode 100644 index 0000000000..7e424cfc0f --- /dev/null +++ b/internal/llmagent/docagent/prompts/modification_analysis_prompt.txt @@ -0,0 +1,59 @@ +You are analyzing a documentation modification request to determine which sections of the document are affected. + +Target Documentation File: %s + +Package Information: +* Package Name: %s +* Title: %s +* Type: %s +* Version: %s +* Description: %s + +Available sections in the document: +%s + +User's modification request: +"%s" + +Your task is to analyze this modification request and determine its scope: + +1. **Global Scope** - Select this if: + - The request affects ALL sections or the entire document structure + - The request involves template-wide changes (like "apply new template", "improve writing style throughout") + - The request adds or removes sections from the document + - The request is about overall formatting or structure + +2. **Specific Scope** - Select this if: + - The request clearly targets specific, named sections + - The request involves content that belongs to particular sections (like "update compatible versions" → Compatibility section) + - You can confidently identify which 1-3 sections are affected + +3. **Ambiguous Scope** - Select this if: + - The request is vague or unclear (like "make it better", "fix issues") + - You cannot confidently determine which sections are affected + - The request could apply to multiple interpretations + +Classification Guidelines: +- If unsure, prefer "global" over "ambiguous" +- For specific scope, list section titles EXACTLY as they appear in the available sections list +- Provide confidence as a decimal from 0.0 to 1.0 (0.0 = no confidence, 1.0 = completely certain) +- Explain your reasoning clearly + +Response Format: +You MUST respond with a JSON object in this exact format: + +{ + "type": "global|specific|ambiguous", + "sections": ["Section Title 1", "Section Title 2"], + "confidence": 0.9, + "reasoning": "Explanation of why you classified it this way" +} + +Important: +- For "global" or "ambiguous" scope, set "sections" to an empty array: [] +- For "specific" scope, list the exact section titles that should be modified +- Always include all four fields: type, sections, confidence, reasoning +- Return ONLY the JSON object, no additional text + +Begin your analysis now. + diff --git a/internal/llmagent/docagent/prompts/modification_prompt.txt b/internal/llmagent/docagent/prompts/modification_prompt.txt new file mode 100644 index 0000000000..b2e248891d --- /dev/null +++ b/internal/llmagent/docagent/prompts/modification_prompt.txt @@ -0,0 +1,53 @@ +You are modifying a specific section of documentation based on user feedback. + +Target Documentation File: %s +Section to modify: %s (Level %d header) + +Package Information: +* Package Name: %s +* Title: %s +* Type: %s +* Version: %s +* Description: %s + +Current section content: +--- +%s +--- + +User's modification request: +"%s" + +%sAvailable Tools (Use These for Information Gathering): + +* list_directory: List files and directories in the package. Use path="" for package root. +* read_file: Read contents of files within the package. +* get_service_info: Get relevant sections from the service_info.md knowledge base. Pass the current README section name to get only relevant information. + +Tool Usage Guidelines: +- Use list_directory and read_file to analyze the package if you need additional context +- Use get_service_info tool to access authoritative information about the service/technology +- Pass the current README section name (e.g., "Overview") to get relevant information +- If service_info.md contains information, treat it as the authoritative source of truth +- All file paths must be relative to package root (e.g., "manifest.yml", "data_stream/logs/manifest.yml") + +Your task: +1. Read the current section content carefully +2. Apply the requested modification appropriately for THIS section only +3. Preserve content that doesn't need to change based on the modification request +4. Maintain the section header at the appropriate level (%s %s) +5. Focus only on changes relevant to this specific section + +Style and Content Guidance: + +* Audience & Tone: Write for a technical audience (DevOps Engineers, SREs, Security Analysts). Be professional, clear, and direct. Use active voice. +* Follow Elastic voice and tone guidelines: + * We're real people, writing for real people - be conversational and address users directly + * Help users get their job done - be timely, succinct, and impactful + * Write like a minimalist - use concise, clear sentences with simple, precise words + * Avoid "please" in documentation + * Use active voice + * Use "for example" instead of "e.g." + +Return ONLY the modified section markdown, starting with the section header (%s %s). Do not include explanations or other sections. + diff --git a/internal/llmagent/docagent/prompts/resources.go b/internal/llmagent/docagent/prompts/resources.go index 3524fe4668..4f8489337a 100644 --- a/internal/llmagent/docagent/prompts/resources.go +++ b/internal/llmagent/docagent/prompts/resources.go @@ -22,3 +22,9 @@ var CriticRejectionCriteria string //go:embed full_formatting_rules.txt var FullFormattingRules string + +//go:embed modification_analysis_prompt.txt +var ModificationAnalysisPrompt string + +//go:embed modification_prompt.txt +var ModificationPrompt string diff --git a/internal/llmagent/docagent/promptsbuilder.go b/internal/llmagent/docagent/promptsbuilder.go index 4d9aeced36..15487190b5 100644 --- a/internal/llmagent/docagent/promptsbuilder.go +++ b/internal/llmagent/docagent/promptsbuilder.go @@ -23,6 +23,12 @@ func (d *DocumentationAgent) buildPrompt(promptType PromptType, ctx PromptContex case PromptTypeSectionGeneration: promptContent = prompts.Load(prompts.TypeSectionGeneration) formatArgs = d.buildSectionGenerationPromptArgs(ctx).ToFormatArgs() + case PromptTypeModificationAnalysis: + promptContent = prompts.Load(prompts.TypeModificationAnalysis) + formatArgs = d.buildModificationAnalysisPromptArgs(ctx) + case PromptTypeModification: + promptContent = prompts.Load(prompts.TypeModification) + formatArgs = d.buildModificationPromptArgs(ctx) } return fmt.Sprintf(promptContent, formatArgs...) @@ -157,6 +163,45 @@ func (d *DocumentationAgent) buildSectionGenerationPromptArgs(ctx PromptContext) } } +// buildModificationAnalysisPromptArgs prepares arguments for modification analysis prompt +func (d *DocumentationAgent) buildModificationAnalysisPromptArgs(ctx PromptContext) []interface{} { + return []interface{}{ + ctx.TargetDocFile, // target file + ctx.Manifest.Name, // package name + ctx.Manifest.Title, // package title + ctx.Manifest.Type, // package type + ctx.Manifest.Version, // package version + ctx.Manifest.Description, // package description + ctx.SectionTitle, // section list (stored temporarily in SectionTitle field) + ctx.Changes, // modification request + } +} + +// buildModificationPromptArgs prepares arguments for modification prompt +func (d *DocumentationAgent) buildModificationPromptArgs(ctx PromptContext) []interface{} { + levelStr := "##" + if ctx.SectionLevel == 3 { + levelStr = "###" + } + + return []interface{}{ + ctx.TargetDocFile, // target file + ctx.SectionTitle, // section title + ctx.SectionLevel, // section level number + ctx.Manifest.Name, // package name + ctx.Manifest.Title, // package title + ctx.Manifest.Type, // package type + ctx.Manifest.Version, // package version + ctx.Manifest.Description, // package description + ctx.TemplateSection, // current section content + ctx.Changes, // modification request + levelStr, // level prefix for header instruction + ctx.SectionTitle, // section title for header instruction + levelStr, // level prefix for final reminder + ctx.SectionTitle, // section title for final reminder + } +} + // Helper to create context with service info func (d *DocumentationAgent) createPromptContext(manifest *packages.PackageManifest, changes string) PromptContext { return PromptContext{ diff --git a/tools/readme/readme.md.tmpl b/tools/readme/readme.md.tmpl index f32fc1bbf9..a0827facec 100644 --- a/tools/readme/readme.md.tmpl +++ b/tools/readme/readme.md.tmpl @@ -256,7 +256,7 @@ The following settings are available per profile: ### AI-powered Documentation Configuration -The `elastic-package update documentation` command can generate or update package documentation using an LLM. +The `elastic-package update documentation` command can generate or update package documentation using an LLM. It supports full rewrites and targeted modifications. For full details on configuration, LLM provider setup, and the optional service knowledge base, see [docs/howto/ai_documentation.md](docs/howto/ai_documentation.md). From 06c8fe583402c6aa186d9de2edb15afce6d82325 Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Fri, 19 Jun 2026 15:14:02 -0700 Subject: [PATCH 2/4] fix: address code review findings in modify documentation command - Pass existingSections (not templateSections) to analyzeModificationScope so scope analysis matches against the actual document's section titles - Check backupOriginalReadme error in ModifyDocumentation, consistent with UpdateDocumentation - Let ScopeAmbiguous flow through parseScopeAnalysisResponse so the user warning in applyModificationScope is reachable; low-confidence specific scope now becomes ScopeAmbiguous instead of silently falling back to global - Return ShouldContinue=true on empty text area input in handleRequestChanges so the interactive loop re-prompts rather than terminating the session - Remove unused sections parameter from runInteractiveSectionReview - Parallelize modifyAllSections and modifySpecificSections with goroutines, matching the pattern used by GenerateAllSectionsWithValidation - Pre-compute affected sections before modification to use an accurate denominator in modifySpecificSections progress messages - Add dedicated SectionList field to PromptContext instead of overloading SectionTitle in scope analysis prompt building - Remove false-positive direction from isSectionAffected fuzzy match - Replace inline ModificationScope error literals with defaultModificationScope Co-Authored-By: Claude Sonnet 4.6 --- internal/llmagent/docagent/docagent.go | 178 ++++++++++-------- internal/llmagent/docagent/interactive.go | 6 +- .../llmagent/docagent/modificationanalyzer.go | 44 ++--- internal/llmagent/docagent/promptsbuilder.go | 2 +- 4 files changed, 117 insertions(+), 113 deletions(-) diff --git a/internal/llmagent/docagent/docagent.go b/internal/llmagent/docagent/docagent.go index 7be903c435..3aa4875dc8 100644 --- a/internal/llmagent/docagent/docagent.go +++ b/internal/llmagent/docagent/docagent.go @@ -99,6 +99,7 @@ type PromptContext struct { SectionLevel int TemplateSection string ExampleSection string + SectionList string // formatted list of all available sections (for scope analysis) PackageContext *validators.PackageContext // For section-specific instructions } @@ -290,7 +291,7 @@ func (d *DocumentationAgent) UpdateDocumentation(ctx context.Context, nonInterac // In interactive mode, allow review if !nonInteractive { - return d.runInteractiveSectionReview(ctx, sections) + return d.runInteractiveSectionReview(ctx) } return nil @@ -344,7 +345,9 @@ func (d *DocumentationAgent) ModifyDocumentation(ctx context.Context, nonInterac } // Backup original README content before making any changes - d.backupOriginalReadme() + if err := d.backupOriginalReadme(); err != nil { + return fmt.Errorf("failed to backup original readme: %w", err) + } fmt.Println("šŸ“ Analyzing modification request...") @@ -359,12 +362,8 @@ func (d *DocumentationAgent) ModifyDocumentation(ctx context.Context, nonInterac return fmt.Errorf("no sections found in existing documentation") } - // Get template sections for reference (structure) - templateContent := archetype.GetPackageDocsReadmeTemplate() - templateSections := parsing.ParseSections(templateContent) - - // Analyze modification scope - scope, err := d.analyzeModificationScope(ctx, instructions, templateSections) + // Analyze modification scope against the actual document sections + scope, err := d.analyzeModificationScope(ctx, instructions, existingSections) if err != nil { logger.Debugf("Scope analysis failed, defaulting to global: %v", err) scope = defaultModificationScope("Scope analysis failed, defaulting to global") @@ -402,7 +401,7 @@ func (d *DocumentationAgent) ModifyDocumentation(ctx context.Context, nonInterac // In interactive mode, allow review if !nonInteractive { - return d.runInteractiveSectionReview(ctx, finalSections) + return d.runInteractiveSectionReview(ctx) } return nil @@ -585,7 +584,7 @@ func (d *DocumentationAgent) applyModificationScope(ctx context.Context, existin } // runInteractiveSectionReview allows user to review and request changes in interactive mode -func (d *DocumentationAgent) runInteractiveSectionReview(ctx context.Context, sections []Section) error { +func (d *DocumentationAgent) runInteractiveSectionReview(ctx context.Context) error { for { // Display the generated documentation if err := d.displayReadme(); err != nil { @@ -622,11 +621,8 @@ func (d *DocumentationAgent) runInteractiveSectionReview(ctx context.Context, se } sectionsFromFile := parsing.ParseSections(existingContent) - // Analyze modification scope - templateContent := archetype.GetPackageDocsReadmeTemplate() - templateSections := parsing.ParseSections(templateContent) - - scope, err := d.analyzeModificationScope(ctx, actionResult.Changes, templateSections) + // Analyze modification scope against the current document sections + scope, err := d.analyzeModificationScope(ctx, actionResult.Changes, sectionsFromFile) if err != nil { logger.Debugf("Scope analysis failed, defaulting to global: %v", err) scope = defaultModificationScope("") @@ -648,51 +644,65 @@ func (d *DocumentationAgent) runInteractiveSectionReview(ctx context.Context, se } } -// modifyAllSections regenerates all sections with modification context +// modifyAllSections regenerates all sections with modification context, running in parallel func (d *DocumentationAgent) modifyAllSections(ctx context.Context, existingSections []Section, modificationPrompt string) ([]Section, error) { - var modifiedSections []Section + fmt.Printf("šŸ“ Modifying all %d sections concurrently...\n", len(existingSections)) + + resultsChan := make(chan sectionResult, len(existingSections)) + var wg sync.WaitGroup for i, section := range existingSections { - fmt.Printf("šŸ“ Modifying section %d/%d: %s\n", i+1, len(existingSections), section.Title) - - // Build modification prompt for this section - promptCtx := PromptContext{ - Manifest: d.manifest, - TargetDocFile: d.targetDocFile, - Changes: modificationPrompt, - SectionTitle: section.Title, - SectionLevel: section.Level, - TemplateSection: section.Content, - } + wg.Add(1) + go func(idx int, sec Section) { + defer wg.Done() + promptCtx := PromptContext{ + Manifest: d.manifest, + TargetDocFile: d.targetDocFile, + Changes: modificationPrompt, + SectionTitle: sec.Title, + SectionLevel: sec.Level, + TemplateSection: sec.Content, + } + prompt := d.buildPrompt(PromptTypeModification, promptCtx) + modifiedSection, err := d.generateModifiedSection(ctx, sec, prompt) + if err != nil { + logger.Debugf("Failed to modify section %s: %v", sec.Title, err) + resultsChan <- sectionResult{index: idx, section: sec, err: err} + return + } + resultsChan <- sectionResult{index: idx, section: modifiedSection} + }(i, section) + } - prompt := d.buildPrompt(PromptTypeModification, promptCtx) + go func() { + wg.Wait() + close(resultsChan) + }() - // Generate modified section - modifiedSection, err := d.generateModifiedSection(ctx, section, prompt) - if err != nil { - logger.Debugf("Failed to modify section %s: %v", section.Title, err) - // On error, keep the original section - modifiedSections = append(modifiedSections, section) - continue + results := make([]Section, len(existingSections)) + for r := range resultsChan { + if r.err != nil { + results[r.index] = existingSections[r.index] + } else { + results[r.index] = r.section } - - modifiedSections = append(modifiedSections, modifiedSection) } - return modifiedSections, nil + fmt.Printf("āœ“ Modified all %d sections\n", len(results)) + return results, nil } -// modifySpecificSections regenerates only affected sections -// For hierarchical sections, if a subsection is affected, the entire parent section is regenerated +// modifySpecificSections regenerates only affected sections, running in parallel. +// For hierarchical sections, if a subsection is affected, the entire parent section is regenerated. func (d *DocumentationAgent) modifySpecificSections(ctx context.Context, existingSections []Section, affectedSectionTitles []string, modificationPrompt string) ([]Section, error) { - var finalSections []Section - modifiedCount := 0 - - for _, section := range existingSections { - // Check if this section or any of its subsections are affected + // Pre-compute which top-level sections need modification so the denominator is accurate. + type sectionWork struct { + index int + section Section + } + var toModify []sectionWork + for i, section := range existingSections { isAffected := isSectionAffected(section.Title, affectedSectionTitles) - - // Check subsections - if any subsection is affected, modify the parent if !isAffected { for _, subsection := range section.Subsections { if isSectionAffected(subsection.Title, affectedSectionTitles) { @@ -702,52 +712,66 @@ func (d *DocumentationAgent) modifySpecificSections(ctx context.Context, existin } } } - if isAffected { - modifiedCount++ - fmt.Printf("šŸ“ Modifying section %d/%d: %s", modifiedCount, len(affectedSectionTitles), section.Title) - if section.HasSubsections() { - fmt.Printf(" (with %d subsections)", len(section.Subsections)) - } - fmt.Println() + toModify = append(toModify, sectionWork{index: i, section: section}) + } + } + + fmt.Printf("šŸ“ Modifying %d of %d sections concurrently...\n", len(toModify), len(existingSections)) + + if len(toModify) == 0 { + fmt.Println("āœ“ No matching sections found; document unchanged.") + return existingSections, nil + } - // Build modification prompt for this section (use FullContent for hierarchical context) + resultsChan := make(chan sectionResult, len(toModify)) + var wg sync.WaitGroup + + for _, work := range toModify { + wg.Add(1) + go func(idx int, sec Section) { + defer wg.Done() promptCtx := PromptContext{ Manifest: d.manifest, TargetDocFile: d.targetDocFile, Changes: modificationPrompt, - SectionTitle: section.Title, - SectionLevel: section.Level, - TemplateSection: section.GetAllContent(), // Include subsections in context + SectionTitle: sec.Title, + SectionLevel: sec.Level, + TemplateSection: sec.GetAllContent(), // include subsections in context } - prompt := d.buildPrompt(PromptTypeModification, promptCtx) - - // Generate modified section (includes subsections) - modifiedSection, err := d.generateModifiedSection(ctx, section, prompt) + modifiedSection, err := d.generateModifiedSection(ctx, sec, prompt) if err != nil { - logger.Debugf("Failed to modify section %s: %v", section.Title, err) - // On error, keep the original section - finalSections = append(finalSections, section) - continue + logger.Debugf("Failed to modify section %s: %v", sec.Title, err) + resultsChan <- sectionResult{index: idx, section: sec, err: err} + return } - - // Parse the generated content to extract hierarchical structure + // Parse to restore hierarchical structure parsedModified := parsing.ParseSections(modifiedSection.Content) if len(parsedModified) > 0 { - modifiedSection = parsedModified[0] // Take the full hierarchical section + modifiedSection = parsedModified[0] } + resultsChan <- sectionResult{index: idx, section: modifiedSection} + }(work.index, work.section) + } - finalSections = append(finalSections, modifiedSection) + go func() { + wg.Wait() + close(resultsChan) + }() + + finalSections := make([]Section, len(existingSections)) + copy(finalSections, existingSections) + for r := range resultsChan { + if r.err != nil { + finalSections[r.index] = existingSections[r.index] } else { - // Preserve entire section unchanged (including subsections) - finalSections = append(finalSections, section) + finalSections[r.index] = r.section } } - preservedCount := len(existingSections) - modifiedCount - fmt.Printf("āœ“ Modified: %d sections, Preserved: %d sections\n", modifiedCount, preservedCount) - + preservedCount := len(existingSections) - len(toModify) + fmt.Printf("āœ“ Modified: %d sections, Preserved: %d sections\n", len(toModify), preservedCount) return finalSections, nil } @@ -797,7 +821,7 @@ func (d *DocumentationAgent) generateModifiedSection(ctx context.Context, origin return modifiedSection, nil } -// sectionResult holds the result of generating a single section +// sectionResult holds the result of processing a single section (generation or modification) type sectionResult struct { index int section Section diff --git a/internal/llmagent/docagent/interactive.go b/internal/llmagent/docagent/interactive.go index c35ff9a04f..473fce9d33 100644 --- a/internal/llmagent/docagent/interactive.go +++ b/internal/llmagent/docagent/interactive.go @@ -173,10 +173,10 @@ func (d *DocumentationAgent) handleRequestChanges() ActionResult { return ActionResult{Err: fmt.Errorf("prompt failed: %w", err)} } - // Check if no changes were provided + // Check if no changes were provided — loop back to menu instead of exiting if strings.TrimSpace(changes) == "" { - fmt.Println("āš ļø No changes specified.") - return ActionResult{ShouldContinue: false} + fmt.Println("āš ļø No changes specified. Please enter your requested changes.") + return ActionResult{ShouldContinue: true} } promptCtx := d.createPromptContext(d.manifest, changes) newPrompt := d.buildPrompt(PromptTypeRevision, promptCtx) diff --git a/internal/llmagent/docagent/modificationanalyzer.go b/internal/llmagent/docagent/modificationanalyzer.go index 7084591b5f..1cee0f1614 100644 --- a/internal/llmagent/docagent/modificationanalyzer.go +++ b/internal/llmagent/docagent/modificationanalyzer.go @@ -57,15 +57,6 @@ type scopeAnalysisResponse struct { // analyzeModificationScope determines which sections a modification request affects func (d *DocumentationAgent) analyzeModificationScope(ctx context.Context, modificationPrompt string, availableSections []Section) (*ModificationScope, error) { - // Build list of section titles (including subsections) - sectionTitles := make([]string, 0) - for _, section := range availableSections { - sectionTitles = append(sectionTitles, section.Title) - for _, subsection := range section.Subsections { - sectionTitles = append(sectionTitles, subsection.Title) - } - } - // Build the analysis prompt with hierarchical structure prompt := d.buildScopeAnalysisPromptHierarchical(modificationPrompt, availableSections) @@ -74,24 +65,14 @@ func (d *DocumentationAgent) analyzeModificationScope(ctx context.Context, modif result, err := d.executor.ExecuteTask(ctx, prompt) if err != nil { logger.Debugf("Scope analysis failed, defaulting to global: %v", err) - return &ModificationScope{ - Type: ScopeGlobal, - AffectedSections: sectionTitles, - Confidence: 0.5, - Reasoning: "Analysis failed, defaulting to global scope", - }, nil + return defaultModificationScope("Analysis failed, defaulting to global scope"), nil } // Parse the response - scope, err := d.parseScopeAnalysisResponse(result.FinalContent, sectionTitles) + scope, err := d.parseScopeAnalysisResponse(result.FinalContent) if err != nil { logger.Debugf("Failed to parse scope analysis, defaulting to global: %v", err) - return &ModificationScope{ - Type: ScopeGlobal, - AffectedSections: sectionTitles, - Confidence: 0.5, - Reasoning: "Failed to parse analysis, defaulting to global scope", - }, nil + return defaultModificationScope("Failed to parse analysis, defaulting to global scope"), nil } logger.Debugf("Scope analysis complete: type=%s, sections=%v, confidence=%.2f", scope.Type, scope.AffectedSections, scope.Confidence) @@ -121,14 +102,13 @@ func (d *DocumentationAgent) buildScopeAnalysisPromptHierarchical(modificationPr Changes: modificationPrompt, } - // Store section list in a temporary field for the prompt - promptCtx.SectionTitle = sectionsBuilder.String() + promptCtx.SectionList = sectionsBuilder.String() return d.buildPrompt(PromptTypeModificationAnalysis, promptCtx) } // parseScopeAnalysisResponse parses the LLM's scope analysis response -func (d *DocumentationAgent) parseScopeAnalysisResponse(content string, availableSections []string) (*ModificationScope, error) { +func (d *DocumentationAgent) parseScopeAnalysisResponse(content string) (*ModificationScope, error) { // Try to find JSON in the response jsonStart := strings.Index(content, "{") jsonEnd := strings.LastIndex(content, "}") @@ -161,13 +141,11 @@ func (d *DocumentationAgent) parseScopeAnalysisResponse(content string, availabl // If specific scope but no sections listed, treat as global if scopeType == ScopeSpecific && len(response.Sections) == 0 { scopeType = ScopeGlobal - response.Sections = availableSections } - // If ambiguous or low confidence, default to global - if scopeType == ScopeAmbiguous || response.Confidence < 0.6 { - scopeType = ScopeGlobal - response.Sections = availableSections + // If specific scope with low confidence, treat as ambiguous so the user sees a warning + if scopeType == ScopeSpecific && response.Confidence < 0.6 { + scopeType = ScopeAmbiguous } return &ModificationScope{ @@ -190,8 +168,10 @@ func isSectionAffected(sectionTitle string, affectedTitles []string) bool { return true } - // Fuzzy match: check if one contains the other - if strings.Contains(titleLower, affectedLower) || strings.Contains(affectedLower, titleLower) { + // Fuzzy match: check if the affected title from the LLM contains the actual section title + // (handles cases where the LLM returns a longer name like "Data Streams Configuration" for section "Configuration") + // We do NOT check the reverse direction to avoid matching short terms like "Data" against "Data streams". + if strings.Contains(affectedLower, titleLower) { return true } } diff --git a/internal/llmagent/docagent/promptsbuilder.go b/internal/llmagent/docagent/promptsbuilder.go index 15487190b5..d7880f8fec 100644 --- a/internal/llmagent/docagent/promptsbuilder.go +++ b/internal/llmagent/docagent/promptsbuilder.go @@ -172,7 +172,7 @@ func (d *DocumentationAgent) buildModificationAnalysisPromptArgs(ctx PromptConte ctx.Manifest.Type, // package type ctx.Manifest.Version, // package version ctx.Manifest.Description, // package description - ctx.SectionTitle, // section list (stored temporarily in SectionTitle field) + ctx.SectionList, // available section list ctx.Changes, // modification request } } From 41bb030bbda5cdffbe681aedb2db2559e464aa8c Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Mon, 22 Jun 2026 15:02:06 -0700 Subject: [PATCH 3/4] fix: resolve staticcheck and unparam lint errors in modificationanalyzer - Replace WriteString(fmt.Sprintf(...)) with fmt.Fprintf() (QF1012) - Remove always-nil error return from analyzeModificationScope (unparam) - Update call sites in docagent.go to match new signature Co-Authored-By: Claude Sonnet 4.6 --- cmd/updatedocumentation.go | 3 --- internal/llmagent/docagent/docagent.go | 17 ++++------------- .../llmagent/docagent/modificationanalyzer.go | 12 ++++++------ 3 files changed, 10 insertions(+), 22 deletions(-) diff --git a/cmd/updatedocumentation.go b/cmd/updatedocumentation.go index 53253ba22e..32e475f1aa 100644 --- a/cmd/updatedocumentation.go +++ b/cmd/updatedocumentation.go @@ -44,7 +44,6 @@ const ( modePromptModify = "Modify (targeted changes)" ) - // discoverDocumentationTemplates finds all .md files in _dev/build/docs/ func discoverDocumentationTemplates(packageRoot string) ([]string, error) { docsDir := filepath.Join(packageRoot, "_dev", "build", "docs") @@ -206,7 +205,6 @@ func runDocumentationUpdate(cmd *cobra.Command, docAgent *docagent.Documentation return nil } - func updateDocumentationCommandAction(cmd *cobra.Command, _ []string) error { p, err := cobraext.GetProfileFlag(cmd) if err != nil { @@ -278,4 +276,3 @@ func handleStandardMode(cmd *cobra.Command, p *profile.Profile, cfg llmconfig.LL return runDocumentationUpdate(cmd, docAgent, useModifyMode, flags.nonInteractive, flags.modifyPrompt) } - diff --git a/internal/llmagent/docagent/docagent.go b/internal/llmagent/docagent/docagent.go index 3aa4875dc8..bfdde425e5 100644 --- a/internal/llmagent/docagent/docagent.go +++ b/internal/llmagent/docagent/docagent.go @@ -99,7 +99,7 @@ type PromptContext struct { SectionLevel int TemplateSection string ExampleSection string - SectionList string // formatted list of all available sections (for scope analysis) + SectionList string // formatted list of all available sections (for scope analysis) PackageContext *validators.PackageContext // For section-specific instructions } @@ -115,7 +115,7 @@ type SectionGenerationContext struct { // AgentConfig holds configuration for creating a DocumentationAgent type AgentConfig struct { - Provider string // LLM provider (e.g. gemini); default gemini + Provider string // LLM provider (e.g. gemini); default gemini APIKey string ModelID string PackageRoot string @@ -363,11 +363,7 @@ func (d *DocumentationAgent) ModifyDocumentation(ctx context.Context, nonInterac } // Analyze modification scope against the actual document sections - scope, err := d.analyzeModificationScope(ctx, instructions, existingSections) - if err != nil { - logger.Debugf("Scope analysis failed, defaulting to global: %v", err) - scope = defaultModificationScope("Scope analysis failed, defaulting to global") - } + scope := d.analyzeModificationScope(ctx, instructions, existingSections) // Report scope to user fmt.Printf("āœ“ Scope: %s", scope.Type) @@ -622,11 +618,7 @@ func (d *DocumentationAgent) runInteractiveSectionReview(ctx context.Context) er sectionsFromFile := parsing.ParseSections(existingContent) // Analyze modification scope against the current document sections - scope, err := d.analyzeModificationScope(ctx, actionResult.Changes, sectionsFromFile) - if err != nil { - logger.Debugf("Scope analysis failed, defaulting to global: %v", err) - scope = defaultModificationScope("") - } + scope := d.analyzeModificationScope(ctx, actionResult.Changes, sectionsFromFile) // Apply modifications modifiedSections, err := d.applyModificationScope(ctx, sectionsFromFile, scope, actionResult.Changes) @@ -828,7 +820,6 @@ type sectionResult struct { err error } - // loadTemplateExampleExistingSections loads template, example, and existing doc sections and returns top-level template sections. func (d *DocumentationAgent) loadTemplateExampleExistingSections() (topLevelSections, exampleSections, existingSections []Section, err error) { templateSections := parsing.ParseSections(archetype.GetPackageDocsReadmeTemplate()) diff --git a/internal/llmagent/docagent/modificationanalyzer.go b/internal/llmagent/docagent/modificationanalyzer.go index 1cee0f1614..4da94f3612 100644 --- a/internal/llmagent/docagent/modificationanalyzer.go +++ b/internal/llmagent/docagent/modificationanalyzer.go @@ -56,7 +56,7 @@ type scopeAnalysisResponse struct { } // analyzeModificationScope determines which sections a modification request affects -func (d *DocumentationAgent) analyzeModificationScope(ctx context.Context, modificationPrompt string, availableSections []Section) (*ModificationScope, error) { +func (d *DocumentationAgent) analyzeModificationScope(ctx context.Context, modificationPrompt string, availableSections []Section) *ModificationScope { // Build the analysis prompt with hierarchical structure prompt := d.buildScopeAnalysisPromptHierarchical(modificationPrompt, availableSections) @@ -65,18 +65,18 @@ func (d *DocumentationAgent) analyzeModificationScope(ctx context.Context, modif result, err := d.executor.ExecuteTask(ctx, prompt) if err != nil { logger.Debugf("Scope analysis failed, defaulting to global: %v", err) - return defaultModificationScope("Analysis failed, defaulting to global scope"), nil + return defaultModificationScope("Analysis failed, defaulting to global scope") } // Parse the response scope, err := d.parseScopeAnalysisResponse(result.FinalContent) if err != nil { logger.Debugf("Failed to parse scope analysis, defaulting to global: %v", err) - return defaultModificationScope("Failed to parse analysis, defaulting to global scope"), nil + return defaultModificationScope("Failed to parse analysis, defaulting to global scope") } logger.Debugf("Scope analysis complete: type=%s, sections=%v, confidence=%.2f", scope.Type, scope.AffectedSections, scope.Confidence) - return scope, nil + return scope } // buildScopeAnalysisPromptHierarchical creates the prompt for scope analysis with hierarchical structure @@ -86,12 +86,12 @@ func (d *DocumentationAgent) buildScopeAnalysisPromptHierarchical(modificationPr counter := 1 for _, section := range sections { - sectionsBuilder.WriteString(fmt.Sprintf("%d. %s\n", counter, section.Title)) + fmt.Fprintf(§ionsBuilder, "%d. %s\n", counter, section.Title) counter++ // List subsections with indentation for _, subsection := range section.Subsections { - sectionsBuilder.WriteString(fmt.Sprintf(" %d. %s (subsection of %s)\n", counter, subsection.Title, section.Title)) + fmt.Fprintf(§ionsBuilder, " %d. %s (subsection of %s)\n", counter, subsection.Title, section.Title) counter++ } } From 26ab1713b3fc1d7660aa2396c3a2575bd9bc9278 Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Mon, 6 Jul 2026 11:31:30 -0700 Subject: [PATCH 4/4] Change default LLM model to gemini-3.5-flash. Change the default LLM model to gemini-3.5-flash, as the gemini-3-flash-preview model is deprecated. --- docs/howto/ai_documentation.md | 2 +- internal/llmagent/README.md | 2 +- internal/llmagent/config/config.go | 2 +- internal/llmagent/tracing/tracing.go | 2 +- internal/profile/_static/config.yml.example | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/howto/ai_documentation.md b/docs/howto/ai_documentation.md index 7c96fcb10a..0e48c38af5 100644 --- a/docs/howto/ai_documentation.md +++ b/docs/howto/ai_documentation.md @@ -34,7 +34,7 @@ Currently only the Gemini provider is supported. Gemini is the default provider. ```yaml llm.provider: gemini llm.gemini.api_key: "YOUR_API_KEY" -llm.gemini.model: "gemini-3-flash-preview" +llm.gemini.model: "gemini-3.5-flash" llm.gemini.thinking_budget: "128" ``` diff --git a/internal/llmagent/README.md b/internal/llmagent/README.md index 6c94d2cd11..2ad562586e 100644 --- a/internal/llmagent/README.md +++ b/internal/llmagent/README.md @@ -281,7 +281,7 @@ Example: `workflow.DefaultConfig().WithModel(m).WithModelID("...").WithTools(too ```go agent, err := docagent.NewDocumentationAgent(ctx, docagent.AgentConfig{ APIKey: apiKey, - ModelID: "gemini-3-flash-preview", + ModelID: "gemini-3.5-flash", PackageRoot: "/path/to/package", DocFile: "README.md", }) diff --git a/internal/llmagent/config/config.go b/internal/llmagent/config/config.go index 0fcfa9cdc8..abdfd6411f 100644 --- a/internal/llmagent/config/config.go +++ b/internal/llmagent/config/config.go @@ -17,7 +17,7 @@ import ( ) const ( - DefaultModelID = "gemini-3-flash-preview" + DefaultModelID = "gemini-3.5-flash" DefaultThinkingBudget int32 = 128 DefaultProvider = "gemini" ) diff --git a/internal/llmagent/tracing/tracing.go b/internal/llmagent/tracing/tracing.go index 995e157839..e8b695ce10 100644 --- a/internal/llmagent/tracing/tracing.go +++ b/internal/llmagent/tracing/tracing.go @@ -17,7 +17,7 @@ const ( DefaultEndpoint = "http://localhost:6006/v1/traces" DefaultProjectName = "elastic-package" TracerName = "elastic-package-llmagent" - DefaultModelID = "gemini-3-flash-preview" + DefaultModelID = "gemini-3.5-flash" ) // Config holds LLM tracing configuration. diff --git a/internal/profile/_static/config.yml.example b/internal/profile/_static/config.yml.example index e718051cc3..2a42dfecc5 100644 --- a/internal/profile/_static/config.yml.example +++ b/internal/profile/_static/config.yml.example @@ -40,7 +40,7 @@ # llm.provider: "gemini" # Gemini: API key and model # llm.gemini.api_key: "your-gemini-api-key" -# llm.gemini.model: "gemini-3-flash-preview" +# llm.gemini.model: "gemini-3.5-flash" ## LLM Tracing (OpenTelemetry/Phoenix) # Enable or disable LLM tracing (defaults to true)