diff --git a/.cline/instructions.md b/.cline/instructions.md index 4a0854a..f118b15 100644 --- a/.cline/instructions.md +++ b/.cline/instructions.md @@ -230,6 +230,61 @@ The Common AST must be designed for execution by the Expression Evaluation (EE) ### Graphviz Purpose **Important**: Graphviz DOT file generation is solely for developer diagnosis and understanding of AST structure during development. It is NOT for end-user visualization or production use. +## Collaborative Design Process + +### How to Design New AST Constructs Together +**CRITICAL**: When a user requests a new AST construct (like ProjectNode), follow this collaborative process: + +1. **Start with Questions**: Don't jump to implementation. Ask: + - "What are some examples of how this would be used?" + - "Should we challenge any assumptions about this design?" + - "What are the alternatives we should consider?" + +2. **Grammar-Driven Discovery**: Examine both KQL and TraceQL grammars together + - Show the user the relevant grammar sections + - Discuss how each language expresses the concept + - Identify commonalities and differences + +3. **Design Iteration**: Propose initial designs and refine based on feedback + - Present multiple options with trade-offs + - Ask "Why do we need X?" for each requirement + - Challenge assumptions about complexity and type systems + +4. **Document Decisions**: Capture the reasoning behind design choices + - Update memory-bank/designProcess.md with insights + - Include examples of questions that led to better designs + - Document what we learned for future features + +### Design Questions to Always Ask +When designing new constructs, always explore: + +1. **Type System Questions** + - "Is type information actually needed here?" + - "Can downstream systems infer this information?" + - "Are we over-engineering simple cases?" + +2. **Complexity Questions** + - "What's the minimum viable implementation?" + - "How will we handle complex cases later?" + - "Should we implement Level 1 (simple) vs Level 2 (complex) features?" + +3. **Engine Compatibility Questions** + - "Is this design engine-agnostic?" + - "Does it work with Arrow data operations?" + - "Are we avoiding engine-specific dependencies?" + +4. **Validation Questions** + - "Where should validation happen?" + - "What information does the AST need to provide?" + - "How do we separate concerns cleanly?" + +### Successful Collaboration Example: ProjectNode +Reference memory-bank/designProcess.md for how we successfully designed ProjectNode through: +- Questioning type system requirements +- Grammar analysis of both languages +- Iterative refinement based on user feedback +- Smart design that avoids over-engineering + ## Common Development Tasks ### Adding New AST Node Types (Grammar-Driven Process) diff --git a/memory-bank/designProcess.md b/memory-bank/designProcess.md new file mode 100644 index 0000000..1d86a83 --- /dev/null +++ b/memory-bank/designProcess.md @@ -0,0 +1,177 @@ +# Design Process and Collaboration Patterns + +## How We Designed ProjectNode Together + +### 📋 **Initial Request and Context Gathering** +The user requested implementation of a ProjectNode to map KQL's `project` operator to TraceQL's `select` operation. Before jumping into implementation, we followed a structured discovery process: + +1. **Grammar Analysis**: We examined both KQL and TraceQL grammar files to understand the exact syntax patterns +2. **Requirements Clarification**: Discussed scope, type system needs, and engine compatibility +3. **Design Questions**: Challenged assumptions about wildcards, type requirements, and validation responsibilities + +### 🤔 **Key Design Questions We Explored** + +#### Question 1: "Should we support wildcard projections (`*`)?" +**Decision**: No wildcard support for Level 1 +**Reasoning**: If all fields are needed, the project operation should be omitted entirely +**Impact**: Simplified design, clearer semantics + +#### Question 2: "Do we need type information for all projections?" +**Initial Assumption**: All projections need type info for Expression Evaluation engine +**Challenge**: "Why does project operation require a result type? It doesn't evaluate anything..." +**Refined Decision**: Type info only for transformative operations (calculations, function calls) +**Impact**: Much cleaner API, reduced complexity for simple field selections + +#### Question 3: "What level of expression complexity should we support?" +**Decision**: Level 1 (simple fields, aliases, basic arithmetic, simple functions) now, Level 2 (complex expressions) later +**Reasoning**: Start with common use cases, document future TODOs clearly +**Impact**: Focused implementation, clear expansion path + +#### Question 4: "Where should validation happen?" +**Decision**: AST contains all info needed, validation happens in downstream phases +**Reasoning**: Separation of concerns, AST focused on representation not validation +**Impact**: Clean architecture, flexible validation strategies + +### 🔄 **Iterative Design Refinement** + +#### Round 1: Initial Structure +```csharp +// Initial design - too rigid +public class ProjectNode : OperationNode +{ + public List Fields { get; set; } // Too simple + public Dictionary Aliases { get; set; } // Separate aliases +} +``` + +#### Round 2: Expression-Based Design +```csharp +// Improved - but still had issues +public class ProjectionExpression : ASTNode +{ + public Expression Expression { get; set; } + public string? Alias { get; set; } + public ExpressionType ResultType { get; set; } // Always required - wrong! +} +``` + +#### Round 3: Final Smart Design +```csharp +// Final design - smart about when type info is needed +public class ProjectionExpression : ASTNode +{ + public required Expression Expression { get; set; } + public string? Alias { get; set; } + public ExpressionType? ResultType { get; set; } // Optional - only for transformations +} +``` + +### 🎯 **Design Patterns We Established** + +#### Pattern 1: Grammar-Driven Design +- Always start by analyzing the actual grammar files +- Understand the syntax before designing the AST representation +- Map language constructs directly to AST nodes + +#### Pattern 2: Progressive Complexity +- Implement Level 1 features first (common cases) +- Document Level 2 features as TODOs with clear comments +- Show examples of future complexity in code comments + +#### Pattern 3: Smart Type System +- Don't over-engineer simple cases +- Type information only where actually needed +- Let downstream systems infer types when possible + +#### Pattern 4: Cross-Language Compatibility +- Use keywords to distinguish language-specific syntax +- Design AST nodes to represent concepts, not syntax +- Enable round-trip generation to different languages + +### 💡 **Critical Design Insights** + +#### Insight 1: "Type Information Isn't Always Needed" +The breakthrough moment was realizing that simple field selections (`name`, `duration`) don't need type information because the Expression Evaluation engine can infer types from the data schema. Type info is only needed for transformations. + +#### Insight 2: "AST Should Represent Intent, Not Syntax" +Rather than literally translating syntax, we designed the AST to represent the semantic intent: "project these expressions with these optional aliases and types." + +#### Insight 3: "Future-Proofing Through Documentation" +By clearly documenting Level 2 TODOs in comments, we make it easy for future developers to understand the expansion path without over-engineering the current implementation. + +## Collaborative Decision-Making Process + +### 🤝 **How We Made Design Decisions** + +1. **Question Everything**: Started with "why does this need...?" +2. **Analyze Examples**: Looked at real KQL and TraceQL query patterns +3. **Challenge Assumptions**: "Is this really needed for all cases?" +4. **Iterate Quickly**: Made changes based on new insights +5. **Document Decisions**: Captured reasoning for future reference + +### 📝 **Documentation Strategy** + +#### In-Code Documentation +- Comments explaining Level 1 vs Level 2 support +- Examples showing usage patterns +- Clear reasoning for design choices + +#### Memory Bank Updates +- Progress tracking with implementation status +- Design process documentation (this file) +- Patterns for future implementations + +### 🔍 **Questions to Ask for Future Designs** + +When designing new AST constructs, always ask: + +1. **Grammar Analysis** + - What does the actual grammar say? + - How do both languages express this concept? + - What are the edge cases in the syntax? + +2. **Type System** + - Is type information actually needed here? + - Can downstream systems infer this information? + - What are the performance implications? + +3. **Expression Complexity** + - What's the minimum viable implementation? + - How will we handle complex cases later? + - Where should we document future TODOs? + +4. **Engine Compatibility** + - Is this design engine-agnostic? + - Does it work with Arrow data operations? + - Are we avoiding engine-specific dependencies? + +5. **Validation Strategy** + - Where should validation happen? + - What information does the AST need to provide? + - How do we separate concerns cleanly? + +## Recommendations for Future Collaborations + +### 🎯 **For Users** +When requesting new features: +- **Challenge the AI**: Ask "why do we need this?" and "what are the alternatives?" +- **Provide Examples**: Show real-world usage patterns you want to support +- **Ask Questions**: Request explanations of design choices and trade-offs +- **Iterate**: Be willing to refine requirements based on technical insights + +### 🤖 **For AI Assistants** +When implementing new features: +- **Start with Grammar**: Always analyze the actual language grammars first +- **Ask Clarifying Questions**: Don't assume requirements, ask for details +- **Propose Options**: Present multiple design approaches with trade-offs +- **Document Decisions**: Capture the reasoning behind design choices +- **Plan for Growth**: Design Level 1 with clear path to Level 2 + +### 🏗️ **Architecture Principles** +- **Grammar-Driven**: Let language specifications guide AST design +- **Engine-Agnostic**: Avoid dependencies on specific execution engines +- **Type-Conscious**: Be smart about when type information is needed +- **Progressive**: Implement common cases first, document complex cases as TODOs +- **Collaborative**: Use questions and challenges to improve design quality + +This collaborative approach resulted in a much better ProjectNode design than either human or AI could have achieved alone. The key was the iterative questioning and refinement process that led to genuine insights about type systems and AST design. diff --git a/memory-bank/progress.md b/memory-bank/progress.md index 3fe6cf9..901725b 100644 --- a/memory-bank/progress.md +++ b/memory-bank/progress.md @@ -104,11 +104,12 @@ - Error handling and validation - Graphviz output generation -### Extended Query Operations 📋 -- **Project Operations**: Select/project functionality - - Column selection and aliasing - - Expression projection - - AST representation and processing +### Extended Query Operations ✅ +- **Project Operations**: Select/project functionality ✅ + - Column selection and aliasing ✅ + - Expression projection ✅ + - AST representation and processing ✅ + - KQL Visitor Support ✅ - **Summarize Operations**: Aggregation functionality - Group by operations diff --git a/memory-bank/systemPatterns.md b/memory-bank/systemPatterns.md index f2923d7..8a42100 100644 --- a/memory-bank/systemPatterns.md +++ b/memory-bank/systemPatterns.md @@ -326,3 +326,99 @@ Consistent error handling across components: - Clear error messages with actionable guidance This architecture provides a solid foundation for cross-language query processing while maintaining extensibility for future enhancements. + +## Visitor Pattern Requirements for New AST Constructs + +**CRITICAL**: When implementing new AST node types, the visitor pattern must be updated to handle the new constructs. This is essential for complete implementation. + +### Required Visitor Updates + +When adding a new AST construct (like ProjectNode), you must update **ALL** visitor implementations: + +1. **KqlToCommonAstVisitor** (most critical) + - Add new case to the `Visit(SyntaxNode node)` switch statement + - Implement the corresponding `VisitXxxOperator()` method + - Handle language-specific syntax parsing + - Convert to Common AST representation + +2. **Future Visitors** (TraceQL, others) + - Any future visitor implementations must also support the new constructs + - Follow the same pattern for consistency + +### Example: Adding ProjectNode Support + +**Step 1**: Add case to switch statement +```csharp +case SyntaxKind.ProjectOperator: + VisitProjectOperator(node as ProjectOperator); + break; +``` + +**Step 2**: Implement visitor method +```csharp +private void VisitProjectOperator(ProjectOperator node) +{ + // Parse KQL project syntax + // Extract expressions and aliases + // Convert to Common AST ProjectNode + // Add to query operations +} +``` + +**Step 3**: Handle language-specific nuances +- KQL: `| project field1, alias = field2, calculation = field3 / 1000` +- TraceQL: `select(span.field1, span.field2)` (different syntax, same concept) + +### Common Visitor Patterns + +**Expression Stack Pattern**: Use stack to handle nested expressions +```csharp +private Stack _expressionStack = new Stack(); + +// In visitor methods: +Visit(childExpression); +if (_expressionStack.Count > 0) +{ + var expr = _expressionStack.Pop(); + // Use expression +} +``` + +**Separated Elements Pattern**: Handle comma-separated lists +```csharp +foreach (var separatedElement in node.Expressions) +{ + var actualExpression = separatedElement.Element; + Visit(actualExpression); +} +``` + +**Alias Handling Pattern**: Extract optional aliases from syntax +```csharp +if (column is SimpleNamedExpression namedExpr) +{ + string? alias = namedExpr.Name?.SimpleName; + // Process with alias +} +else +{ + // Process without alias +} +``` + +### Testing Visitor Implementation + +Always test visitor updates with: +1. **Simple queries**: Basic syntax verification +2. **Complex queries**: Nested expressions, multiple operations +3. **Edge cases**: Empty lists, null expressions +4. **Integration tests**: End-to-end query processing + +### Memory Bank Update Requirement + +When adding visitor support for new constructs, **MUST** update: +- `memory-bank/progress.md`: Mark visitor support as complete +- `memory-bank/systemPatterns.md`: Document any new patterns +- `memory-bank/designProcess.md`: Capture design decisions + +This ensures future developers understand the complete implementation requirements for new AST constructs. diff --git a/src/AST/CommonAST.cs b/src/AST/CommonAST.cs index 735b7c6..5a9fd6e 100644 --- a/src/AST/CommonAST.cs +++ b/src/AST/CommonAST.cs @@ -8,6 +8,7 @@ public enum NodeKind { Query, Filter, + Project, Literal, Identifier, BinaryExpression, @@ -16,7 +17,8 @@ public enum NodeKind ParenthesizedExpression, SpecialOperatorExpression, WildcardExpression, - PathExpression + PathExpression, + ProjectionExpression } /// @@ -142,6 +144,68 @@ public enum SpanFilterCombination All } +/// +/// Project operation node representing both KQL's project operator and TraceQL's select operation +/// Projects specified fields/columns from the data, optionally with aliases +/// +/// Level 1 Support (Current): +/// - Simple field references +/// - Aliasing +/// - Basic arithmetic operations (+, -, *, /) +/// - Simple function calls (string, math, conversion) +/// +/// Level 2 Support (Future TODO): +/// - Complex nested expressions +/// - Conditional expressions (case) +/// - Advanced function calls +/// +public class ProjectNode : OperationNode +{ + public override NodeKind NodeKind => NodeKind.Project; + + /// + /// List of field projections to include in the output + /// No wildcard (*) support - if all fields needed, project operation should be omitted + /// + public List Projections { get; set; } = new List(); + + /// + /// Keyword for language-specific syntax ('project' for KQL, 'select' for TraceQL) + /// + public string? Keyword { get; set; } +} + +/// +/// Represents a single field projection with optional alias and type information +/// +public class ProjectionExpression : ASTNode +{ + public override NodeKind NodeKind => NodeKind.ProjectionExpression; + + /// + /// The expression to project (field name, calculation, function call, etc.) + /// + public required Expression Expression { get; set; } + + /// + /// Optional alias for the projected field + /// + public string? Alias { get; set; } + + /// + /// Internal declared type determined during AST construction + /// Users should call GetResultType() instead of accessing this directly + /// + internal ExpressionType? DeclaredType { get; set; } + + /// + /// Gets the effective type of this projection, considering both declared and semantic analysis + /// Returns resolved type from semantic analysis if available, otherwise returns declared type + /// Returns Unknown if no type information is available (simple field projections) + /// + public ExpressionType GetResultType() => DeclaredType ?? ExpressionType.Unknown; +} + /// /// Parameter for operators (KQL specific) /// @@ -188,6 +252,24 @@ public enum LiteralKind Dynamic } +/// +/// Type information for expressions - required for Expression Evaluation engine +/// Level 1 Support: Basic types for simple field projections and arithmetic +/// Level 2 Support (Future TODO): Complex types, arrays, nested objects +/// +public enum ExpressionType +{ + Unknown, // Type cannot be determined at syntax analysis time - requires semantic analysis + String, + Integer, + Float, + Boolean, + Duration, + DateTime, + Guid, + Dynamic +} + /// /// Binary operators used in expressions /// @@ -443,6 +525,47 @@ public static SpecialOperatorExpression CreateSpecialOperatorExpression(Expressi Right = right }; } + + /// + /// Creates a ProjectNode with specified projections + /// Level 1 Support: Simple fields, aliases, basic arithmetic, simple functions + /// + public static ProjectNode CreateProject(List projections, string? keyword = null) + { + return new ProjectNode + { + Projections = projections, + Keyword = keyword + }; + } + + /// + /// Creates a ProjectionExpression with optional alias and declared type + /// Defaults to null - semantic analysis will resolve actual types + /// + public static ProjectionExpression CreateProjection(Expression expression, string? alias = null, ExpressionType? declaredType = null) + { + return new ProjectionExpression + { + Expression = expression, + Alias = alias, + DeclaredType = declaredType + }; + } + + /// + /// Convenience method for simple field projection with alias + /// Defaults to null type - semantic analysis will resolve from schema + /// + public static ProjectionExpression CreateFieldProjection(string fieldName, string? alias = null, ExpressionType? declaredType = null, string? ns = null) + { + return new ProjectionExpression + { + Expression = CreateIdentifier(fieldName, ns), + Alias = alias, + DeclaredType = declaredType + }; + } } /// @@ -587,4 +710,125 @@ public static QueryNode SpansOnlyFilterExample() return query; } -} \ No newline at end of file + + // KQL: | project name, duration, status + public static QueryNode KqlSimpleProjectExample() + { + var projections = new List + { + AstBuilder.CreateFieldProjection("name"), + AstBuilder.CreateFieldProjection("duration"), + AstBuilder.CreateFieldProjection("status") + }; + + var projectNode = AstBuilder.CreateProject(projections, "project"); + + var query = AstBuilder.CreateQuery("MyTable"); + query.Operations.Add(projectNode); + + return query; + } + + // KQL: | project service_name = name, duration_ms = duration / 1000, upper_name = toupper(name) + public static QueryNode KqlProjectWithAliasesAndCalculationsExample() + { + var projections = new List + { + // Simple alias: service_name = name + AstBuilder.CreateProjection( + AstBuilder.CreateIdentifier("name"), + alias: "service_name" + ), + + // Calculated field: duration_ms = duration / 1000 + AstBuilder.CreateProjection( + AstBuilder.CreateBinaryExpression( + AstBuilder.CreateIdentifier("duration"), + BinaryOperatorKind.Divide, + AstBuilder.CreateLiteral(1000, LiteralKind.Integer) + ), + alias: "duration_ms", + declaredType: ExpressionType.Float + ), + + // Function call: upper_name = toupper(name) + AstBuilder.CreateProjection( + AstBuilder.CreateCallExpression("toupper", new List + { + AstBuilder.CreateIdentifier("name") + }), + alias: "upper_name", + declaredType: ExpressionType.String + ) + }; + + var projectNode = AstBuilder.CreateProject(projections, "project"); + + var query = AstBuilder.CreateQuery("Logs"); + query.Operations.Add(projectNode); + + return query; + } + + // TraceQL: select(span.name, span.duration, .service.name) + public static QueryNode TraceQLSelectExample() + { + var projections = new List + { + AstBuilder.CreateFieldProjection("name", ns: "span"), + AstBuilder.CreateFieldProjection("duration", ns: "span"), + AstBuilder.CreateFieldProjection("name", ns: "service") + }; + + var selectNode = AstBuilder.CreateProject(projections, "select"); + + var query = AstBuilder.CreateQuery(); + query.Operations.Add(selectNode); + + return query; + } + + // KQL: MyTable | where timestamp > ago(1h) | project name, duration, category = case(duration > 1000, "slow", "fast") + public static QueryNode QueryWithFilterAndProjectExample() + { + // Filter operation + var filterExpression = AstBuilder.CreateBinaryExpression( + AstBuilder.CreateIdentifier("timestamp"), + BinaryOperatorKind.GreaterThan, + AstBuilder.CreateCallExpression("ago", new List + { + AstBuilder.CreateLiteral("1h", LiteralKind.Duration) + }) + ); + var filterNode = AstBuilder.CreateFilter(filterExpression, "where"); + + // Project operation + var projections = new List + { + AstBuilder.CreateFieldProjection("name"), + AstBuilder.CreateFieldProjection("duration"), + + // Level 2 TODO: case expressions - for now showing basic conditional concept + AstBuilder.CreateProjection( + AstBuilder.CreateCallExpression("case", new List + { + AstBuilder.CreateBinaryExpression( + AstBuilder.CreateIdentifier("duration"), + BinaryOperatorKind.GreaterThan, + AstBuilder.CreateLiteral(1000, LiteralKind.Integer) + ), + AstBuilder.CreateLiteral("slow", LiteralKind.String), + AstBuilder.CreateLiteral("fast", LiteralKind.String) + }), + alias: "category", + declaredType: ExpressionType.String + ) + }; + var projectNode = AstBuilder.CreateProject(projections, "project"); + + // Create query with both operations + var operations = new List { filterNode, projectNode }; + + return AstBuilder.CreateQueryWithOperations(operations, "MyTable"); + } +} diff --git a/src/AST/KqlToCommonAstVisitor.cs b/src/AST/KqlToCommonAstVisitor.cs index 1d25f7b..2ebc965 100644 --- a/src/AST/KqlToCommonAstVisitor.cs +++ b/src/AST/KqlToCommonAstVisitor.cs @@ -43,6 +43,9 @@ public void Visit(SyntaxNode node) case SyntaxKind.FilterOperator: VisitFilterOperator(node as FilterOperator); break; + case SyntaxKind.ProjectOperator: + VisitProjectOperator(node as ProjectOperator); + break; // Handle all binary expression types case SyntaxKind.EqualExpression: case SyntaxKind.NotEqualExpression: @@ -167,6 +170,66 @@ private void VisitWhereClause(Kusto.Language.Syntax.WhereClause node) } } + private void VisitProjectOperator(ProjectOperator node) + { + if (node == null) return; + + var projections = new List(); + + // Process the expression list - ProjectOperator has Expressions property + if (node.Expressions != null) + { + foreach (var separatedElement in node.Expressions) + { + var column = separatedElement.Element; + + // For each column, we need to extract: + // 1. The expression (field reference, calculation, etc.) + // 2. The optional alias (if using '=' syntax) + + if (column is SimpleNamedExpression namedExpr) + { + // This handles cases like: field = expression or just field + string? alias = null; + + if (namedExpr.Name != null) + { + alias = namedExpr.Name.SimpleName; + } + + // Visit the expression part + if (namedExpr.Expression != null) + { + Visit(namedExpr.Expression); + + if (_expressionStack.Count > 0) + { + var expr = _expressionStack.Pop(); + var projection = AstBuilder.CreateProjection(expr, alias); + projections.Add(projection); + } + } + } + else + { + // This handles simple field references without aliases + Visit(column); + + if (_expressionStack.Count > 0) + { + var expr = _expressionStack.Pop(); + var projection = AstBuilder.CreateProjection(expr); + projections.Add(projection); + } + } + } + } + + // Create the project node and add it to the query operations + var projectNode = AstBuilder.CreateProject(projections, "project"); + _rootNode.Operations.Add(projectNode); + } + private void VisitBinaryExpression(Kusto.Language.Syntax.BinaryExpression node) { if (node == null) return; @@ -379,4 +442,4 @@ private BinaryOperatorKind MapKqlOperatorToCommonAST(string op) default: throw new NotSupportedException($"Unsupported binary operator: {op}"); } } -} \ No newline at end of file +} diff --git a/src/Program.cs b/src/Program.cs index cc2b095..03b6cbb 100644 --- a/src/Program.cs +++ b/src/Program.cs @@ -179,6 +179,19 @@ static void GenerateGraphvizForCommonAST(ASTNode node, StreamWriter writer, stri case SpecialOperatorExpression specOpExpr: label += $"\\nOperator: {specOpExpr.Operator}"; break; + + case ProjectNode projectNode: + if (!string.IsNullOrEmpty(projectNode.Keyword)) + label += $"\\nKeyword: {projectNode.Keyword}"; + label += $"\\nProjections: {projectNode.Projections.Count}"; + break; + + case ProjectionExpression projExpr: + if (!string.IsNullOrEmpty(projExpr.Alias)) + label += $"\\nAlias: {projExpr.Alias}"; + if (projExpr.GetResultType() != ExpressionType.Unknown) + label += $"\\nResultType: {projExpr.GetResultType()}"; + break; } writer.WriteLine($"\"{nodeId}\" [label=\"{label}\"];"); @@ -244,6 +257,15 @@ static void GenerateGraphvizForCommonAST(ASTNode node, StreamWriter writer, stri foreach (var item in specOpExpr.Right) GenerateGraphvizForCommonAST(item, writer, nodeId); break; + + case ProjectNode projectNode: + foreach (var projection in projectNode.Projections) + GenerateGraphvizForCommonAST(projection, writer, nodeId); + break; + + case ProjectionExpression projExpr: + GenerateGraphvizForCommonAST(projExpr.Expression, writer, nodeId); + break; } } diff --git a/tests/CommonAST.Tests/CommonASTTests.cs b/tests/CommonAST.Tests/CommonASTTests.cs index d6a054b..2b45683 100644 --- a/tests/CommonAST.Tests/CommonASTTests.cs +++ b/tests/CommonAST.Tests/CommonASTTests.cs @@ -568,6 +568,408 @@ public void CreateQuery_WithComplexOperationPipeline_CreatesCorrectly() #endregion + #region Project Node Tests + + [TestMethod] + public void CreateProject_WithSimpleFieldProjections_CreatesProjectNodeCorrectly() + { + // Arrange + var projections = new List + { + AstBuilder.CreateFieldProjection("name"), + AstBuilder.CreateFieldProjection("duration"), + AstBuilder.CreateFieldProjection("status") + }; + + // Act + var projectNode = AstBuilder.CreateProject(projections, "project"); + + // Assert + Assert.IsNotNull(projectNode); + Assert.AreEqual(NodeKind.Project, projectNode.NodeKind); + Assert.AreEqual("project", projectNode.Keyword); + Assert.AreEqual(3, projectNode.Projections.Count); + + // Verify each projection + Assert.AreEqual("name", ((Identifier)projectNode.Projections[0].Expression).Name); + Assert.AreEqual("duration", ((Identifier)projectNode.Projections[1].Expression).Name); + Assert.AreEqual("status", ((Identifier)projectNode.Projections[2].Expression).Name); + } + + [TestMethod] + public void CreateProjection_WithAlias_CreatesProjectionExpressionCorrectly() + { + // Arrange + var expression = AstBuilder.CreateIdentifier("name"); + + // Act + var projection = AstBuilder.CreateProjection(expression, "service_name"); + + // Assert + Assert.IsNotNull(projection); + Assert.AreEqual(NodeKind.ProjectionExpression, projection.NodeKind); + Assert.AreSame(expression, projection.Expression); + Assert.AreEqual("service_name", projection.Alias); + Assert.IsNull(projection.ResultType); + } + + [TestMethod] + public void CreateProjection_WithCalculatedFieldAndType_CreatesProjectionExpressionCorrectly() + { + // Arrange + var calculatedExpression = AstBuilder.CreateBinaryExpression( + AstBuilder.CreateIdentifier("duration"), + BinaryOperatorKind.Divide, + AstBuilder.CreateLiteral(1000, LiteralKind.Integer) + ); + + // Act + var projection = AstBuilder.CreateProjection( + calculatedExpression, + "duration_ms", + ExpressionType.Float + ); + + // Assert + Assert.IsNotNull(projection); + Assert.AreSame(calculatedExpression, projection.Expression); + Assert.AreEqual("duration_ms", projection.Alias); + Assert.AreEqual(ExpressionType.Float, projection.ResultType); + } + + [TestMethod] + public void CreateFieldProjection_WithNamespace_CreatesProjectionCorrectly() + { + // Arrange & Act + var projection = AstBuilder.CreateFieldProjection("name", "service_name", ExpressionType.String, "span"); + + // Assert + Assert.IsNotNull(projection); + var identifier = projection.Expression as Identifier; + Assert.IsNotNull(identifier); + Assert.AreEqual("name", identifier.Name); + Assert.AreEqual("span", identifier.Namespace); + Assert.AreEqual("service_name", projection.Alias); + Assert.AreEqual(ExpressionType.String, projection.ResultType); + } + + [TestMethod] + public void CreateProject_WithTraceQLSelectKeyword_CreatesProjectNodeCorrectly() + { + // Arrange + var projections = new List + { + AstBuilder.CreateFieldProjection("name", ns: "span"), + AstBuilder.CreateFieldProjection("duration", ns: "span") + }; + + // Act + var selectNode = AstBuilder.CreateProject(projections, "select"); + + // Assert + Assert.IsNotNull(selectNode); + Assert.AreEqual("select", selectNode.Keyword); + Assert.AreEqual(2, selectNode.Projections.Count); + } + + [TestMethod] + public void CreateProject_WithMixedProjectionTypes_CreatesProjectNodeCorrectly() + { + // Arrange + var projections = new List + { + // Simple field + AstBuilder.CreateFieldProjection("name"), + + // Field with alias + AstBuilder.CreateProjection( + AstBuilder.CreateIdentifier("duration"), + "response_time" + ), + + // Calculated field with type + AstBuilder.CreateProjection( + AstBuilder.CreateBinaryExpression( + AstBuilder.CreateIdentifier("count"), + BinaryOperatorKind.Multiply, + AstBuilder.CreateLiteral(2, LiteralKind.Integer) + ), + "double_count", + ExpressionType.Integer + ), + + // Function call with type + AstBuilder.CreateProjection( + AstBuilder.CreateCallExpression("toupper", new List + { + AstBuilder.CreateIdentifier("status") + }), + "upper_status", + ExpressionType.String + ) + }; + + // Act + var projectNode = AstBuilder.CreateProject(projections, "project"); + + // Assert + Assert.IsNotNull(projectNode); + Assert.AreEqual(4, projectNode.Projections.Count); + + // Verify simple field (no alias, no type) + Assert.IsNull(projectNode.Projections[0].Alias); + Assert.IsNull(projectNode.Projections[0].ResultType); + + // Verify field with alias (no type) + Assert.AreEqual("response_time", projectNode.Projections[1].Alias); + Assert.IsNull(projectNode.Projections[1].ResultType); + + // Verify calculated field with type + Assert.AreEqual("double_count", projectNode.Projections[2].Alias); + Assert.AreEqual(ExpressionType.Integer, projectNode.Projections[2].ResultType); + + // Verify function call with type + Assert.AreEqual("upper_status", projectNode.Projections[3].Alias); + Assert.AreEqual(ExpressionType.String, projectNode.Projections[3].ResultType); + } + + [TestMethod] + public void CreateProject_WithEmptyProjections_CreatesProjectNodeCorrectly() + { + // Arrange + var projections = new List(); + + // Act + var projectNode = AstBuilder.CreateProject(projections, "project"); + + // Assert + Assert.IsNotNull(projectNode); + Assert.AreEqual(0, projectNode.Projections.Count); + Assert.AreEqual("project", projectNode.Keyword); + } + + [TestMethod] + public void ProjectionExpression_AllExpressionTypes_CreatesCorrectly() + { + // Test Level 1 Support: Simple fields, aliases, basic arithmetic, simple functions + + // Arrange & Act - Test different expression types in projections + + // 1. Simple identifier + var simpleProjection = AstBuilder.CreateProjection(AstBuilder.CreateIdentifier("field")); + + // 2. Binary arithmetic expression + var arithmeticProjection = AstBuilder.CreateProjection( + AstBuilder.CreateBinaryExpression( + AstBuilder.CreateIdentifier("a"), + BinaryOperatorKind.Add, + AstBuilder.CreateIdentifier("b") + ), + "sum_ab", + ExpressionType.Integer + ); + + // 3. Function call expression + var functionProjection = AstBuilder.CreateProjection( + AstBuilder.CreateCallExpression("substring", new List + { + AstBuilder.CreateIdentifier("text"), + AstBuilder.CreateLiteral(0, LiteralKind.Integer), + AstBuilder.CreateLiteral(10, LiteralKind.Integer) + }), + "short_text", + ExpressionType.String + ); + + // 4. Unary expression + var unaryProjection = AstBuilder.CreateProjection( + AstBuilder.CreateUnaryExpression("-", AstBuilder.CreateIdentifier("value")), + "negative_value", + ExpressionType.Integer + ); + + // Assert + Assert.AreEqual(NodeKind.Identifier, simpleProjection.Expression.NodeKind); + Assert.AreEqual(NodeKind.BinaryExpression, arithmeticProjection.Expression.NodeKind); + Assert.AreEqual(NodeKind.CallExpression, functionProjection.Expression.NodeKind); + Assert.AreEqual(NodeKind.UnaryExpression, unaryProjection.Expression.NodeKind); + + // Verify types are set only for calculated expressions + Assert.IsNull(simpleProjection.ResultType); + Assert.AreEqual(ExpressionType.Integer, arithmeticProjection.ResultType); + Assert.AreEqual(ExpressionType.String, functionProjection.ResultType); + Assert.AreEqual(ExpressionType.Integer, unaryProjection.ResultType); + } + + [TestMethod] + public void ExpressionType_AllTypes_Covered() + { + // Test all ExpressionType enum values + var projections = new List + { + AstBuilder.CreateProjection(AstBuilder.CreateIdentifier("str_field"), "str", ExpressionType.String), + AstBuilder.CreateProjection(AstBuilder.CreateIdentifier("int_field"), "int", ExpressionType.Integer), + AstBuilder.CreateProjection(AstBuilder.CreateIdentifier("float_field"), "float", ExpressionType.Float), + AstBuilder.CreateProjection(AstBuilder.CreateIdentifier("bool_field"), "bool", ExpressionType.Boolean), + AstBuilder.CreateProjection(AstBuilder.CreateIdentifier("dur_field"), "dur", ExpressionType.Duration), + AstBuilder.CreateProjection(AstBuilder.CreateIdentifier("dt_field"), "dt", ExpressionType.DateTime), + AstBuilder.CreateProjection(AstBuilder.CreateIdentifier("guid_field"), "guid", ExpressionType.Guid), + AstBuilder.CreateProjection(AstBuilder.CreateIdentifier("dyn_field"), "dyn", ExpressionType.Dynamic) + }; + + // Act + var projectNode = AstBuilder.CreateProject(projections, "project"); + + // Assert + Assert.AreEqual(8, projectNode.Projections.Count); + Assert.AreEqual(ExpressionType.String, projectNode.Projections[0].ResultType); + Assert.AreEqual(ExpressionType.Integer, projectNode.Projections[1].ResultType); + Assert.AreEqual(ExpressionType.Float, projectNode.Projections[2].ResultType); + Assert.AreEqual(ExpressionType.Boolean, projectNode.Projections[3].ResultType); + Assert.AreEqual(ExpressionType.Duration, projectNode.Projections[4].ResultType); + Assert.AreEqual(ExpressionType.DateTime, projectNode.Projections[5].ResultType); + Assert.AreEqual(ExpressionType.Guid, projectNode.Projections[6].ResultType); + Assert.AreEqual(ExpressionType.Dynamic, projectNode.Projections[7].ResultType); + } + + #endregion + + #region Project Node Example Tests + + [TestMethod] + public void KqlSimpleProjectExample_CreatesExpectedAst() + { + // Arrange & Act + var query = Examples.KqlSimpleProjectExample(); + + // Assert + Assert.IsNotNull(query); + Assert.AreEqual("MyTable", query.Source); + Assert.AreEqual(1, query.Operations.Count); + + var projectNode = query.Operations[0] as ProjectNode; + Assert.IsNotNull(projectNode); + Assert.AreEqual("project", projectNode.Keyword); + Assert.AreEqual(3, projectNode.Projections.Count); + + // Verify projection fields + Assert.AreEqual("name", ((Identifier)projectNode.Projections[0].Expression).Name); + Assert.AreEqual("duration", ((Identifier)projectNode.Projections[1].Expression).Name); + Assert.AreEqual("status", ((Identifier)projectNode.Projections[2].Expression).Name); + } + + [TestMethod] + public void KqlProjectWithAliasesAndCalculationsExample_CreatesExpectedAst() + { + // Arrange & Act + var query = Examples.KqlProjectWithAliasesAndCalculationsExample(); + + // Assert + Assert.IsNotNull(query); + Assert.AreEqual("Logs", query.Source); + Assert.AreEqual(1, query.Operations.Count); + + var projectNode = query.Operations[0] as ProjectNode; + Assert.IsNotNull(projectNode); + Assert.AreEqual("project", projectNode.Keyword); + Assert.AreEqual(3, projectNode.Projections.Count); + + // Verify first projection (alias without calculation) + var firstProj = projectNode.Projections[0]; + Assert.AreEqual("service_name", firstProj.Alias); + Assert.IsNull(firstProj.ResultType); + Assert.AreEqual(NodeKind.Identifier, firstProj.Expression.NodeKind); + + // Verify second projection (calculated field) + var secondProj = projectNode.Projections[1]; + Assert.AreEqual("duration_ms", secondProj.Alias); + Assert.AreEqual(ExpressionType.Float, secondProj.ResultType); + Assert.AreEqual(NodeKind.BinaryExpression, secondProj.Expression.NodeKind); + + // Verify third projection (function call) + var thirdProj = projectNode.Projections[2]; + Assert.AreEqual("upper_name", thirdProj.Alias); + Assert.AreEqual(ExpressionType.String, thirdProj.ResultType); + Assert.AreEqual(NodeKind.CallExpression, thirdProj.Expression.NodeKind); + } + + [TestMethod] + public void TraceQLSelectExample_CreatesExpectedAst() + { + // Arrange & Act + var query = Examples.TraceQLSelectExample(); + + // Assert + Assert.IsNotNull(query); + Assert.IsNull(query.Source); + Assert.AreEqual(1, query.Operations.Count); + + var selectNode = query.Operations[0] as ProjectNode; + Assert.IsNotNull(selectNode); + Assert.AreEqual("select", selectNode.Keyword); + Assert.AreEqual(3, selectNode.Projections.Count); + + // Verify span.name projection + var firstProj = selectNode.Projections[0]; + var firstId = firstProj.Expression as Identifier; + Assert.IsNotNull(firstId); + Assert.AreEqual("name", firstId.Name); + Assert.AreEqual("span", firstId.Namespace); + + // Verify span.duration projection + var secondProj = selectNode.Projections[1]; + var secondId = secondProj.Expression as Identifier; + Assert.IsNotNull(secondId); + Assert.AreEqual("duration", secondId.Name); + Assert.AreEqual("span", secondId.Namespace); + + // Verify service.name projection + var thirdProj = selectNode.Projections[2]; + var thirdId = thirdProj.Expression as Identifier; + Assert.IsNotNull(thirdId); + Assert.AreEqual("name", thirdId.Name); + Assert.AreEqual("service", thirdId.Namespace); + } + + [TestMethod] + public void QueryWithFilterAndProjectExample_CreatesExpectedAst() + { + // Arrange & Act + var query = Examples.QueryWithFilterAndProjectExample(); + + // Assert + Assert.IsNotNull(query); + Assert.AreEqual("MyTable", query.Source); + Assert.AreEqual(2, query.Operations.Count); + + // Verify first operation is filter + var filterNode = query.Operations[0] as FilterNode; + Assert.IsNotNull(filterNode); + Assert.AreEqual("where", filterNode.Keyword); + + // Verify second operation is project + var projectNode = query.Operations[1] as ProjectNode; + Assert.IsNotNull(projectNode); + Assert.AreEqual("project", projectNode.Keyword); + Assert.AreEqual(3, projectNode.Projections.Count); + + // Verify simple field projections + Assert.AreEqual("name", ((Identifier)projectNode.Projections[0].Expression).Name); + Assert.AreEqual("duration", ((Identifier)projectNode.Projections[1].Expression).Name); + + // Verify complex case expression (Level 2 TODO example) + var caseProj = projectNode.Projections[2]; + Assert.AreEqual("category", caseProj.Alias); + Assert.AreEqual(ExpressionType.String, caseProj.ResultType); + Assert.AreEqual(NodeKind.CallExpression, caseProj.Expression.NodeKind); + + var caseCall = caseProj.Expression as CallExpression; + Assert.IsNotNull(caseCall); + Assert.AreEqual("case", caseCall.Callee.Name); + } + + #endregion + #region Example Tests [TestMethod] @@ -699,4 +1101,4 @@ public void SpansOnlyFilterExample_CreatesExpectedAst() #endregion } -} \ No newline at end of file +}