WIP: Attribute system - attributes.pm, MODIFY/FETCH callbacks

dev/prompts/test-failures-not-quick-fix.md

@@ -745,8 +745,8 @@ After rebasing `feature/test-failure-fixes` onto latest master, the following re
 
 ## Recommended Next Steps
 
-1. **(?{...}) non-fatal workaround** (Medium) - change `UNIMPLEMENTED_CODE_BLOCK` from fatal to `(?:)` fallback - 500+ tests
-2. **64-bit integer ops** (Medium-Hard) - unsigned semantics, overflow handling
-3. **caller() extended fields** (Medium-Hard) - wantarray, evaltext, is_require
-4. **Attribute system** (Medium-Hard) - attributes.pm module, MODIFY_*_ATTRIBUTES callbacks
+1. ~~**(?{...}) non-fatal workaround**~~ - **NOT AN OPTION** - silently replacing code blocks with no-ops would mask real failures and produce incorrect test results
+2. **64-bit integer ops** (Medium-Hard) - Note: PerlOnJava declares itself as 32-bit, so many of these test failures may be expected/irrelevant
+3. **Attribute system** (Medium-Hard) - attributes.pm module, MODIFY_*_ATTRIBUTES callbacks - **NEXT TARGET**
+4. **caller() extended fields** (Medium-Hard) - wantarray, evaltext, is_require
 5. **op/for.t glob/read-only regression** (Medium) - from master's GlobalVariable.java changes

src/main/java/org/perlonjava/backend/bytecode/BytecodeCompiler.java

@@ -11,6 +11,7 @@
 import org.perlonjava.frontend.semantic.ScopedSymbolTable;
 import org.perlonjava.frontend.semantic.SymbolTable;
 import org.perlonjava.runtime.debugger.DebugState;
+import org.perlonjava.runtime.perlmodule.Attributes;
 import org.perlonjava.runtime.perlmodule.Strict;
 import org.perlonjava.runtime.runtimetypes.*;
 
@@ -2348,6 +2349,9 @@ void compileVariableDeclaration(OperatorNode node, String op) {
                             default -> throwCompilerException("Unsupported variable type: " + sigil);
                         }
 
+                        // Runtime attribute dispatch for state variables with attributes
+                        emitVarAttrsIfNeeded(node, reg, sigil);
+
                         // If this is a declared reference, create a reference to it
                         if (isDeclaredReference && currentCallContext != RuntimeContextType.VOID) {
                             int refReg = allocateRegister();
@@ -2381,6 +2385,9 @@ void compileVariableDeclaration(OperatorNode node, String op) {
                         default -> throwCompilerException("Unsupported variable type: " + sigil);
                     }
 
+                    // Runtime attribute dispatch for my variables with attributes
+                    emitVarAttrsIfNeeded(node, reg, sigil);
+
                     // If this is a declared reference, create a reference to it
                     if (isDeclaredReference && currentCallContext != RuntimeContextType.VOID) {
                         int refReg = allocateRegister();
@@ -4158,9 +4165,19 @@ void compileVariableReference(OperatorNode node, String op) {
             if (node.operand != null) {
                 // Special case: \&name — CODE is already a reference type.
                 // Emit LOAD_GLOBAL_CODE directly without CREATE_REF, matching JVM compiler.
+                // Also set isSymbolicReference so defined(\&stub) returns true, matching
+                // the JVM backend's createCodeReference behavior.
                 if (node.operand instanceof OperatorNode operandOp
                         && operandOp.operator.equals("&")
-                        && operandOp.operand instanceof IdentifierNode) {
+                        && operandOp.operand instanceof IdentifierNode idNode) {
+                    // Set isSymbolicReference before loading, so defined(\&Name) returns true
+                    String subName = NameNormalizer.normalizeVariableName(
+                            idNode.name, getCurrentPackage());
+                    RuntimeScalar codeRef = GlobalVariable.getGlobalCodeRef(subName);
+                    if (codeRef.type == RuntimeScalarType.CODE
+                            && codeRef.value instanceof RuntimeCode rc) {
+                        rc.isSymbolicReference = true;
+                    }
                     node.operand.accept(this);
                     // lastResultReg already holds the CODE scalar — no wrapping needed
                     return;
@@ -4329,6 +4346,32 @@ int addToStringPool(String str) {
         return index;
     }
 
+    /**
+     * Emit DISPATCH_VAR_ATTRS opcode if the node has variable attributes.
+     * Called after a my/state variable is initialized in its register.
+     */
+    @SuppressWarnings("unchecked")
+    void emitVarAttrsIfNeeded(OperatorNode node, int varReg, String sigil) {
+        if (node.annotations == null || !node.annotations.containsKey("attributes")) return;
+
+        List<String> attrs = (List<String>) node.annotations.get("attributes");
+        String packageName = (String) node.annotations.get("attributePackage");
+        if (packageName == null) packageName = getCurrentPackage();
+
+        String fileName = sourceName;
+        int lineNum = sourceLine;
+
+        // Store metadata in constant pool as Object[]
+        Object[] data = new Object[]{
+                packageName, sigil, attrs.toArray(new String[0]), fileName, lineNum
+        };
+        int constIdx = addToConstantPool(data);
+
+        emit(Opcodes.DISPATCH_VAR_ATTRS);
+        emitReg(varReg);
+        emit(constIdx);
+    }
+
     private int addToConstantPool(Object obj) {
         // Use HashMap for O(1) lookup instead of O(n) ArrayList.indexOf()
         Integer cached = constantPoolIndex.get(obj);
@@ -4789,6 +4832,11 @@ private void visitAnonymousSubroutine(SubroutineNode node) {
             // No closures - just wrap the InterpretedCode
             RuntimeScalar codeScalar = new RuntimeScalar(subCode);
             subCode.__SUB__ = codeScalar;  // Set __SUB__ for self-reference
+            // Dispatch MODIFY_CODE_ATTRIBUTES for anonymous subs with non-builtin attributes
+            if (subCode.attributes != null && !subCode.attributes.isEmpty()
+                    && subCode.packageName != null) {
+                Attributes.runtimeDispatchModifyCodeAttributes(subCode.packageName, codeScalar);
+            }
             int constIdx = addToConstantPool(codeScalar);
             emit(Opcodes.LOAD_CONST);
             emitReg(codeReg);

src/main/java/org/perlonjava/backend/bytecode/BytecodeInterpreter.java

@@ -1956,6 +1956,10 @@ public static RuntimeList execute(InterpretedCode code, RuntimeArray args, int c
                                 pc = SlowOpcodeHandler.executeArrayKVSliceDelete(bytecode, pc, registers);
                             }
 
+                            case Opcodes.DISPATCH_VAR_ATTRS -> {
+                                pc = SlowOpcodeHandler.executeDispatchVarAttrs(bytecode, pc, registers, code.constants);
+                            }
+
                             default -> {
                                 int opcodeInt = opcode;
                                 throw new RuntimeException(

src/main/java/org/perlonjava/backend/bytecode/CompileAssignment.java

@@ -362,6 +362,10 @@ public static void compileAssignmentOperator(BytecodeCompiler bytecodeCompiler,
                                 bytecodeCompiler.emit(persistId);
 
                                 bytecodeCompiler.registerVariable(varName, reg);
+
+                                // Runtime attribute dispatch for state variables with attributes
+                                bytecodeCompiler.emitVarAttrsIfNeeded(leftOp, reg, "$");
+
                                 bytecodeCompiler.lastResultReg = reg;
                                 return;
                             }
@@ -375,9 +379,23 @@ public static void compileAssignmentOperator(BytecodeCompiler bytecodeCompiler,
                             // Now allocate register for new lexical variable and add to symbol table
                             int reg = bytecodeCompiler.addVariable(varName, "my");
 
-                            bytecodeCompiler.emit(Opcodes.MY_SCALAR);
-                            bytecodeCompiler.emitReg(reg);
-                            bytecodeCompiler.emitReg(valueReg);
+                            boolean hasAttrs = leftOp.annotations != null
+                                    && leftOp.annotations.containsKey("attributes");
+                            if (hasAttrs) {
+                                // When attributes are present (e.g., my $x : TieLoop = $i),
+                                // we must create the scalar first, dispatch attributes (which
+                                // may tie the variable), then assign the value so STORE fires.
+                                bytecodeCompiler.emit(Opcodes.LOAD_UNDEF);
+                                bytecodeCompiler.emitReg(reg);
+                                bytecodeCompiler.emitVarAttrsIfNeeded(leftOp, reg, "$");
+                                bytecodeCompiler.emit(Opcodes.SET_SCALAR);
+                                bytecodeCompiler.emitReg(reg);
+                                bytecodeCompiler.emitReg(valueReg);
+                            } else {
+                                bytecodeCompiler.emit(Opcodes.MY_SCALAR);
+                                bytecodeCompiler.emitReg(reg);
+                                bytecodeCompiler.emitReg(valueReg);
+                            }
 
                             bytecodeCompiler.lastResultReg = reg;
                             return;
@@ -432,6 +450,9 @@ public static void compileAssignmentOperator(BytecodeCompiler bytecodeCompiler,
                             bytecodeCompiler.emitReg(arrayReg);
                             bytecodeCompiler.emitReg(listReg);
 
+                            // Runtime attribute dispatch for my variables with attributes
+                            bytecodeCompiler.emitVarAttrsIfNeeded(leftOp, arrayReg, "@");
+
                             if (rhsContext == RuntimeContextType.SCALAR) {
                                 int countReg = bytecodeCompiler.allocateRegister();
                                 bytecodeCompiler.emit(Opcodes.ARRAY_SIZE);
@@ -490,6 +511,9 @@ public static void compileAssignmentOperator(BytecodeCompiler bytecodeCompiler,
                             bytecodeCompiler.emitReg(hashReg);
                             bytecodeCompiler.emitReg(listReg);
 
+                            // Runtime attribute dispatch for my variables with attributes
+                            bytecodeCompiler.emitVarAttrsIfNeeded(leftOp, hashReg, "%");
+
                             bytecodeCompiler.lastResultReg = hashReg;
                             return;
                         }

src/main/java/org/perlonjava/backend/bytecode/InterpretedCode.java

@@ -233,6 +233,10 @@ public InterpreterState.InterpreterFrame getOrCreateFrame(String packageName, St
      */
     @Override
     public RuntimeList apply(RuntimeArray args, int callContext) {
+        // Return cached constant value if this sub has been const-folded
+        if (constantValue != null) {
+            return new RuntimeList(constantValue);
+        }
         // Push args for getCallerArgs() support (used by List::Util::any/all/etc.)
         // This matches what RuntimeCode.apply() does for JVM-compiled subs
         RuntimeCode.pushArgs(args);
@@ -253,6 +257,10 @@ public RuntimeList apply(RuntimeArray args, int callContext) {
 
     @Override
     public RuntimeList apply(String subroutineName, RuntimeArray args, int callContext) {
+        // Return cached constant value if this sub has been const-folded
+        if (constantValue != null) {
+            return new RuntimeList(constantValue);
+        }
         // Push args for getCallerArgs() support (used by List::Util::any/all/etc.)
         RuntimeCode.pushArgs(args);
         // Push warning bits for FATAL warnings support

src/main/java/org/perlonjava/backend/bytecode/OpcodeHandlerExtended.java

@@ -1,6 +1,7 @@
 package org.perlonjava.backend.bytecode;
 
 import org.perlonjava.runtime.operators.*;
+import org.perlonjava.runtime.perlmodule.Attributes;
 import org.perlonjava.runtime.regex.RuntimeRegex;
 import org.perlonjava.runtime.runtimetypes.*;
 
@@ -890,6 +891,13 @@ public static int executeCreateClosure(int[] bytecode, int pc, RuntimeBase[] reg
         RuntimeScalar codeRef = new RuntimeScalar(closureCode);
         closureCode.__SUB__ = codeRef;
         registers[rd] = codeRef;
+
+        // Dispatch MODIFY_CODE_ATTRIBUTES for anonymous subs with non-builtin attributes
+        // Pass isClosure=true since CREATE_CLOSURE always creates a closure
+        if (closureCode.attributes != null && !closureCode.attributes.isEmpty()
+                && closureCode.packageName != null) {
+            Attributes.runtimeDispatchModifyCodeAttributes(closureCode.packageName, codeRef, true);
+        }
         return pc;
     }
 

src/main/java/org/perlonjava/backend/bytecode/Opcodes.java

@@ -2124,6 +2124,14 @@ public class Opcodes {
      */
     public static final short ARRAY_SLICE_DELETE_LOCAL = 450;
 
+    // variable attribute dispatch
+    /**
+     * Dispatch MODIFY_*_ATTRIBUTES at runtime for my/state variable declarations.
+     * Format: DISPATCH_VAR_ATTRS var_reg const_idx
+     * const_idx points to Object[] in constant pool: [packageName, sigil, String[] attrs, fileName, lineNum]
+     */
+    public static final short DISPATCH_VAR_ATTRS = 451;
+
     private Opcodes() {
     } // Utility class - no instantiation
 }

src/main/java/org/perlonjava/backend/bytecode/SlowOpcodeHandler.java

@@ -1369,4 +1369,27 @@ public static int executeCodeDerefNonStrict(int[] bytecode, int pc,
 
         return pc;
     }
+
+    /**
+     * Dispatch MODIFY_*_ATTRIBUTES at runtime for my/state variable declarations.
+     * Format: DISPATCH_VAR_ATTRS var_reg const_idx
+     * const_idx points to Object[] in constant pool: [packageName, sigil, String[] attrs, fileName, lineNum]
+     */
+    public static int executeDispatchVarAttrs(int[] bytecode, int pc,
+                                               RuntimeBase[] registers, Object[] constants) {
+        int varReg = bytecode[pc++];
+        int constIdx = bytecode[pc++];
+
+        Object[] data = (Object[]) constants[constIdx];
+        String packageName = (String) data[0];
+        String sigil = (String) data[1];
+        String[] attributes = (String[]) data[2];
+        String fileName = (String) data[3];
+        int lineNum = (Integer) data[4];
+
+        org.perlonjava.runtime.perlmodule.Attributes.runtimeDispatchModifyVariableAttributes(
+                packageName, registers[varReg], sigil, attributes, fileName, lineNum);
+
+        return pc;
+    }
 }

src/main/java/org/perlonjava/backend/jvm/EmitSubroutine.java

@@ -314,11 +314,11 @@ public static void emitSubroutine(EmitterContext ctx, SubroutineNode node) {
             // Set prototype if needed
             if (node.prototype != null) {
                 mv.visitInsn(Opcodes.DUP);
-                mv.visitMethodInsn(Opcodes.INVOKEVIRTUAL,
+                mv.visitFieldInsn(Opcodes.GETFIELD,
                         "org/perlonjava/runtime/runtimetypes/RuntimeScalar",
-                        "getCode",
-                        "()Lorg/perlonjava/runtime/runtimetypes/RuntimeCode;",
-                        false);
+                        "value",
+                        "Ljava/lang/Object;");
+                mv.visitTypeInsn(Opcodes.CHECKCAST, "org/perlonjava/runtime/runtimetypes/RuntimeCode");
                 mv.visitLdcInsn(node.prototype);
                 mv.visitFieldInsn(Opcodes.PUTFIELD,
                         "org/perlonjava/runtime/runtimetypes/RuntimeCode",
@@ -327,6 +327,76 @@ public static void emitSubroutine(EmitterContext ctx, SubroutineNode node) {
             }
         }
 
+        // Set attributes if needed (after try-catch, both paths leave RuntimeScalar on stack)
+        if (node.attributes != null && !node.attributes.isEmpty()) {
+            mv.visitInsn(Opcodes.DUP);
+            mv.visitFieldInsn(Opcodes.GETFIELD,
+                    "org/perlonjava/runtime/runtimetypes/RuntimeScalar",
+                    "value",
+                    "Ljava/lang/Object;");
+            mv.visitTypeInsn(Opcodes.CHECKCAST, "org/perlonjava/runtime/runtimetypes/RuntimeCode");
+            // Create a new ArrayList and populate it
+            mv.visitTypeInsn(Opcodes.NEW, "java/util/ArrayList");
+            mv.visitInsn(Opcodes.DUP);
+            mv.visitMethodInsn(Opcodes.INVOKESPECIAL,
+                    "java/util/ArrayList",
+                    "<init>",
+                    "()V",
+                    false);
+            for (String attr : node.attributes) {
+                mv.visitInsn(Opcodes.DUP);
+                mv.visitLdcInsn(attr);
+                mv.visitMethodInsn(Opcodes.INVOKEINTERFACE,
+                        "java/util/List",
+                        "add",
+                        "(Ljava/lang/Object;)Z",
+                        true);
+                mv.visitInsn(Opcodes.POP); // pop boolean return of add()
+            }
+            mv.visitFieldInsn(Opcodes.PUTFIELD,
+                    "org/perlonjava/runtime/runtimetypes/RuntimeCode",
+                    "attributes",
+                    "Ljava/util/List;");
+        }
+
+        // Dispatch MODIFY_CODE_ATTRIBUTES for anonymous subs with non-builtin attributes.
+        // Named subs have their dispatch in SubroutineParser.handleNamedSub at compile time.
+        // Anonymous subs need runtime dispatch because the code ref only exists at runtime.
+        if (node.name == null && node.attributes != null && !node.attributes.isEmpty()) {
+            java.util.Set<String> builtinAttrs = java.util.Set.of("lvalue", "method", "const");
+            boolean hasNonBuiltin = false;
+            for (String attr : node.attributes) {
+                String name = attr.startsWith("-") ? attr.substring(1) : attr;
+                int parenIdx = name.indexOf('(');
+                String baseName = parenIdx >= 0 ? name.substring(0, parenIdx) : name;
+                if (!builtinAttrs.contains(baseName) && !baseName.equals("prototype")) {
+                    hasNonBuiltin = true;
+                    break;
+                }
+            }
+            if (hasNonBuiltin) {
+                // Determine if this sub is a closure (captures outer lexical variables).
+                // Closures get closure prototype semantics: MODIFY_CODE_ATTRIBUTES receives
+                // the prototype (non-callable), and the expression result is a callable clone.
+                boolean isClosure = visibleVariables.size() > skipVariables;
+
+                // Stack: [RuntimeScalar(codeRef)]
+                mv.visitInsn(Opcodes.DUP);
+                // Stack: [codeRef, codeRef]
+                mv.visitLdcInsn(ctx.symbolTable.getCurrentPackage());
+                mv.visitInsn(Opcodes.SWAP);
+                // Stack: [codeRef, pkg, codeRef]
+                mv.visitInsn(isClosure ? Opcodes.ICONST_1 : Opcodes.ICONST_0);
+                // Stack: [codeRef, pkg, codeRef, isClosure]
+                mv.visitMethodInsn(Opcodes.INVOKESTATIC,
+                        "org/perlonjava/runtime/perlmodule/Attributes",
+                        "runtimeDispatchModifyCodeAttributes",
+                        "(Ljava/lang/String;Lorg/perlonjava/runtime/runtimetypes/RuntimeScalar;Z)V",
+                        false);
+                // Stack: [codeRef] (codeRef.value now points to clone if isClosure)
+            }
+        }
+
         // 6. Clean up the stack if context is VOID
         if (ctx.contextType == RuntimeContextType.VOID) {
             mv.visitInsn(Opcodes.POP); // Remove the RuntimeScalar object from the stack