diff --git a/app/main.py b/app/main.py index 4e4f4b8..33ff790 100644 --- a/app/main.py +++ b/app/main.py @@ -1,5 +1,7 @@ from flask import Flask, request, Response, jsonify from flask_cors import CORS +import pandas as pd +from app.services.metrics import EyeTrackingBenchmark import numpy as np # Local imports from app @@ -39,6 +41,66 @@ def batch_predict(): return session_route.batch_predict() return Response('Invalid request method for route', status=405, mimetype='application/json') +""" +POST /api/session/benchmark + +Runs eye-tracking benchmark evaluation. + +Expected JSON: +{ + "screen_width_px": int, + "screen_width_cm": float, + "viewing_distance_cm": float, + "samples": [ + { + "True X": float, + "True Y": float, + "Predicted X": float, + "Predicted Y": float + } + ] +} +""" +""" +POST /api/session/benchmark + +Evaluates eye-tracking accuracy and precision. +""" +@app.route('/api/session/benchmark', methods=['POST']) +def run_benchmark(): + try: + data = request.get_json() + + required_keys = { + "samples", + "screen_width_px", + "screen_width_cm", + "viewing_distance_cm" + } + + if not required_keys.issubset(data.keys()): + missing = required_keys - set(data.keys()) + return jsonify({"error": f"Missing fields: {missing}"}), 400 + + df = pd.DataFrame(data["samples"]) + + benchmark = EyeTrackingBenchmark( + df=df, + screen_width_px=data["screen_width_px"], + screen_width_cm=data["screen_width_cm"], + viewing_distance_cm=data["viewing_distance_cm"] + ) + + results = { + "overall": benchmark.evaluate(), + "per_target": benchmark.evaluate_per_target() + } + + return jsonify(results), 200 + + except Exception as e: + return jsonify({"error": str(e)}), 500 + @app.route('/api/realtime-validation', methods=['POST', 'OPTIONS']) def realtime_validation(): global realtime_request_count diff --git a/app/requirements.txt b/app/requirements.txt index ba4582f..b2c3bb8 100644 --- a/app/requirements.txt +++ b/app/requirements.txt @@ -17,4 +17,5 @@ threadpoolctl==3.6.0 tzdata==2025.2 Werkzeug==3.1.3 gunicorn==23.0.0 -requests==2.31.0 \ No newline at end of file +requests==2.31.0 +reportlab \ No newline at end of file diff --git a/app/routes/session.py b/app/routes/session.py index 0a6b19b..02b0258 100644 --- a/app/routes/session.py +++ b/app/routes/session.py @@ -22,6 +22,9 @@ # from app.services import database as db from app.services import gaze_tracker +# Import utility for generating benchmark PDF reports +from app.services.benchmark.report_generator import generate_benchmark_report + # Constants @@ -72,7 +75,11 @@ def convert_nan_to_none(obj): return float(obj) if isinstance(obj, np.floating) else int(obj) return obj - +# ------------------------------------------------------------------ +# Calibration endpoint +# Generates training CSV files from calibration points and runs the +# gaze prediction model to compute calibration results. +# ------------------------------------------------------------------ def calib_results(): @@ -164,6 +171,12 @@ def calib_results(): data = convert_nan_to_none(data) return Response(json.dumps(data), status=200, mimetype='application/json') + +# ------------------------------------------------------------------ +# Batch prediction endpoint +# Uses stored calibration data to predict gaze positions for new +# iris tracking samples sent from the client. +# ------------------------------------------------------------------ def batch_predict(): try: data = request.get_json() @@ -213,4 +226,135 @@ def batch_predict(): except Exception as e: print("Erro batch_predict:", e) traceback.print_exc() - return Response("Erro interno", status=500) \ No newline at end of file + return Response("Erro interno", status=500) + +# ------------------------------------------------------------------ +# Benchmark evaluation endpoint +# +# Endpoint: +# POST /api/session//benchmark +# +# Computes benchmark metrics for eye-tracking predictions including: +# - Accuracy metrics +# - Precision metrics +# - Per-target analysis +# +# Also warns if the number of samples is small (<30) since metrics +# like p95 error may be statistically unreliable. +# ------------------------------------------------------------------ +@app.route('/api/session//benchmark', methods=['POST']) +def run_benchmark(session_id): + try: + data = request.get_json() + + samples = data.get("samples") + + if not samples: + return jsonify({"error": "Missing samples"}), 400 + + # Convert to DataFrame + df = pd.DataFrame(samples) + + # Minimum sample validation + if len(df) < 30: + warning = "Sample size is small; metrics may be statistically unreliable" + else: + warning = None + + # Get session metadata + session = Session.get(session_id) + + if not session: + return jsonify({"error": "Session not found"}), 404 + + screen_width_px = session.get("screen_width_px") + screen_width_cm = session.get("screen_width_cm") + viewing_distance_cm = session.get("viewing_distance_cm") + + # Import benchmark module + from app.services.calib_validation.metrics import EyeTrackingBenchmark + + benchmark = EyeTrackingBenchmark( + df=df, + screen_width_px=screen_width_px, + screen_width_cm=screen_width_cm, + viewing_distance_cm=viewing_distance_cm + ) + + results = { + "overall": benchmark.evaluate(), + "per_target": benchmark.evaluate_per_target() + } + + if warning: + results["warning"] = warning + + return jsonify(convert_nan_to_none(results)), 200 + + except Exception as e: + traceback.print_exc() + return jsonify({"error": str(e)}), 500 + +# ------------------------------------------------------------------ +# Benchmark report generation endpoint +# +# Endpoint: +# POST /api/session//benchmark/report +# +# Generates a downloadable PDF report summarizing benchmark results +# and visualizing true vs predicted gaze points. +# ------------------------------------------------------------------ +@app.route('/api/session//benchmark/report', methods=['POST']) +def benchmark_report(session_id): + try: + data = request.get_json() + samples = data.get("samples") + + if not samples: + return jsonify({"error": "Missing samples"}), 400 + + # Convert samples to DataFrame + df = pd.DataFrame(samples) + + # Retrieve session metadata (screen + device parameters) + session = Session.get(session_id) + + if not session: + return jsonify({"error": "Session not found"}), 404 + + screen_width_px = session.get("screen_width_px") + screen_width_cm = session.get("screen_width_cm") + viewing_distance_cm = session.get("viewing_distance_cm") + + # Import benchmark evaluator + from app.services.calib_validation.metrics import EyeTrackingBenchmark + + # Run benchmark evaluation + benchmark = EyeTrackingBenchmark( + df=df, + screen_width_px=screen_width_px, + screen_width_cm=screen_width_cm, + viewing_distance_cm=viewing_distance_cm + ) + + results = { + "overall": benchmark.evaluate(), + "per_target": benchmark.evaluate_per_target() + } + + # Path where the report will be saved + report_path = f"reports/{session_id}_benchmark_report.pdf" + + # Generate the PDF report + generate_benchmark_report( + samples, + results["overall"]["accuracy"], + report_path + ) + + # Return generated file + return send_file(report_path, as_attachment=True) + + except Exception as e: + traceback.print_exc() + return jsonify({"error": str(e)}), 500 \ No newline at end of file diff --git a/app/services/benchmark/report_generator.py b/app/services/benchmark/report_generator.py new file mode 100644 index 0000000..3a87eaf --- /dev/null +++ b/app/services/benchmark/report_generator.py @@ -0,0 +1,42 @@ +import pandas as pd +import matplotlib.pyplot as plt +from reportlab.platypus import SimpleDocTemplate, Paragraph, Spacer, Image +from reportlab.lib.styles import getSampleStyleSheet + + +def generate_benchmark_report(samples, metrics, output_path): + + df = pd.DataFrame(samples) + + # Create visualization + plt.figure() + plt.scatter(df["True X"], df["True Y"], label="True") + plt.scatter(df["Predicted X"], df["Predicted Y"], label="Predicted") + + plt.legend() + plt.title("Gaze Prediction Accuracy") + + plot_path = "benchmark_plot.png" + plt.savefig(plot_path) + + styles = getSampleStyleSheet() + + doc = SimpleDocTemplate(output_path) + elements = [] + + elements.append(Paragraph("Eye Tracking Benchmark Report", styles["Title"])) + elements.append(Spacer(1, 20)) + + elements.append( + Paragraph( + f"Mean Accuracy Error (px): {metrics['mean_accuracy_error_px']}", + styles["BodyText"], + ) + ) + + elements.append(Spacer(1, 20)) + elements.append(Image(plot_path)) + + doc.build(elements) + + return output_path \ No newline at end of file diff --git a/app/services/metrics.py b/app/services/metrics.py index 0473ecb..32e46f9 100644 --- a/app/services/metrics.py +++ b/app/services/metrics.py @@ -72,4 +72,117 @@ def func_total_accuracy(group): np.square(group["True X"] - group["Predicted X"]) + np.square(group["True Y"] - group["Predicted Y"]) ) - return np.mean(distances) # Returns average error in pixels \ No newline at end of file + return np.mean(distances) # Returns average error in pixels + + +""" +EyeTrackingBenchmark + +Implements: +- Spatial accuracy (Euclidean pixel error) +- Angular accuracy (visual angle degrees) +- Temporal precision (frame-to-frame RMS) +- Data quality (sample loss percentage) +- Per-target spatial breakdown + +Designed for benchmarking eye-tracking systems across devices and setups. +""" + + +class EyeTrackingBenchmark: + + def __init__(self, df, screen_width_px, screen_width_cm, viewing_distance_cm): + required_cols = {"True X", "True Y", "Predicted X", "Predicted Y"} + + if not required_cols.issubset(df.columns): + missing = required_cols - set(df.columns) + raise ValueError( + f"Missing required columns: {missing}. " + f"Expected columns: {required_cols}" + ) + + self.df = df + self.screen_width_px = screen_width_px + self.screen_width_cm = screen_width_cm + self.viewing_distance_cm = viewing_distance_cm + + def _euclidean_error_px(self): + dx = self.df["True X"] - self.df["Predicted X"] + dy = self.df["True Y"] - self.df["Predicted Y"] + return np.sqrt(dx**2 + dy**2) + + def accuracy_metrics(self): + errors = self._euclidean_error_px() + return { + "mean_accuracy_error_px": float(np.mean(errors)), + "median_accuracy_error_px": float(np.median(errors)), + "p95_accuracy_error_px": float(np.percentile(errors, 95)), + "mean_accuracy_error_deg": float(self._mean_error_deg(errors)) + } + + + def _mean_error_deg(self, errors_px): + pixel_size_cm = self.screen_width_cm / self.screen_width_px + errors_cm = errors_px * pixel_size_cm + errors_deg = 2 * np.degrees( + np.arctan(errors_cm / (2 * self.viewing_distance_cm)) + ) + return np.mean(errors_deg) + + def precision_metrics(self): + grouped = self.df.groupby(["True X", "True Y"]) + + rms_values = [] + + for (_, _), group in grouped: + x = group["Predicted X"].values + y = group["Predicted Y"].values + + dx = x - np.mean(x) + dy = y - np.mean(y) + + rms = np.sqrt(np.mean(dx ** 2 + dy ** 2)) + rms_values.append(rms) + + return { + "mean_rms_px": float(np.mean(rms_values)), + "median_rms_px": float(np.median(rms_values)) + } + + def data_quality_metrics(self): + total = len(self.df) + missing = self.df["Predicted X"].isna().sum() + return { + "data_loss_percent": float((missing / total) * 100), + "num_samples": int(total) + } + + def evaluate(self): + return { + "accuracy": self.accuracy_metrics(), + "precision": self.precision_metrics(), + "data_quality": self.data_quality_metrics() + } + + def evaluate_per_target(self): + grouped = self.df.groupby(["True X", "True Y"]) + + results = [] + + for (tx, ty), group in grouped: + bench = EyeTrackingBenchmark( + group, + self.screen_width_px, + self.screen_width_cm, + self.viewing_distance_cm + ) + + metrics = bench.accuracy_metrics() + + results.append({ + "true_x": float(tx), + "true_y": float(ty), + **metrics + }) + + return results \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 4fa362a..f752873 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,11 +4,13 @@ charset-normalizer==3.4.7 click==8.1.8 contourpy==1.3.3 cycler==0.12.1 +dotenv==0.9.9 Flask==3.1.0 flask-cors==5.0.1 fonttools==4.63.0 gunicorn==23.0.0 idna==3.18 +iniconfig==2.3.0 itsdangerous==2.2.0 Jinja2==3.1.6 joblib==1.4.2 @@ -19,9 +21,14 @@ numpy==2.2.4 packaging==26.2 pandas==2.2.3 pillow==12.2.0 +pluggy==1.6.0 +Pygments==2.20.0 pyparsing==3.3.2 +pytest==9.0.3 python-dateutil==2.9.0.post0 +python-dotenv==1.2.2 pytz==2025.2 +reportlab==4.5.1 requests==2.31.0 scikit-learn==1.6.1 scipy==1.15.2 @@ -30,4 +37,3 @@ threadpoolctl==3.6.0 tzdata==2025.2 urllib3==2.7.0 Werkzeug==3.1.3 -dotenv==0.9.9 \ No newline at end of file