diff --git a/.github/workflows/validate-and-process.yml b/.github/workflows/validate-and-process.yml
index 9465caa35..387e17477 100644
--- a/.github/workflows/validate-and-process.yml
+++ b/.github/workflows/validate-and-process.yml
@@ -3,13 +3,55 @@ name: Validate and Process EvalAI Challenge
on:
push:
branches:
- - challenge
+ - 'challenge'
+ - 'challenge-*-*'
+ pull_request:
+ types: [opened, synchronize, reopened, edited]
permissions:
contents: read
issues: write
jobs:
+ detect-and-validate-challenge-branch:
+ runs-on: ubuntu-latest
+ outputs:
+ is_challenge_branch: ${{ steps.branch-check.outputs.is_challenge_branch }}
+ challenge_branch: ${{ steps.branch-check.outputs.challenge_branch }}
+ steps:
+ - name: Detect and validate challenge branch
+ id: branch-check
+ run: |
+ # Get the actual branch name
+ if [ "${{ github.event_name }}" == "pull_request" ]; then
+ BRANCH="${{ github.head_ref }}"
+ else
+ BRANCH="${{ github.ref_name }}"
+ fi
+
+ echo "š Analyzing branch: $BRANCH"
+ echo "challenge_branch=$BRANCH" >> $GITHUB_OUTPUT
+
+ # Check if this is a challenge branch (challenge or challenge-YYYY-version)
+ if [[ "$BRANCH" =~ ^challenge(-[0-9]{4}-.*)?$ ]]; then
+ echo "is_challenge_branch=true" >> $GITHUB_OUTPUT
+ echo "ā
Valid challenge branch detected: $BRANCH"
+
+ # Extract branch suffix for versioning
+ if [[ "$BRANCH" =~ ^challenge-([0-9]{4}-.+)$ ]]; then
+ SUFFIX="${BASH_REMATCH[1]}"
+ echo "branch_suffix=$SUFFIX" >> $GITHUB_OUTPUT
+ echo "š Branch version: $SUFFIX"
+ else
+ echo "branch_suffix=main" >> $GITHUB_OUTPUT
+ echo "š Main challenge branch (no suffix)"
+ fi
+ else
+ echo "is_challenge_branch=false" >> $GITHUB_OUTPUT
+ echo "ā¹ļø Not a challenge branch: $BRANCH"
+ echo "āļø Challenge processing will be skipped (valid patterns: 'challenge' or 'challenge-YYYY-version')"
+ fi
+
validate-host-config:
runs-on: ubuntu-latest
outputs:
@@ -120,9 +162,10 @@ jobs:
process-evalai-challenge:
- needs: [validate-host-config, check-self-hosted-requirements]
+ needs: [detect-and-validate-challenge-branch, validate-host-config, check-self-hosted-requirements]
if: |
always() &&
+ needs.detect-and-validate-challenge-branch.outputs.is_challenge_branch == 'true' &&
needs.validate-host-config.outputs.is_valid == 'true' &&
(needs.check-self-hosted-requirements.result == 'success' || needs.check-self-hosted-requirements.result == 'skipped')
runs-on: ${{ needs.validate-host-config.outputs.requires_self_hosted == 'true' && 'self-hosted' || 'ubuntu-latest' }}
@@ -131,7 +174,7 @@ jobs:
- name: Checkout challenge branch
uses: actions/checkout@v3
with:
- ref: challenge
+ ref: ${{ needs.detect-and-validate-challenge-branch.outputs.challenge_branch }}
- name: Set up Python (GitHub-hosted only)
if: needs.validate-host-config.outputs.requires_self_hosted != 'true'
@@ -183,10 +226,11 @@ jobs:
run: |
echo "š VALIDATING CHALLENGE CONFIGURATION"
echo "====================================="
- python3 github/challenge_processing_script.py
+ python3 github/challenge_processing_script.py ${{ needs.detect-and-validate-challenge-branch.outputs.challenge_branch }}
env:
IS_VALIDATION: 'True'
GITHUB_CONTEXT: ${{ toJson(github) }}
+ GITHUB_REPOSITORY: ${{ github.repository }}
GITHUB_AUTH_TOKEN: ${{ secrets.AUTH_TOKEN }}
- name: Validate challenge configuration (Self-hosted with Docker)
@@ -202,25 +246,27 @@ jobs:
-e GITHUB_CONTEXT='${{ toJson(github) }}' \
-e GITHUB_REPOSITORY='${{ github.repository }}' \
-e GITHUB_AUTH_TOKEN='${{ secrets.AUTH_TOKEN }}' \
+ -e GIT_BRANCH='${{ needs.detect-and-validate-challenge-branch.outputs.challenge_branch }}' \
python:3.9-slim \
bash -c "
pip install -r github/requirements.txt &&
- python3 github/challenge_processing_script.py
+ python3 github/challenge_processing_script.py \$GIT_BRANCH
"
- name: Create or update challenge (GitHub-hosted)
- if: success() && needs.validate-host-config.outputs.requires_self_hosted != 'true'
+ if: github.event_name == 'push' && success() && needs.validate-host-config.outputs.requires_self_hosted != 'true'
run: |
echo "š CREATING/UPDATING CHALLENGE"
echo "=============================="
- python3 github/challenge_processing_script.py
+ python3 github/challenge_processing_script.py ${{ needs.detect-and-validate-challenge-branch.outputs.challenge_branch }}
env:
IS_VALIDATION: 'False'
GITHUB_CONTEXT: ${{ toJson(github) }}
+ GITHUB_REPOSITORY: ${{ github.repository }}
GITHUB_AUTH_TOKEN: ${{ secrets.AUTH_TOKEN }}
- name: Create or update challenge (Self-hosted with Docker)
- if: success() && needs.validate-host-config.outputs.requires_self_hosted == 'true'
+ if: github.event_name == 'push' && success() && needs.validate-host-config.outputs.requires_self_hosted == 'true'
run: |
echo "š CREATING/UPDATING CHALLENGE (Docker)"
echo "========================================"
@@ -232,8 +278,9 @@ jobs:
-e GITHUB_CONTEXT='${{ toJson(github) }}' \
-e GITHUB_REPOSITORY='${{ github.repository }}' \
-e GITHUB_AUTH_TOKEN='${{ secrets.AUTH_TOKEN }}' \
+ -e GIT_BRANCH='${{ needs.detect-and-validate-challenge-branch.outputs.challenge_branch }}' \
python:3.9-slim \
bash -c "
pip install -r github/requirements.txt &&
- python3 github/challenge_processing_script.py
+ python3 github/challenge_processing_script.py \$GIT_BRANCH
"
diff --git a/README.md b/README.md
index a078f9c78..15dc13226 100644
--- a/README.md
+++ b/README.md
@@ -48,18 +48,22 @@ If you are looking for a simple challenge configuration that you can replicate t
5. Create a branch with name `challenge` in the forked repository from the `master` branch.
Note: Only changes in `challenge` branch will be synchronized with challenge on EvalAI.
+You may maintain multiple versions by using additional branches whose names start with `challenge-` (e.g. `challenge-2024`, `challenge-v2`).
+Note: The CI pipeline only runs for branches that match the pattern `challenge` or `challenge-*`. Any other branch name will be ignored.
+
+If you trigger the processing script manually and do **not** pass a branch argument, it will assume the branch name is `challenge` by default.
6. Add `evalai_user_auth_token` and `host_team_pk` in `github/host_config.json`.
7. Read [EvalAI challenge creation documentation](https://evalai.readthedocs.io/en/latest/configuration.html) to know more about how you want to structure your challenge. Once you are ready, start making changes in the yaml file, HTML templates, evaluation script according to your need.
-8. Commit the changes and push the `challenge` branch in the repository and wait for the build to complete. View the [logs of your build](https://docs.github.com/en/free-pro-team@latest/actions/managing-workflow-runs/using-workflow-run-logs#viewing-logs-to-diagnose-failures).
+8. Commit the changes and push the relevant `challenge*` branch to the repository and wait for the build to complete. View the [logs of your build](https://docs.github.com/en/free-pro-team@latest/actions/managing-workflow-runs/using-workflow-run-logs#viewing-logs-to-diagnose-failures).
-9. If challenge config contains errors then a `issue` will be opened automatically in the repository with the errors otherwise the challenge will be created on EvalAI.
+9. If the challenge config contains errors, a GitHub **Issue** will be opened automatically in the repository with the error details; otherwise the challenge will be created on EvalAI.
10. Go to [Hosted Challenges](https://eval.ai/web/hosted-challenges) to view your challenge. The challenge will be publicly available once EvalAI admin approves the challenge.
-11. To update the challenge on EvalAI, make changes in the repository and push on `challenge` branch and wait for the build to complete.
+11. To update the challenge on EvalAI, make changes in the repository and push to the corresponding `challenge*` branch and wait for the build to complete.
## Add custom dependencies for evaluation (Optional)
To add custom dependency packages in the evaluation script, refer to [this guide](./evaluation_script/dependency-installation.md).
diff --git a/challenge_config.yaml b/challenge_config.yaml
index 6afc14ce4..423596821 100755
--- a/challenge_config.yaml
+++ b/challenge_config.yaml
@@ -1,164 +1,177 @@
-# If you are not sure what all these fields mean, please refer our documentation here:
-# https://evalai.readthedocs.io/en/latest/configuration.html
-title: Random Number Generator Challenge
-short_description: Random number generation challenge for each submission
-description: templates/description.html
-evaluation_details: templates/evaluation_details.html
-terms_and_conditions: templates/terms_and_conditions.html
-image: logo.jpg
-submission_guidelines: templates/submission_guidelines.html
-leaderboard_description: Lorem ipsum dolor sit amet, consectetur adipiscing elit. Cras egestas a libero nec sagittis.
-evaluation_script: evaluation_script.zip
-remote_evaluation: False
-start_date: 2019-01-01 00:00:00
-end_date: 2099-05-31 23:59:59
-published: True
-tags:
- - random-number-generation
- - machine-learning
- - data-science
- - computer-vision
-leaderboard:
- - id: 1
- schema:
- {
- "labels": ["Metric1", "Metric2", "Metric3", "Total"],
- "default_order_by": "Total",
- "metadata": {
- "Metric1": {
- "sort_ascending": True,
- "description": "Lorem ipsum dolor sit amet, consectetur adipiscing elit.",
- },
- "Metric2": {
- "sort_ascending": True,
- "description": "Lorem ipsum dolor sit amet, consectetur adipiscing elit.",
- }
- }
- }
-
-challenge_phases:
- - id: 1
- name: Dev Phase
- description: templates/challenge_phase_1_description.html
- leaderboard_public: False
- is_public: True
- challenge: 1
- is_active: True
- max_concurrent_submissions_allowed: 3
- allowed_email_ids: []
- disable_logs: False
- is_submission_public: True
- start_date: 2019-01-19 00:00:00
- end_date: 2099-04-25 23:59:59
- test_annotation_file: annotations/test_annotations_devsplit.json
- codename: dev
- max_submissions_per_day: 5
- max_submissions_per_month: 50
- max_submissions: 50
- default_submission_meta_attributes:
- - name: method_name
- is_visible: True
- - name: method_description
- is_visible: True
- - name: project_url
- is_visible: True
- - name: publication_url
- is_visible: True
- submission_meta_attributes:
- - name: TextAttribute
- description: Sample
- type: text
- required: False
- - name: SingleOptionAttribute
- description: Sample
- type: radio
- options: ["A", "B", "C"]
- - name: MultipleChoiceAttribute
- description: Sample
- type: checkbox
- options: ["alpha", "beta", "gamma"]
- - name: TrueFalseField
- description: Sample
- type: boolean
- required: True
- is_restricted_to_select_one_submission: False
- is_partial_submission_evaluation_enabled: False
- allowed_submission_file_types: ".json, .zip, .txt, .tsv, .gz, .csv, .h5, .npy, .npz"
- - id: 2
- name: Test Phase
- description: templates/challenge_phase_2_description.html
- leaderboard_public: True
- is_public: True
- challenge: 2
- is_active: True
- max_concurrent_submissions_allowed: 3
- allowed_email_ids: []
- disable_logs: False
- is_submission_public: True
- start_date: 2019-01-01 00:00:00
- end_date: 2099-05-24 23:59:59
- test_annotation_file: annotations/test_annotations_testsplit.json
- codename: test
- max_submissions_per_day: 5
- max_submissions_per_month: 50
- max_submissions: 50
- default_submission_meta_attributes:
- - name: method_name
- is_visible: True
- - name: method_description
- is_visible: True
- - name: project_url
- is_visible: True
- - name: publication_url
- is_visible: True
- submission_meta_attributes:
- - name: TextAttribute
- description: Sample
- type: text
- - name: SingleOptionAttribute
- description: Sample
- type: radio
- options: ["A", "B", "C"]
- - name: MultipleChoiceAttribute
- description: Sample
- type: checkbox
- options: ["alpha", "beta", "gamma"]
- - name: TrueFalseField
- description: Sample
- type: boolean
- is_restricted_to_select_one_submission: False
- is_partial_submission_evaluation_enabled: False
-
-dataset_splits:
- - id: 1
- name: Train Split
- codename: train_split
- - id: 2
- name: Test Split
- codename: test_split
-
-challenge_phase_splits:
- - challenge_phase_id: 1
- leaderboard_id: 1
- dataset_split_id: 1
- visibility: 1
- leaderboard_decimal_precision: 2
- is_leaderboard_order_descending: True
- show_execution_time: True
- show_leaderboard_by_latest_submission: True
- - challenge_phase_id: 2
- leaderboard_id: 1
- dataset_split_id: 1
- visibility: 3
- leaderboard_decimal_precision: 2
- is_leaderboard_order_descending: True
- showeceution_time: False
- show_leaderboard_by_latest_submission: False
- - challenge_phase_id: 2
- leaderboard_id: 1
- dataset_split_id: 2
- visibility: 1
- leaderboard_decimal_precision: 2
- is_leaderboard_order_descending: True
- show_execution_time: True
- show_leaderboard_by_latest_submission: True
+# If you are not sure what all these fields mean, please refer our documentation here:
+# https://evalai.readthedocs.io/en/latest/configuration.html
+title: Advanced Multi-Phase AI Challenge
+short_description: Multi-phase challenge with development and validation stages
+description: templates/description.html
+evaluation_details: templates/evaluation_details.html
+terms_and_conditions: templates/terms_and_conditions.html
+image: logo.jpg
+submission_guidelines: templates/submission_guidelines.html
+leaderboard_description: This challenge features multiple phases with different evaluation criteria and submission limits.
+evaluation_script: evaluation_script.zip
+remote_evaluation: False
+start_date: 2024-01-01 00:00:00
+end_date: 2025-12-31 23:59:59
+published: True
+tags:
+ - multi-phase-challenge
+ - machine-learning
+ - data-science
+ - computer-vision
+leaderboard:
+ - id: 1
+ schema:
+ {
+ "labels": ["Accuracy", "Precision", "Recall", "F1-Score", "AUC"],
+ "default_order_by": "F1-Score",
+ "metadata": {
+ "Accuracy": {
+ "sort_ascending": False,
+ "description": "Overall accuracy of the model predictions.",
+ },
+ "Precision": {
+ "sort_ascending": False,
+ "description": "Precision metric measuring positive prediction accuracy.",
+ },
+ "Recall": {
+ "sort_ascending": False,
+ "description": "Recall metric measuring sensitivity of the model.",
+ },
+ "F1-Score": {
+ "sort_ascending": False,
+ "description": "Harmonic mean of precision and recall.",
+ },
+ "AUC": {
+ "sort_ascending": False,
+ "description": "Area Under the ROC Curve metric.",
+ }
+ }
+ }
+
+challenge_phases:
+ - id: 1
+ name: Development Phase
+ description: templates/challenge_phase_1_description.html
+ leaderboard_public: True
+ is_public: True
+ challenge: 1
+ is_active: True
+ max_concurrent_submissions_allowed: 5
+ allowed_email_ids: []
+ disable_logs: False
+ is_submission_public: True
+ start_date: 2024-01-01 00:00:00
+ end_date: 2024-08-31 23:59:59
+ test_annotation_file: annotations/test_annotations_devsplit.json
+ codename: development
+ max_submissions_per_day: 20
+ max_submissions_per_month: 200
+ max_submissions: 500
+ default_submission_meta_attributes:
+ - name: method_name
+ is_visible: True
+ - name: method_description
+ is_visible: True
+ - name: project_url
+ is_visible: True
+ - name: publication_url
+ is_visible: True
+ submission_meta_attributes:
+ - name: Model Architecture
+ description: Brief description of your model architecture
+ type: text
+ required: True
+ - name: Pre-trained Model
+ description: Are you using a pre-trained model?
+ type: radio
+ options: ["Yes", "No"]
+ required: True
+ - name: Training Data
+ description: What training data did you use?
+ type: checkbox
+ options: ["ImageNet", "COCO", "Custom Dataset", "Synthetic Data", "Other"]
+ required: False
+ - name: Data Augmentation
+ description: Did you use data augmentation?
+ type: boolean
+ required: False
+ is_restricted_to_select_one_submission: False
+ is_partial_submission_evaluation_enabled: True
+ allowed_submission_file_types: ".json, .zip, .txt, .tsv, .gz, .csv, .h5, .npy, .npz, .pkl"
+
+ - id: 2
+ name: Validation Phase
+ description: templates/challenge_phase_2_description.html
+ leaderboard_public: True
+ is_public: True
+ challenge: 2
+ is_active: True
+ max_concurrent_submissions_allowed: 3
+ allowed_email_ids: []
+ disable_logs: False
+ is_submission_public: False
+ start_date: 2024-09-01 00:00:00
+ end_date: 2024-11-30 23:59:59
+ test_annotation_file: annotations/test_annotations_testsplit.json
+ codename: validation
+ max_submissions_per_day: 10
+ max_submissions_per_month: 100
+ max_submissions: 200
+ default_submission_meta_attributes:
+ - name: method_name
+ is_visible: True
+ - name: method_description
+ is_visible: True
+ - name: project_url
+ is_visible: True
+ submission_meta_attributes:
+ - name: Model Version
+ description: Version identifier for your model
+ type: text
+ required: True
+ - name: Computational Resources
+ description: What computational resources were used?
+ type: checkbox
+ options: ["CPU", "Single GPU", "Multiple GPUs", "TPU", "Cloud Computing", "Other"]
+ required: True
+ - name: Training Time
+ description: Approximate training time
+ type: radio
+ options: ["< 1 hour", "1-6 hours", "6-24 hours", "1-7 days", "> 1 week"]
+ required: False
+ - name: Ensemble Method
+ description: Did you use ensemble methods?
+ type: boolean
+ required: False
+ is_restricted_to_select_one_submission: False
+ is_partial_submission_evaluation_enabled: False
+ allowed_submission_file_types: ".json, .zip, .txt, .tsv, .gz, .csv, .h5, .npy, .npz"
+
+dataset_splits:
+ - id: 1
+ name: Development Split
+ codename: dev_split
+ - id: 2
+ name: Test Split
+ codename: test_split
+
+challenge_phase_splits:
+ # Development Phase - uses dev split with public leaderboard
+ - challenge_phase_id: 1
+ leaderboard_id: 1
+ dataset_split_id: 1
+ visibility: 1
+ leaderboard_decimal_precision: 4
+ is_leaderboard_order_descending: True
+ show_execution_time: True
+ show_leaderboard_by_latest_submission: True
+
+ # Validation Phase - uses test split with public leaderboard
+ - challenge_phase_id: 2
+ leaderboard_id: 1
+ dataset_split_id: 2
+ visibility: 1
+ leaderboard_decimal_precision: 4
+ is_leaderboard_order_descending: True
+ show_execution_time: True
+ show_leaderboard_by_latest_submission: True
diff --git a/github/challenge_processing_script.py b/github/challenge_processing_script.py
index 32c9aae10..72370b075 100644
--- a/github/challenge_processing_script.py
+++ b/github/challenge_processing_script.py
@@ -3,10 +3,23 @@
import os
import requests
import sys
+import argparse
+import re
+import config
import urllib3
from urllib.parse import urlparse
-from config import *
+from config import (
+ HOST_CONFIG_FILE_PATH,
+ CHALLENGE_CONFIG_VALIDATION_URL,
+ CHALLENGE_CREATE_OR_UPDATE_URL,
+ EVALAI_ERROR_CODES,
+ API_HOST_URL,
+ IGNORE_DIRS,
+ IGNORE_FILES,
+ CHALLENGE_ZIP_FILE_PATH,
+ GITHUB_EVENT_NAME,
+)
from utils import (
add_pull_request_comment,
check_for_errors,
@@ -23,6 +36,19 @@
GITHUB_CONTEXT = json.loads(os.getenv("GITHUB_CONTEXT", "{}"))
GITHUB_AUTH_TOKEN = os.getenv("GITHUB_AUTH_TOKEN")
+
+# START of the FIX: Explicitly read GITHUB_REPOSITORY from environment variable
+GITHUB_REPOSITORY = os.getenv("GITHUB_REPOSITORY")
+if not GITHUB_REPOSITORY:
+ print("FATAL: GITHUB_REPOSITORY environment variable is not set.")
+ print("Please ensure your GitHub Actions workflow sets this variable.")
+ sys.exit(1)
+print(f"š GITHUB_REPOSITORY from environment: {GITHUB_REPOSITORY}")
+
+VALIDATION_STEP = os.getenv("IS_VALIDATION")
+print(f"š VALIDATION_STEP from IS_VALIDATION: {VALIDATION_STEP}")
+
+
if not GITHUB_AUTH_TOKEN:
print(
"Please add your github access token to the repository secrets with the name AUTH_TOKEN"
@@ -35,6 +61,20 @@
CHALLENGE_HOST_TEAM_PK = None
EVALAI_HOST_URL = None
+parser = argparse.ArgumentParser(
+ description="Validate or create/update challenge on EvalAI"
+)
+parser.add_argument("branch_name", nargs="?", default=None, help="Name of the git branch whose configuration is being processed")
+
+args = parser.parse_args()
+
+# Determine effective branch name (default to "challenge" if none provided)
+branch_name = args.branch_name if args.branch_name else "challenge"
+
+# Enforce branch naming convention: "challenge" or "challenge-YYYY-version"
+if not re.match(r"^challenge(-\d{4}-.*)?$", branch_name):
+ print("Error: Branch name must be 'challenge' or 'challenge-YYYY-version' (e.g., 'challenge', 'challenge-2024-v1', 'challenge-2025-final').")
+ sys.exit(1)
def is_localhost_url(url):
"""
@@ -72,6 +112,37 @@ def configure_requests_for_localhost():
print("INFO: SSL verification disabled for localhost development server")
+def modify_challenge_title_for_versioning(branch_suffix):
+ """
+ Keep the original challenge title in challenge_config.yaml
+ Different branch versions will create separate challenges through repository name modification
+
+ Arguments:
+ branch_suffix {str}: The branch suffix (e.g., "2025-v1") - not used for title modification
+ """
+ import yaml
+
+ config_file = "challenge_config.yaml"
+
+ try:
+ # Read the current config
+ with open(config_file, 'r') as f:
+ config = yaml.safe_load(f)
+
+ # Get the original title
+ original_title = config.get('title', 'Challenge')
+
+ print(f" š Keeping original title: {original_title}")
+ print(f" ā
Challenge versioning handled through repository name modification")
+ return None # No title modification needed
+
+ except Exception as e:
+ print(f" ā ļø Warning: Could not read challenge title: {e}")
+ print(f" ā¹ļø Continuing with original title...")
+ return None
+
+
+
if __name__ == "__main__":
configs = load_host_configs(HOST_CONFIG_FILE_PATH)
@@ -82,6 +153,11 @@ def configure_requests_for_localhost():
else:
sys.exit(1)
+
+ # Update the global config path for zip file creation
+ # Note: We're not importing config.* anymore, so we need to set this directly
+ CHALLENGE_CONFIG_FILE_PATH = "challenge_config.yaml"
+
# Check if we're using a localhost server and configure accordingly
is_localhost = is_localhost_url(EVALAI_HOST_URL)
runner_info = get_runner_info()
@@ -112,13 +188,53 @@ def configure_requests_for_localhost():
headers = get_request_header(HOST_AUTH_TOKEN)
+ # Add the branch name (if provided) so that EvalAI can distinguish between multiple
+ # versions of the challenge present in the same repository.
+
+ # For branches with year-version format (e.g., challenge-2025-v1, challenge-2025-v2),
+ # create separate challenges by modifying the repository identifier
+ effective_repo_name = GITHUB_REPOSITORY
+
+ if branch_name and branch_name != "challenge":
+ # Extract year-version suffix from branch name and append to repo name
+ # challenge-2025-v1 -> 2025-v1
+ branch_suffix = branch_name.replace("challenge-", "")
+ effective_repo_name = f"{GITHUB_REPOSITORY}-{branch_suffix}"
+ print(f"š Creating separate challenge for branch: {branch_name}")
+ print(f"š Effective repository name: {effective_repo_name}")
+
+ # Note: Challenge versioning is handled through repository name modification
+ # The title remains unchanged to keep it clean
+ print(f"š Challenge versioning handled through repository name")
+ modify_challenge_title_for_versioning(branch_suffix)
+
# Creating the challenge zip file and storing in a dict to send to EvalAI
+ # IMPORTANT: This must happen AFTER title modification
print(f"\nš¦ Creating challenge configuration package...")
create_challenge_zip_file(CHALLENGE_ZIP_FILE_PATH, IGNORE_DIRS, IGNORE_FILES)
zip_file = open(CHALLENGE_ZIP_FILE_PATH, "rb")
file = {"zip_configuration": zip_file}
+
+ data = {"GITHUB_REPOSITORY": effective_repo_name}
+ if branch_name:
+ data["BRANCH_NAME"] = branch_name
- data = {"GITHUB_REPOSITORY": GITHUB_REPOSITORY}
+ # Debug output
+ print(f"š Challenge identification:")
+ print(f" Original repo: {GITHUB_REPOSITORY}")
+ print(f" Effective repo: {effective_repo_name}")
+ print(f" Branch name: {branch_name}")
+ print(f" Data being sent: {data}")
+
+ # Verify challenge title in the config file
+ try:
+ import yaml
+ with open("challenge_config.yaml", 'r') as f:
+ config = yaml.safe_load(f)
+ current_title = config.get('title', 'Unknown')
+ print(f" Current challenge title: {current_title}")
+ except Exception as e:
+ print(f" ā ļø Could not read current title: {e}")
# Configure SSL verification based on whether we're using localhost
verify_ssl = not is_localhost
@@ -126,12 +242,35 @@ def configure_requests_for_localhost():
try:
print(f"\nš Sending request to EvalAI server...")
+ print(f"š¤ Request details:")
+ print(f" URL: {url}")
+ print(f" Data: {data}")
+ print(f" Headers: {headers}")
+
response = requests.post(url, data=data, headers=headers, files=file, verify=verify_ssl)
+ print(f"š„ Response received:")
+ print(f" Status code: {response.status_code}")
+ print(f" Status: {response.status_code == http.HTTPStatus.CREATED and 'CREATED' or response.status_code == http.HTTPStatus.OK and 'UPDATED' or 'OTHER'}")
+
if response.status_code != http.HTTPStatus.OK and response.status_code != http.HTTPStatus.CREATED:
+ print(f" Response content: {response.text}")
response.raise_for_status()
else:
- print("\nā
Challenge processed successfully on EvalAI")
+ if response.status_code == http.HTTPStatus.CREATED:
+ print("\nā
NEW Challenge CREATED successfully on EvalAI")
+ elif response.status_code == http.HTTPStatus.OK:
+ print("\nš Existing Challenge UPDATED successfully on EvalAI")
+
+ # Try to parse response for additional info
+ try:
+ response_data = response.json()
+ if 'title' in response_data:
+ print(f" Challenge title: {response_data['title']}")
+ if 'id' in response_data:
+ print(f" Challenge ID: {response_data['id']}")
+ except:
+ print(" (Could not parse response JSON)")
except requests.exceptions.ConnectionError as conn_err:
# Handle connection errors specifically for localhost
@@ -237,7 +376,7 @@ def configure_requests_for_localhost():
else:
add_pull_request_comment(
GITHUB_AUTH_TOKEN,
- os.path.basename(GITHUB_REPOSITORY),
+ os.path.basename(effective_repo_name),
pr_number,
errors,
)
@@ -245,7 +384,7 @@ def configure_requests_for_localhost():
issue_title = (
"Following errors occurred while validating the challenge config:"
)
- repo_name = os.path.basename(GITHUB_REPOSITORY) if GITHUB_REPOSITORY else ""
+ repo_name = os.path.basename(effective_repo_name) if effective_repo_name else ""
create_github_repository_issue(
GITHUB_AUTH_TOKEN,
repo_name,
diff --git a/github/requirements.txt b/github/requirements.txt
index ed667b853..3640884b1 100644
--- a/github/requirements.txt
+++ b/github/requirements.txt
@@ -1,2 +1,3 @@
PyGithub===1.53
requests==2.32.4
+PyYAML==6.0.1