diff --git a/.github/workflows/validate-and-process.yml b/.github/workflows/validate-and-process.yml index 9465caa35..387e17477 100644 --- a/.github/workflows/validate-and-process.yml +++ b/.github/workflows/validate-and-process.yml @@ -3,13 +3,55 @@ name: Validate and Process EvalAI Challenge on: push: branches: - - challenge + - 'challenge' + - 'challenge-*-*' + pull_request: + types: [opened, synchronize, reopened, edited] permissions: contents: read issues: write jobs: + detect-and-validate-challenge-branch: + runs-on: ubuntu-latest + outputs: + is_challenge_branch: ${{ steps.branch-check.outputs.is_challenge_branch }} + challenge_branch: ${{ steps.branch-check.outputs.challenge_branch }} + steps: + - name: Detect and validate challenge branch + id: branch-check + run: | + # Get the actual branch name + if [ "${{ github.event_name }}" == "pull_request" ]; then + BRANCH="${{ github.head_ref }}" + else + BRANCH="${{ github.ref_name }}" + fi + + echo "šŸ” Analyzing branch: $BRANCH" + echo "challenge_branch=$BRANCH" >> $GITHUB_OUTPUT + + # Check if this is a challenge branch (challenge or challenge-YYYY-version) + if [[ "$BRANCH" =~ ^challenge(-[0-9]{4}-.*)?$ ]]; then + echo "is_challenge_branch=true" >> $GITHUB_OUTPUT + echo "āœ… Valid challenge branch detected: $BRANCH" + + # Extract branch suffix for versioning + if [[ "$BRANCH" =~ ^challenge-([0-9]{4}-.+)$ ]]; then + SUFFIX="${BASH_REMATCH[1]}" + echo "branch_suffix=$SUFFIX" >> $GITHUB_OUTPUT + echo "šŸ“‹ Branch version: $SUFFIX" + else + echo "branch_suffix=main" >> $GITHUB_OUTPUT + echo "šŸ“‹ Main challenge branch (no suffix)" + fi + else + echo "is_challenge_branch=false" >> $GITHUB_OUTPUT + echo "ā„¹ļø Not a challenge branch: $BRANCH" + echo "ā­ļø Challenge processing will be skipped (valid patterns: 'challenge' or 'challenge-YYYY-version')" + fi + validate-host-config: runs-on: ubuntu-latest outputs: @@ -120,9 +162,10 @@ jobs: process-evalai-challenge: - needs: [validate-host-config, check-self-hosted-requirements] + needs: [detect-and-validate-challenge-branch, validate-host-config, check-self-hosted-requirements] if: | always() && + needs.detect-and-validate-challenge-branch.outputs.is_challenge_branch == 'true' && needs.validate-host-config.outputs.is_valid == 'true' && (needs.check-self-hosted-requirements.result == 'success' || needs.check-self-hosted-requirements.result == 'skipped') runs-on: ${{ needs.validate-host-config.outputs.requires_self_hosted == 'true' && 'self-hosted' || 'ubuntu-latest' }} @@ -131,7 +174,7 @@ jobs: - name: Checkout challenge branch uses: actions/checkout@v3 with: - ref: challenge + ref: ${{ needs.detect-and-validate-challenge-branch.outputs.challenge_branch }} - name: Set up Python (GitHub-hosted only) if: needs.validate-host-config.outputs.requires_self_hosted != 'true' @@ -183,10 +226,11 @@ jobs: run: | echo "šŸ” VALIDATING CHALLENGE CONFIGURATION" echo "=====================================" - python3 github/challenge_processing_script.py + python3 github/challenge_processing_script.py ${{ needs.detect-and-validate-challenge-branch.outputs.challenge_branch }} env: IS_VALIDATION: 'True' GITHUB_CONTEXT: ${{ toJson(github) }} + GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_AUTH_TOKEN: ${{ secrets.AUTH_TOKEN }} - name: Validate challenge configuration (Self-hosted with Docker) @@ -202,25 +246,27 @@ jobs: -e GITHUB_CONTEXT='${{ toJson(github) }}' \ -e GITHUB_REPOSITORY='${{ github.repository }}' \ -e GITHUB_AUTH_TOKEN='${{ secrets.AUTH_TOKEN }}' \ + -e GIT_BRANCH='${{ needs.detect-and-validate-challenge-branch.outputs.challenge_branch }}' \ python:3.9-slim \ bash -c " pip install -r github/requirements.txt && - python3 github/challenge_processing_script.py + python3 github/challenge_processing_script.py \$GIT_BRANCH " - name: Create or update challenge (GitHub-hosted) - if: success() && needs.validate-host-config.outputs.requires_self_hosted != 'true' + if: github.event_name == 'push' && success() && needs.validate-host-config.outputs.requires_self_hosted != 'true' run: | echo "šŸš€ CREATING/UPDATING CHALLENGE" echo "==============================" - python3 github/challenge_processing_script.py + python3 github/challenge_processing_script.py ${{ needs.detect-and-validate-challenge-branch.outputs.challenge_branch }} env: IS_VALIDATION: 'False' GITHUB_CONTEXT: ${{ toJson(github) }} + GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_AUTH_TOKEN: ${{ secrets.AUTH_TOKEN }} - name: Create or update challenge (Self-hosted with Docker) - if: success() && needs.validate-host-config.outputs.requires_self_hosted == 'true' + if: github.event_name == 'push' && success() && needs.validate-host-config.outputs.requires_self_hosted == 'true' run: | echo "šŸš€ CREATING/UPDATING CHALLENGE (Docker)" echo "========================================" @@ -232,8 +278,9 @@ jobs: -e GITHUB_CONTEXT='${{ toJson(github) }}' \ -e GITHUB_REPOSITORY='${{ github.repository }}' \ -e GITHUB_AUTH_TOKEN='${{ secrets.AUTH_TOKEN }}' \ + -e GIT_BRANCH='${{ needs.detect-and-validate-challenge-branch.outputs.challenge_branch }}' \ python:3.9-slim \ bash -c " pip install -r github/requirements.txt && - python3 github/challenge_processing_script.py + python3 github/challenge_processing_script.py \$GIT_BRANCH " diff --git a/README.md b/README.md index a078f9c78..15dc13226 100644 --- a/README.md +++ b/README.md @@ -48,18 +48,22 @@ If you are looking for a simple challenge configuration that you can replicate t 5. Create a branch with name `challenge` in the forked repository from the `master` branch. Note: Only changes in `challenge` branch will be synchronized with challenge on EvalAI. +You may maintain multiple versions by using additional branches whose names start with `challenge-` (e.g. `challenge-2024`, `challenge-v2`). +Note: The CI pipeline only runs for branches that match the pattern `challenge` or `challenge-*`. Any other branch name will be ignored. + +If you trigger the processing script manually and do **not** pass a branch argument, it will assume the branch name is `challenge` by default. 6. Add `evalai_user_auth_token` and `host_team_pk` in `github/host_config.json`. 7. Read [EvalAI challenge creation documentation](https://evalai.readthedocs.io/en/latest/configuration.html) to know more about how you want to structure your challenge. Once you are ready, start making changes in the yaml file, HTML templates, evaluation script according to your need. -8. Commit the changes and push the `challenge` branch in the repository and wait for the build to complete. View the [logs of your build](https://docs.github.com/en/free-pro-team@latest/actions/managing-workflow-runs/using-workflow-run-logs#viewing-logs-to-diagnose-failures). +8. Commit the changes and push the relevant `challenge*` branch to the repository and wait for the build to complete. View the [logs of your build](https://docs.github.com/en/free-pro-team@latest/actions/managing-workflow-runs/using-workflow-run-logs#viewing-logs-to-diagnose-failures). -9. If challenge config contains errors then a `issue` will be opened automatically in the repository with the errors otherwise the challenge will be created on EvalAI. +9. If the challenge config contains errors, a GitHub **Issue** will be opened automatically in the repository with the error details; otherwise the challenge will be created on EvalAI. 10. Go to [Hosted Challenges](https://eval.ai/web/hosted-challenges) to view your challenge. The challenge will be publicly available once EvalAI admin approves the challenge. -11. To update the challenge on EvalAI, make changes in the repository and push on `challenge` branch and wait for the build to complete. +11. To update the challenge on EvalAI, make changes in the repository and push to the corresponding `challenge*` branch and wait for the build to complete. ## Add custom dependencies for evaluation (Optional) To add custom dependency packages in the evaluation script, refer to [this guide](./evaluation_script/dependency-installation.md). diff --git a/challenge_config.yaml b/challenge_config.yaml index 6afc14ce4..423596821 100755 --- a/challenge_config.yaml +++ b/challenge_config.yaml @@ -1,164 +1,177 @@ -# If you are not sure what all these fields mean, please refer our documentation here: -# https://evalai.readthedocs.io/en/latest/configuration.html -title: Random Number Generator Challenge -short_description: Random number generation challenge for each submission -description: templates/description.html -evaluation_details: templates/evaluation_details.html -terms_and_conditions: templates/terms_and_conditions.html -image: logo.jpg -submission_guidelines: templates/submission_guidelines.html -leaderboard_description: Lorem ipsum dolor sit amet, consectetur adipiscing elit. Cras egestas a libero nec sagittis. -evaluation_script: evaluation_script.zip -remote_evaluation: False -start_date: 2019-01-01 00:00:00 -end_date: 2099-05-31 23:59:59 -published: True -tags: - - random-number-generation - - machine-learning - - data-science - - computer-vision -leaderboard: - - id: 1 - schema: - { - "labels": ["Metric1", "Metric2", "Metric3", "Total"], - "default_order_by": "Total", - "metadata": { - "Metric1": { - "sort_ascending": True, - "description": "Lorem ipsum dolor sit amet, consectetur adipiscing elit.", - }, - "Metric2": { - "sort_ascending": True, - "description": "Lorem ipsum dolor sit amet, consectetur adipiscing elit.", - } - } - } - -challenge_phases: - - id: 1 - name: Dev Phase - description: templates/challenge_phase_1_description.html - leaderboard_public: False - is_public: True - challenge: 1 - is_active: True - max_concurrent_submissions_allowed: 3 - allowed_email_ids: [] - disable_logs: False - is_submission_public: True - start_date: 2019-01-19 00:00:00 - end_date: 2099-04-25 23:59:59 - test_annotation_file: annotations/test_annotations_devsplit.json - codename: dev - max_submissions_per_day: 5 - max_submissions_per_month: 50 - max_submissions: 50 - default_submission_meta_attributes: - - name: method_name - is_visible: True - - name: method_description - is_visible: True - - name: project_url - is_visible: True - - name: publication_url - is_visible: True - submission_meta_attributes: - - name: TextAttribute - description: Sample - type: text - required: False - - name: SingleOptionAttribute - description: Sample - type: radio - options: ["A", "B", "C"] - - name: MultipleChoiceAttribute - description: Sample - type: checkbox - options: ["alpha", "beta", "gamma"] - - name: TrueFalseField - description: Sample - type: boolean - required: True - is_restricted_to_select_one_submission: False - is_partial_submission_evaluation_enabled: False - allowed_submission_file_types: ".json, .zip, .txt, .tsv, .gz, .csv, .h5, .npy, .npz" - - id: 2 - name: Test Phase - description: templates/challenge_phase_2_description.html - leaderboard_public: True - is_public: True - challenge: 2 - is_active: True - max_concurrent_submissions_allowed: 3 - allowed_email_ids: [] - disable_logs: False - is_submission_public: True - start_date: 2019-01-01 00:00:00 - end_date: 2099-05-24 23:59:59 - test_annotation_file: annotations/test_annotations_testsplit.json - codename: test - max_submissions_per_day: 5 - max_submissions_per_month: 50 - max_submissions: 50 - default_submission_meta_attributes: - - name: method_name - is_visible: True - - name: method_description - is_visible: True - - name: project_url - is_visible: True - - name: publication_url - is_visible: True - submission_meta_attributes: - - name: TextAttribute - description: Sample - type: text - - name: SingleOptionAttribute - description: Sample - type: radio - options: ["A", "B", "C"] - - name: MultipleChoiceAttribute - description: Sample - type: checkbox - options: ["alpha", "beta", "gamma"] - - name: TrueFalseField - description: Sample - type: boolean - is_restricted_to_select_one_submission: False - is_partial_submission_evaluation_enabled: False - -dataset_splits: - - id: 1 - name: Train Split - codename: train_split - - id: 2 - name: Test Split - codename: test_split - -challenge_phase_splits: - - challenge_phase_id: 1 - leaderboard_id: 1 - dataset_split_id: 1 - visibility: 1 - leaderboard_decimal_precision: 2 - is_leaderboard_order_descending: True - show_execution_time: True - show_leaderboard_by_latest_submission: True - - challenge_phase_id: 2 - leaderboard_id: 1 - dataset_split_id: 1 - visibility: 3 - leaderboard_decimal_precision: 2 - is_leaderboard_order_descending: True - showeceution_time: False - show_leaderboard_by_latest_submission: False - - challenge_phase_id: 2 - leaderboard_id: 1 - dataset_split_id: 2 - visibility: 1 - leaderboard_decimal_precision: 2 - is_leaderboard_order_descending: True - show_execution_time: True - show_leaderboard_by_latest_submission: True +# If you are not sure what all these fields mean, please refer our documentation here: +# https://evalai.readthedocs.io/en/latest/configuration.html +title: Advanced Multi-Phase AI Challenge +short_description: Multi-phase challenge with development and validation stages +description: templates/description.html +evaluation_details: templates/evaluation_details.html +terms_and_conditions: templates/terms_and_conditions.html +image: logo.jpg +submission_guidelines: templates/submission_guidelines.html +leaderboard_description: This challenge features multiple phases with different evaluation criteria and submission limits. +evaluation_script: evaluation_script.zip +remote_evaluation: False +start_date: 2024-01-01 00:00:00 +end_date: 2025-12-31 23:59:59 +published: True +tags: + - multi-phase-challenge + - machine-learning + - data-science + - computer-vision +leaderboard: + - id: 1 + schema: + { + "labels": ["Accuracy", "Precision", "Recall", "F1-Score", "AUC"], + "default_order_by": "F1-Score", + "metadata": { + "Accuracy": { + "sort_ascending": False, + "description": "Overall accuracy of the model predictions.", + }, + "Precision": { + "sort_ascending": False, + "description": "Precision metric measuring positive prediction accuracy.", + }, + "Recall": { + "sort_ascending": False, + "description": "Recall metric measuring sensitivity of the model.", + }, + "F1-Score": { + "sort_ascending": False, + "description": "Harmonic mean of precision and recall.", + }, + "AUC": { + "sort_ascending": False, + "description": "Area Under the ROC Curve metric.", + } + } + } + +challenge_phases: + - id: 1 + name: Development Phase + description: templates/challenge_phase_1_description.html + leaderboard_public: True + is_public: True + challenge: 1 + is_active: True + max_concurrent_submissions_allowed: 5 + allowed_email_ids: [] + disable_logs: False + is_submission_public: True + start_date: 2024-01-01 00:00:00 + end_date: 2024-08-31 23:59:59 + test_annotation_file: annotations/test_annotations_devsplit.json + codename: development + max_submissions_per_day: 20 + max_submissions_per_month: 200 + max_submissions: 500 + default_submission_meta_attributes: + - name: method_name + is_visible: True + - name: method_description + is_visible: True + - name: project_url + is_visible: True + - name: publication_url + is_visible: True + submission_meta_attributes: + - name: Model Architecture + description: Brief description of your model architecture + type: text + required: True + - name: Pre-trained Model + description: Are you using a pre-trained model? + type: radio + options: ["Yes", "No"] + required: True + - name: Training Data + description: What training data did you use? + type: checkbox + options: ["ImageNet", "COCO", "Custom Dataset", "Synthetic Data", "Other"] + required: False + - name: Data Augmentation + description: Did you use data augmentation? + type: boolean + required: False + is_restricted_to_select_one_submission: False + is_partial_submission_evaluation_enabled: True + allowed_submission_file_types: ".json, .zip, .txt, .tsv, .gz, .csv, .h5, .npy, .npz, .pkl" + + - id: 2 + name: Validation Phase + description: templates/challenge_phase_2_description.html + leaderboard_public: True + is_public: True + challenge: 2 + is_active: True + max_concurrent_submissions_allowed: 3 + allowed_email_ids: [] + disable_logs: False + is_submission_public: False + start_date: 2024-09-01 00:00:00 + end_date: 2024-11-30 23:59:59 + test_annotation_file: annotations/test_annotations_testsplit.json + codename: validation + max_submissions_per_day: 10 + max_submissions_per_month: 100 + max_submissions: 200 + default_submission_meta_attributes: + - name: method_name + is_visible: True + - name: method_description + is_visible: True + - name: project_url + is_visible: True + submission_meta_attributes: + - name: Model Version + description: Version identifier for your model + type: text + required: True + - name: Computational Resources + description: What computational resources were used? + type: checkbox + options: ["CPU", "Single GPU", "Multiple GPUs", "TPU", "Cloud Computing", "Other"] + required: True + - name: Training Time + description: Approximate training time + type: radio + options: ["< 1 hour", "1-6 hours", "6-24 hours", "1-7 days", "> 1 week"] + required: False + - name: Ensemble Method + description: Did you use ensemble methods? + type: boolean + required: False + is_restricted_to_select_one_submission: False + is_partial_submission_evaluation_enabled: False + allowed_submission_file_types: ".json, .zip, .txt, .tsv, .gz, .csv, .h5, .npy, .npz" + +dataset_splits: + - id: 1 + name: Development Split + codename: dev_split + - id: 2 + name: Test Split + codename: test_split + +challenge_phase_splits: + # Development Phase - uses dev split with public leaderboard + - challenge_phase_id: 1 + leaderboard_id: 1 + dataset_split_id: 1 + visibility: 1 + leaderboard_decimal_precision: 4 + is_leaderboard_order_descending: True + show_execution_time: True + show_leaderboard_by_latest_submission: True + + # Validation Phase - uses test split with public leaderboard + - challenge_phase_id: 2 + leaderboard_id: 1 + dataset_split_id: 2 + visibility: 1 + leaderboard_decimal_precision: 4 + is_leaderboard_order_descending: True + show_execution_time: True + show_leaderboard_by_latest_submission: True diff --git a/github/challenge_processing_script.py b/github/challenge_processing_script.py index 32c9aae10..72370b075 100644 --- a/github/challenge_processing_script.py +++ b/github/challenge_processing_script.py @@ -3,10 +3,23 @@ import os import requests import sys +import argparse +import re +import config import urllib3 from urllib.parse import urlparse -from config import * +from config import ( + HOST_CONFIG_FILE_PATH, + CHALLENGE_CONFIG_VALIDATION_URL, + CHALLENGE_CREATE_OR_UPDATE_URL, + EVALAI_ERROR_CODES, + API_HOST_URL, + IGNORE_DIRS, + IGNORE_FILES, + CHALLENGE_ZIP_FILE_PATH, + GITHUB_EVENT_NAME, +) from utils import ( add_pull_request_comment, check_for_errors, @@ -23,6 +36,19 @@ GITHUB_CONTEXT = json.loads(os.getenv("GITHUB_CONTEXT", "{}")) GITHUB_AUTH_TOKEN = os.getenv("GITHUB_AUTH_TOKEN") + +# START of the FIX: Explicitly read GITHUB_REPOSITORY from environment variable +GITHUB_REPOSITORY = os.getenv("GITHUB_REPOSITORY") +if not GITHUB_REPOSITORY: + print("FATAL: GITHUB_REPOSITORY environment variable is not set.") + print("Please ensure your GitHub Actions workflow sets this variable.") + sys.exit(1) +print(f"šŸ” GITHUB_REPOSITORY from environment: {GITHUB_REPOSITORY}") + +VALIDATION_STEP = os.getenv("IS_VALIDATION") +print(f"šŸ” VALIDATION_STEP from IS_VALIDATION: {VALIDATION_STEP}") + + if not GITHUB_AUTH_TOKEN: print( "Please add your github access token to the repository secrets with the name AUTH_TOKEN" @@ -35,6 +61,20 @@ CHALLENGE_HOST_TEAM_PK = None EVALAI_HOST_URL = None +parser = argparse.ArgumentParser( + description="Validate or create/update challenge on EvalAI" +) +parser.add_argument("branch_name", nargs="?", default=None, help="Name of the git branch whose configuration is being processed") + +args = parser.parse_args() + +# Determine effective branch name (default to "challenge" if none provided) +branch_name = args.branch_name if args.branch_name else "challenge" + +# Enforce branch naming convention: "challenge" or "challenge-YYYY-version" +if not re.match(r"^challenge(-\d{4}-.*)?$", branch_name): + print("Error: Branch name must be 'challenge' or 'challenge-YYYY-version' (e.g., 'challenge', 'challenge-2024-v1', 'challenge-2025-final').") + sys.exit(1) def is_localhost_url(url): """ @@ -72,6 +112,37 @@ def configure_requests_for_localhost(): print("INFO: SSL verification disabled for localhost development server") +def modify_challenge_title_for_versioning(branch_suffix): + """ + Keep the original challenge title in challenge_config.yaml + Different branch versions will create separate challenges through repository name modification + + Arguments: + branch_suffix {str}: The branch suffix (e.g., "2025-v1") - not used for title modification + """ + import yaml + + config_file = "challenge_config.yaml" + + try: + # Read the current config + with open(config_file, 'r') as f: + config = yaml.safe_load(f) + + # Get the original title + original_title = config.get('title', 'Challenge') + + print(f" šŸ“ Keeping original title: {original_title}") + print(f" āœ… Challenge versioning handled through repository name modification") + return None # No title modification needed + + except Exception as e: + print(f" āš ļø Warning: Could not read challenge title: {e}") + print(f" ā„¹ļø Continuing with original title...") + return None + + + if __name__ == "__main__": configs = load_host_configs(HOST_CONFIG_FILE_PATH) @@ -82,6 +153,11 @@ def configure_requests_for_localhost(): else: sys.exit(1) + + # Update the global config path for zip file creation + # Note: We're not importing config.* anymore, so we need to set this directly + CHALLENGE_CONFIG_FILE_PATH = "challenge_config.yaml" + # Check if we're using a localhost server and configure accordingly is_localhost = is_localhost_url(EVALAI_HOST_URL) runner_info = get_runner_info() @@ -112,13 +188,53 @@ def configure_requests_for_localhost(): headers = get_request_header(HOST_AUTH_TOKEN) + # Add the branch name (if provided) so that EvalAI can distinguish between multiple + # versions of the challenge present in the same repository. + + # For branches with year-version format (e.g., challenge-2025-v1, challenge-2025-v2), + # create separate challenges by modifying the repository identifier + effective_repo_name = GITHUB_REPOSITORY + + if branch_name and branch_name != "challenge": + # Extract year-version suffix from branch name and append to repo name + # challenge-2025-v1 -> 2025-v1 + branch_suffix = branch_name.replace("challenge-", "") + effective_repo_name = f"{GITHUB_REPOSITORY}-{branch_suffix}" + print(f"šŸ”„ Creating separate challenge for branch: {branch_name}") + print(f"šŸ“‹ Effective repository name: {effective_repo_name}") + + # Note: Challenge versioning is handled through repository name modification + # The title remains unchanged to keep it clean + print(f"šŸ“ Challenge versioning handled through repository name") + modify_challenge_title_for_versioning(branch_suffix) + # Creating the challenge zip file and storing in a dict to send to EvalAI + # IMPORTANT: This must happen AFTER title modification print(f"\nšŸ“¦ Creating challenge configuration package...") create_challenge_zip_file(CHALLENGE_ZIP_FILE_PATH, IGNORE_DIRS, IGNORE_FILES) zip_file = open(CHALLENGE_ZIP_FILE_PATH, "rb") file = {"zip_configuration": zip_file} + + data = {"GITHUB_REPOSITORY": effective_repo_name} + if branch_name: + data["BRANCH_NAME"] = branch_name - data = {"GITHUB_REPOSITORY": GITHUB_REPOSITORY} + # Debug output + print(f"šŸ” Challenge identification:") + print(f" Original repo: {GITHUB_REPOSITORY}") + print(f" Effective repo: {effective_repo_name}") + print(f" Branch name: {branch_name}") + print(f" Data being sent: {data}") + + # Verify challenge title in the config file + try: + import yaml + with open("challenge_config.yaml", 'r') as f: + config = yaml.safe_load(f) + current_title = config.get('title', 'Unknown') + print(f" Current challenge title: {current_title}") + except Exception as e: + print(f" āš ļø Could not read current title: {e}") # Configure SSL verification based on whether we're using localhost verify_ssl = not is_localhost @@ -126,12 +242,35 @@ def configure_requests_for_localhost(): try: print(f"\n🌐 Sending request to EvalAI server...") + print(f"šŸ“¤ Request details:") + print(f" URL: {url}") + print(f" Data: {data}") + print(f" Headers: {headers}") + response = requests.post(url, data=data, headers=headers, files=file, verify=verify_ssl) + print(f"šŸ“„ Response received:") + print(f" Status code: {response.status_code}") + print(f" Status: {response.status_code == http.HTTPStatus.CREATED and 'CREATED' or response.status_code == http.HTTPStatus.OK and 'UPDATED' or 'OTHER'}") + if response.status_code != http.HTTPStatus.OK and response.status_code != http.HTTPStatus.CREATED: + print(f" Response content: {response.text}") response.raise_for_status() else: - print("\nāœ… Challenge processed successfully on EvalAI") + if response.status_code == http.HTTPStatus.CREATED: + print("\nāœ… NEW Challenge CREATED successfully on EvalAI") + elif response.status_code == http.HTTPStatus.OK: + print("\nšŸ”„ Existing Challenge UPDATED successfully on EvalAI") + + # Try to parse response for additional info + try: + response_data = response.json() + if 'title' in response_data: + print(f" Challenge title: {response_data['title']}") + if 'id' in response_data: + print(f" Challenge ID: {response_data['id']}") + except: + print(" (Could not parse response JSON)") except requests.exceptions.ConnectionError as conn_err: # Handle connection errors specifically for localhost @@ -237,7 +376,7 @@ def configure_requests_for_localhost(): else: add_pull_request_comment( GITHUB_AUTH_TOKEN, - os.path.basename(GITHUB_REPOSITORY), + os.path.basename(effective_repo_name), pr_number, errors, ) @@ -245,7 +384,7 @@ def configure_requests_for_localhost(): issue_title = ( "Following errors occurred while validating the challenge config:" ) - repo_name = os.path.basename(GITHUB_REPOSITORY) if GITHUB_REPOSITORY else "" + repo_name = os.path.basename(effective_repo_name) if effective_repo_name else "" create_github_repository_issue( GITHUB_AUTH_TOKEN, repo_name, diff --git a/github/requirements.txt b/github/requirements.txt index ed667b853..3640884b1 100644 --- a/github/requirements.txt +++ b/github/requirements.txt @@ -1,2 +1,3 @@ PyGithub===1.53 requests==2.32.4 +PyYAML==6.0.1