diff --git a/.github/workflows/basic.yml b/.github/workflows/basic.yml index 0f8baf0fcc2..0968a48342d 100644 --- a/.github/workflows/basic.yml +++ b/.github/workflows/basic.yml @@ -134,26 +134,6 @@ jobs: env: REFERENCE_BRANCH: ${{ github['base_ref'] || github['head_ref'] }} - pylint-py27: - name: Pylint for Python 2.7 in Pilot files - runs-on: ubuntu-latest - if: github.event_name != 'push' || github.repository == 'DIRACGrid/DIRAC' - steps: - - uses: actions/checkout@v6 - - name: Fail-fast for outdated pipelines - run: .github/workflows/fail-fast.sh - - name: prepare environment - run: | - conda config --set add_pip_as_python_dependency false - conda create -c conda-forge -n test-env python=2.7 pylint=1.9.2 astroid=1.6.5 - - name: run pilot wrapper test - run: | - eval "$(conda shell.bash hook)" && conda activate test-env - pylint -j 0 -E \ - tests/Integration/WorkloadManagementSystem/Test_GenerateAndExecutePilotWrapper.py \ - src/DIRAC/WorkloadManagementSystem/Utilities/PilotWrapper.py \ - src/DIRAC/Resources/Computing/BatchSystems/*.py - diracx: name: DiracX Unit Tests runs-on: ubuntu-latest diff --git a/docs/source/AdministratorGuide/Systems/WorkloadManagement/Pilots/index.rst b/docs/source/AdministratorGuide/Systems/WorkloadManagement/Pilots/index.rst index 97f4bbf6e9f..1787d4f4923 100644 --- a/docs/source/AdministratorGuide/Systems/WorkloadManagement/Pilots/index.rst +++ b/docs/source/AdministratorGuide/Systems/WorkloadManagement/Pilots/index.rst @@ -40,7 +40,7 @@ where: - configure means adding dirac specific configuration files (which, at a minimum, should include the location of a DIRAC configuration service) -A pilot has to run on each and every computing resource type, provided that Python 2.6+ is on the WN. +A pilot has to run on each and every computing resource type, provided that Python 3.6+ is on the WN. The same pilot script can be used everywhere. .. image:: Pilots2.png diff --git a/setup.cfg b/setup.cfg index 7becf2289f7..59f94d62527 100644 --- a/setup.cfg +++ b/setup.cfg @@ -326,10 +326,9 @@ console_scripts = [pycodestyle] # Pep8 codes: # E203 - Whitespace before ':' (spaces should be present for list slices) -# E211 - Whitespace before '(' (black incorrectly formats exec when in Python 2 mode) # E266 - Too many leading '#' for block comment # E402 - module level import not at top of file (for scripts) # W503 - line break before binary operator (stupid, complains after or before...) -ignore = E203, E211, E266, E402, W503 +ignore = E203, E266, E402, W503 # TODO: This should be reduced back to 120 at some point max_line_length=130 diff --git a/src/DIRAC/Core/Utilities/Plotting/FileCoding.py b/src/DIRAC/Core/Utilities/Plotting/FileCoding.py index 23a28944c59..eef7deb9786 100644 --- a/src/DIRAC/Core/Utilities/Plotting/FileCoding.py +++ b/src/DIRAC/Core/Utilities/Plotting/FileCoding.py @@ -41,8 +41,7 @@ def extractRequestFromFileId(fileId): if compressType == "Z": gLogger.info("Compressed request, uncompressing") try: - # Encoding is only required for Python 2 and can be removed when Python 2 support is no longer needed - stub = base64.urlsafe_b64decode(stub.encode()) + stub = base64.urlsafe_b64decode(stub) except Exception as e: gLogger.error("Oops! Plot request is not properly encoded!", str(e)) return S_ERROR(f"Oops! Plot request is not properly encoded!: {str(e)}") diff --git a/src/DIRAC/Resources/Computing/BatchSystems/Condor.py b/src/DIRAC/Resources/Computing/BatchSystems/Condor.py index 5757a7f2cfc..7a95f617499 100644 --- a/src/DIRAC/Resources/Computing/BatchSystems/Condor.py +++ b/src/DIRAC/Resources/Computing/BatchSystems/Condor.py @@ -3,9 +3,6 @@ LocalComputingElement and SSHComputingElement classes """ -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division import json import re import tempfile diff --git a/src/DIRAC/Resources/Computing/BatchSystems/GE.py b/src/DIRAC/Resources/Computing/BatchSystems/GE.py index 4590ecffb75..e01b109c930 100644 --- a/src/DIRAC/Resources/Computing/BatchSystems/GE.py +++ b/src/DIRAC/Resources/Computing/BatchSystems/GE.py @@ -14,9 +14,6 @@ SubmitOption = -l ct=6000 """ -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division import re import shlex import subprocess diff --git a/src/DIRAC/Resources/Computing/BatchSystems/Host.py b/src/DIRAC/Resources/Computing/BatchSystems/Host.py index cdd0a49396b..3b827232013 100644 --- a/src/DIRAC/Resources/Computing/BatchSystems/Host.py +++ b/src/DIRAC/Resources/Computing/BatchSystems/Host.py @@ -8,10 +8,6 @@ with LocalComputingElement or SSHComputingElement objects """ -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division - import os import glob import shutil diff --git a/src/DIRAC/Resources/Computing/BatchSystems/LSF.py b/src/DIRAC/Resources/Computing/BatchSystems/LSF.py index 4d91e92ffe8..6850a0c7dbf 100644 --- a/src/DIRAC/Resources/Computing/BatchSystems/LSF.py +++ b/src/DIRAC/Resources/Computing/BatchSystems/LSF.py @@ -9,9 +9,6 @@ LocalComputingElement and SSHComputingElement classes """ -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division import re import subprocess import shlex diff --git a/src/DIRAC/Resources/Computing/BatchSystems/OAR.py b/src/DIRAC/Resources/Computing/BatchSystems/OAR.py index dfb0fb5d462..24e73bdf1b9 100644 --- a/src/DIRAC/Resources/Computing/BatchSystems/OAR.py +++ b/src/DIRAC/Resources/Computing/BatchSystems/OAR.py @@ -9,10 +9,6 @@ LocalComputingElement and SSHComputingElement classes """ -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division - import subprocess import shlex import os diff --git a/src/DIRAC/Resources/Computing/BatchSystems/SLURM.py b/src/DIRAC/Resources/Computing/BatchSystems/SLURM.py index 7016f2f63ae..16d6093d3e3 100644 --- a/src/DIRAC/Resources/Computing/BatchSystems/SLURM.py +++ b/src/DIRAC/Resources/Computing/BatchSystems/SLURM.py @@ -3,10 +3,6 @@ LocalComputingElement and SSHComputingElement classes """ -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division - import os import re import subprocess diff --git a/src/DIRAC/Resources/Computing/BatchSystems/Torque.py b/src/DIRAC/Resources/Computing/BatchSystems/Torque.py index 22093565682..6eb44670bde 100644 --- a/src/DIRAC/Resources/Computing/BatchSystems/Torque.py +++ b/src/DIRAC/Resources/Computing/BatchSystems/Torque.py @@ -9,10 +9,6 @@ LocalComputingElement and SSHComputingElement classes """ -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division - import subprocess import shlex import os diff --git a/src/DIRAC/Resources/Computing/SSHComputingElement.py b/src/DIRAC/Resources/Computing/SSHComputingElement.py index 25668e62b57..a4b74fa82c0 100644 --- a/src/DIRAC/Resources/Computing/SSHComputingElement.py +++ b/src/DIRAC/Resources/Computing/SSHComputingElement.py @@ -1,4 +1,4 @@ -""" SSH (Virtual) Computing Element +"""SSH (Virtual) Computing Element For a given IP/host it will send jobs directly through ssh @@ -62,6 +62,7 @@ **Code Documentation** """ + import errno import json import os diff --git a/src/DIRAC/WorkloadManagementSystem/Utilities/PilotWrapper.py b/src/DIRAC/WorkloadManagementSystem/Utilities/PilotWrapper.py index a60c7839c3b..0c03842e103 100644 --- a/src/DIRAC/WorkloadManagementSystem/Utilities/PilotWrapper.py +++ b/src/DIRAC/WorkloadManagementSystem/Utilities/PilotWrapper.py @@ -28,12 +28,10 @@ if [ "${current_limit}" = "unlimited" ] || [ "${current_limit}" -gt "${new_limit}" ]; then ulimit -n "${new_limit}" fi -if command -v python &> /dev/null; then - py='python' -elif command -v python3 &> /dev/null; then +if command -v python3 &> /dev/null; then py='python3' -elif command -v python2 &> /dev/null; then - py='python2' +elif command -v python &> /dev/null; then + py='python' fi /usr/bin/env $py << EOF @@ -63,14 +61,8 @@ import shlex from uuid import uuid1 -try: - # For Python 3.0 and later - from urllib.request import urlopen, HTTPError, URLError - from urllib.parse import urlencode -except ImportError: - # Fall back to Python 2's urllib2 - from urllib2 import urlopen, HTTPError, URLError - from urllib import urlencode +from urllib.request import urlopen, HTTPError, URLError +from urllib.parse import urlencode try: from cStringIO import StringIO @@ -93,10 +85,7 @@ def formatTime(self, record, datefmt=None): # formatter = logging.Formatter(fmt='%%(asctime)s UTC %%(levelname)-8s %%(message)s', datefmt='%%Y-%%m-%%d %%H:%%M:%%S') formatter = MicrosecondFormatter('%%(asctime)s %%(levelname)-8s [%%(name)s] %%(message)s') logging.Formatter.converter = time.gmtime -try: - screen_handler = logging.StreamHandler(stream=sys.stdout) -except TypeError: # python2.6 - screen_handler = logging.StreamHandler(strm=sys.stdout) +screen_handler = logging.StreamHandler(stream=sys.stdout) screen_handler.setFormatter(formatter) # add a string buffer handler @@ -276,28 +265,15 @@ def pilotWrapperScript( # Getting the json, tar, and checksum file try: - # urllib is different between python 2 and 3 - if sys.version_info < (3,): - from urllib2 import urlopen as url_library_urlopen - from urllib2 import URLError as url_library_URLError - else: - from urllib.request import urlopen as url_library_urlopen - from urllib.error import URLError as url_library_URLError + from urllib.request import urlopen + from urllib.error import URLError for fileName in ['checksums.sha512', 'pilot.json', 'pilot.tar']: - # needs to distinguish whether urlopen method contains the 'context' param - # in theory, it should be available from python 2.7.9 - # in practice, some prior versions may be composed of recent urllib version containing the param - if 'context' in url_library_urlopen.__code__.co_varnames: - import ssl - context = ssl._create_unverified_context() - remoteFile = url_library_urlopen(os.path.join(loc, fileName), - timeout=10, - context=context) - - else: - remoteFile = url_library_urlopen(os.path.join(loc, fileName), - timeout=10) + import ssl + context = ssl._create_unverified_context() + remoteFile = urlopen(os.path.join(loc, fileName), + timeout=10, + context=context) localFile = open(fileName, 'wb') localFile.write(remoteFile.read()) @@ -320,7 +296,7 @@ def pilotWrapperScript( raise # if we get here we break out of the loop of locations break - except (url_library_URLError, Exception) as e: + except (URLError, Exception) as e: print('%%s unreacheable (this is normal!)' %% loc, file=sys.stderr) logger.error('%%s unreacheable (this is normal!)' %% loc) logger.exception(e) diff --git a/tests/CI/run_pilot.sh b/tests/CI/run_pilot.sh index f7827b8503f..3a64802a5f0 100755 --- a/tests/CI/run_pilot.sh +++ b/tests/CI/run_pilot.sh @@ -31,15 +31,13 @@ cp /ca/certs/pilot_proxy /tmp/x509up_u$UID eval "${PILOT_DOWNLOAD_COMMAND}" -echo "${PILOT_JSON}" > pilot.json -jq < pilot.json +echo "${PILOT_JSON}" >pilot.json +jq /dev/null; then - py='python' -elif command -v python3 &> /dev/null; then +if command -v python3 &>/dev/null; then py='python3' -elif command -v python2 &> /dev/null; then - py='python2' +elif command -v python &>/dev/null; then + py='python' fi # shellcheck disable=SC2086 diff --git a/tests/Integration/WorkloadManagementSystem/Test_GenerateAndExecutePilotWrapper.py b/tests/Integration/WorkloadManagementSystem/Test_GenerateAndExecutePilotWrapper.py index 7c93b06d458..81dc37d1375 100644 --- a/tests/Integration/WorkloadManagementSystem/Test_GenerateAndExecutePilotWrapper.py +++ b/tests/Integration/WorkloadManagementSystem/Test_GenerateAndExecutePilotWrapper.py @@ -6,8 +6,6 @@ # - starts it # # It should be executed for different versions of python, e.g.: -# - 2.7.x (x < 9) -# - 2.7.x (x >= 9) # - 3.6.x # - 3.11.x # @@ -25,20 +23,12 @@ # 1) gets the (DIRAC-free) PilotWrapper.py -# urllib is different between python 2 and 3 -if sys.version_info < (3,): - from urllib2 import urlopen as url_library_urlopen # pylint: disable=import-error -else: - from urllib.request import urlopen as url_library_urlopen # pylint: disable=import-error,no-name-in-module +from urllib.request import urlopen # pylint: disable=import-error,no-name-in-module +import ssl # pylint: disable=import-error -if sys.version_info >= (2, 7, 9): - import ssl # pylint: disable=import-error - - context = ssl._create_unverified_context() - rf = url_library_urlopen(sys.argv[1], context=context) -else: - rf = url_library_urlopen(sys.argv[1]) +context = ssl._create_unverified_context() +rf = urlopen(sys.argv[1], context=context) locc = sys.argv[2] with open("PilotWrapper.py", "wb") as pj: