From a657d41e2645efdeb672eb9dbd6e5543ba3a8426 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Wed, 11 Nov 2020 14:07:18 -0700 Subject: [PATCH 1/8] Rename encrypt_variable() to encrypt_variable_travis() --- doctr/__init__.py | 4 ++-- doctr/__main__.py | 6 +++--- doctr/local.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/doctr/__init__.py b/doctr/__init__.py index 53b3a702..db553827 100644 --- a/doctr/__init__.py +++ b/doctr/__init__.py @@ -1,4 +1,4 @@ -from .local import (encrypt_variable, encrypt_to_file, GitHub_post, +from .local import (encrypt_variable_travis, encrypt_to_file, GitHub_post, generate_GitHub_token, upload_GitHub_deploy_key, generate_ssh_key, check_repo_exists, guess_github_repo) from .travis import (decrypt_file, setup_deploy_key, get_token, run, @@ -7,7 +7,7 @@ commit_docs, push_docs, get_current_repo, find_sphinx_build_dir) __all__ = [ - 'encrypt_variable', 'encrypt_to_file', 'GitHub_post', + 'encrypt_variable_travis', 'encrypt_to_file', 'GitHub_post', 'generate_GitHub_token', 'upload_GitHub_deploy_key', 'generate_ssh_key', 'check_repo_exists', 'guess_github_repo', diff --git a/doctr/__main__.py b/doctr/__main__.py index 8a35510e..195e2709 100644 --- a/doctr/__main__.py +++ b/doctr/__main__.py @@ -34,7 +34,7 @@ from textwrap import dedent -from .local import (generate_GitHub_token, encrypt_variable, encrypt_to_file, +from .local import (generate_GitHub_token, encrypt_variable_travis, encrypt_to_file, upload_GitHub_deploy_key, generate_ssh_key, check_repo_exists, GitHub_login, guess_github_repo, AuthenticationFailed, GitHubError, get_travis_token) @@ -472,7 +472,7 @@ def configure(args, parser): if args.token: if not GitHub_token: GitHub_token = generate_GitHub_token(**login_kwargs)['token'] - encrypted_variable = encrypt_variable("GH_TOKEN={GitHub_token}".format(GitHub_token=GitHub_token).encode('utf-8'), + encrypted_variable = encrypt_variable_travis("GH_TOKEN={GitHub_token}".format(GitHub_token=GitHub_token).encode('utf-8'), build_repo=build_repo, tld=tld, travis_token=travis_token, **login_kwargs) print(dedent(""" A personal access token for doctr has been created. @@ -487,7 +487,7 @@ def configure(args, parser): key = encrypt_to_file(private_ssh_key, keypath + '.enc') del private_ssh_key # Prevent accidental use below public_ssh_key = public_ssh_key.decode('ASCII') - encrypted_variable = encrypt_variable(env_name.encode('utf-8') + b"=" + key, + encrypted_variable = encrypt_variable_travis(env_name.encode('utf-8') + b"=" + key, build_repo=build_repo, tld=tld, travis_token=travis_token, **login_kwargs) deploy_keys_url = 'https://github.com/{deploy_repo}/settings/keys'.format(deploy_repo=deploy_key_repo) diff --git a/doctr/local.py b/doctr/local.py index 2dcee2e7..b5e6eb60 100644 --- a/doctr/local.py +++ b/doctr/local.py @@ -25,7 +25,7 @@ Travis_APIv2 = {'Accept': 'application/vnd.travis-ci.2.1+json'} Travis_APIv3 = {"Travis-API-Version": "3"} -def encrypt_variable(variable, build_repo, *, tld='.org', public_key=None, +def encrypt_variable_travis(variable, build_repo, *, tld='.org', public_key=None, travis_token=None, **login_kwargs): """ Encrypt an environment variable for ``build_repo`` for Travis From b33d5e088426fc66cd6c3202b3ca241391331824 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Wed, 11 Nov 2020 17:08:06 -0700 Subject: [PATCH 2/8] Add GitHub_get() to correspond to GitHub_post() --- doctr/local.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/doctr/local.py b/doctr/local.py index b5e6eb60..e81ca205 100644 --- a/doctr/local.py +++ b/doctr/local.py @@ -224,6 +224,17 @@ def GitHub_post(data, url, *, auth, headers): return r.json() +def GitHub_get(url, *, auth=None, headers): + """ + GET the URL on GitHub. + + Returns the json response from the server, or raises on error status. + + """ + r = requests.get(url, auth=auth, headers=headers) + GitHub_raise_for_status(r) + return r.json() + def get_travis_token(*, GitHub_token=None, **login_kwargs): """ Generate a temporary token for authenticating with Travis From ec7f9f341305367c6416a0812b2f25a689d6d473 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Wed, 11 Nov 2020 17:08:27 -0700 Subject: [PATCH 3/8] Add encrypt_variable_github_actions() This uses pynacl, which is what is used in the GitHub example docs. We should figure out if it would be better to use cryptography instead. --- doctr/local.py | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/doctr/local.py b/doctr/local.py index e81ca205..be949d0e 100644 --- a/doctr/local.py +++ b/doctr/local.py @@ -11,6 +11,7 @@ import urllib import datetime +from nacl import encoding, public import requests from requests.auth import HTTPBasicAuth @@ -83,6 +84,48 @@ def encrypt_variable_travis(variable, build_repo, *, tld='.org', public_key=None return base64.b64encode(key.encrypt(variable, pad)) +def encrypt_variable_github_actions(variable, build_repo, *, public_key=None, + **login_kwargs): + """ + Encrypt an environment variable for ``build_repo`` for Travis + + ``variable`` should be a bytes object, of the form ``b'ENV=value'``. + + ``build_repo`` is the repo that ``doctr deploy`` will be run from. It + should be like 'drdoctr/doctr'. + + ``public_key`` should be the GitHub actions public key, obtained from + GitHub if not provided. + + This is based on + https://docs.github.com/en/free-pro-team@latest/rest/reference/actions#create-or-update-an-organization-secret + + """ + if not isinstance(variable, bytes): + raise TypeError("variable should be bytes") + + if not b"=" in variable: + raise ValueError("variable should be of the form 'VARIABLE=value'") + + if not public_key: + # See + # https://docs.github.com/en/free-pro-team@latest/rest/reference/actions#get-an-organization-public-key + + headers = { + 'accept': 'application/vnd.github.v3+json', + 'org': build_repo, + } + if 'headers' in login_kwargs: + headers.update(login_kwargs.pop('headers')) + url = 'https://api.github.com/repos/{build_repo}/actions/secrets/public-key'.format(build_repo=urllib.parse.quote(build_repo, safe='/')) + res = GitHub_get(url, headers=headers, **login_kwargs) + public_key = res['key'] + + public_key = public.PublicKey(public_key.encode("utf-8"), encoding.Base64Encoder()) + sealed_box = public.SealedBox(public_key) + encrypted = sealed_box.encrypt(variable) + return base64.b64encode(encrypted).decode("utf-8") + def encrypt_to_file(contents, filename): """ Encrypts ``contents`` and writes it to ``filename``. From d7621b0a52bd645352382d37239a3779f66af873 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Wed, 11 Nov 2020 17:56:21 -0700 Subject: [PATCH 4/8] Split out Travis specific stuff from the CI code The CI code now runs in a class, and the Travis specific stuff is in a subclass. These are split into the files ci.py and travis.py. There are still some mentions of "Travis" in ci.py, but they shouldn't affect functionality. --- doctr/__init__.py | 13 +- doctr/__main__.py | 18 +- doctr/ci.py | 544 +++++++++++++++++++++++++++++++++++++ doctr/tests/test_travis.py | 98 +++---- doctr/travis.py | 516 ++--------------------------------- 5 files changed, 637 insertions(+), 552 deletions(-) create mode 100644 doctr/ci.py diff --git a/doctr/__init__.py b/doctr/__init__.py index db553827..2b00a12f 100644 --- a/doctr/__init__.py +++ b/doctr/__init__.py @@ -1,19 +1,18 @@ from .local import (encrypt_variable_travis, encrypt_to_file, GitHub_post, generate_GitHub_token, upload_GitHub_deploy_key, generate_ssh_key, check_repo_exists, guess_github_repo) -from .travis import (decrypt_file, setup_deploy_key, get_token, run, - setup_GitHub_push, checkout_deploy_branch, deploy_branch_exists, - set_git_user_email, create_deploy_branch, copy_to_tmp, sync_from_log, - commit_docs, push_docs, get_current_repo, find_sphinx_build_dir) +from .travis import Travis + +from .ci import sync_from_log, copy_to_tmp, find_sphinx_build_dir __all__ = [ 'encrypt_variable_travis', 'encrypt_to_file', 'GitHub_post', 'generate_GitHub_token', 'upload_GitHub_deploy_key', 'generate_ssh_key', 'check_repo_exists', 'guess_github_repo', - 'decrypt_file', 'setup_deploy_key', 'get_token', 'run', - 'setup_GitHub_push', 'set_git_user_email', 'checkout_deploy_branch', 'deploy_branch_exists', - 'create_deploy_branch', 'copy_to_tmp', 'sync_from_log', 'commit_docs', 'push_docs', 'get_current_repo', 'find_sphinx_build_dir' + 'Travis', + + 'sync_from_log', 'copy_to_tmp', 'find_sphinx_build_dir', ] from ._version import get_versions diff --git a/doctr/__main__.py b/doctr/__main__.py index 195e2709..4cb79b17 100644 --- a/doctr/__main__.py +++ b/doctr/__main__.py @@ -38,9 +38,8 @@ upload_GitHub_deploy_key, generate_ssh_key, check_repo_exists, GitHub_login, guess_github_repo, AuthenticationFailed, GitHubError, get_travis_token) -from .travis import (setup_GitHub_push, commit_docs, push_docs, - get_current_repo, sync_from_log, find_sphinx_build_dir, run, - get_travis_branch, copy_to_tmp, checkout_deploy_branch) +from .travis import Travis +from .ci import sync_from_log, copy_to_tmp, find_sphinx_build_dir, run from .common import (red, green, blue, bold_black, BOLD_BLACK, BOLD_MAGENTA, RESET, input) @@ -289,7 +288,8 @@ def deploy(args, parser): deploy_dir = args.gh_pages_docs or args.deploy_directory - build_repo = get_current_repo() + CI = Travis() + build_repo = CI.get_current_repo() deploy_repo = args.deploy_repo or build_repo if args.deploy_branch_name: @@ -301,14 +301,14 @@ def deploy(args, parser): current_commit = subprocess.check_output(['git', 'rev-parse', 'HEAD']).decode('utf-8').strip() try: - branch_whitelist = set() if args.require_master else set(get_travis_branch()) + branch_whitelist = set() if args.require_master else set(CI.branch()) branch_whitelist.update(set(config.get('branches', set()))) if args.branch_whitelist is not None: branch_whitelist.update(set(args.branch_whitelist)) elif not branch_whitelist: branch_whitelist = {'master'} - canpush = setup_GitHub_push(deploy_repo, deploy_branch=deploy_branch, + canpush = CI.setup_GitHub_push(deploy_repo, deploy_branch=deploy_branch, auth_type='token' if args.token else 'deploy_key', full_key_path=keypath, branch_whitelist=branch_whitelist, @@ -325,7 +325,7 @@ def deploy(args, parser): # Reset in case there are modified files that are tracked in the # deploy branch. run(['git', 'stash', '--all']) - checkout_deploy_branch(deploy_branch, canpush=canpush) + CI.checkout_deploy_branch(deploy_branch, canpush=canpush) if args.sync: log_file = os.path.join(deploy_dir, '.doctr-files') @@ -340,10 +340,10 @@ def deploy(args, parser): if args.command: run(args.command, shell=True) - changes = commit_docs(added=added, removed=removed) + changes = CI.commit_docs(added=added, removed=removed) if changes: if canpush and args.push: - push_docs(deploy_branch) + CI.push_docs(deploy_branch) else: print("Don't have permission to push. Not trying.") else: diff --git a/doctr/ci.py b/doctr/ci.py new file mode 100644 index 00000000..2fedb238 --- /dev/null +++ b/doctr/ci.py @@ -0,0 +1,544 @@ +""" +Functions that are common to all CIs +""" + +import os +import shlex +import shutil +import subprocess +import sys +import glob +import re +import pathlib +import tempfile +import time + +import requests + +from cryptography.fernet import Fernet + +from .common import red, blue, yellow +DOCTR_WORKING_BRANCH = '__doctr_working_branch' + +class CI: + def decrypt_file(self, file, key): + """ + Decrypts the file ``file``. + + The encrypted file is assumed to end with the ``.enc`` extension. The + decrypted file is saved to the same location without the ``.enc`` + extension. + + The permissions on the decrypted file are automatically set to 0o600. + + See also :func:`doctr.local.encrypt_file`. + + """ + if not file.endswith('.enc'): + raise ValueError("%s does not end with .enc" % file) + + fer = Fernet(key) + + with open(file, 'rb') as f: + decrypted_file = fer.decrypt(f.read()) + + with open(file[:-4], 'wb') as f: + f.write(decrypted_file) + + os.chmod(file[:-4], 0o600) + + def setup_deploy_key(self, keypath='github_deploy_key', key_ext='.enc', env_name='DOCTR_DEPLOY_ENCRYPTION_KEY'): + """ + Decrypts the deploy key and configures it with ssh + + The key is assumed to be encrypted as keypath + key_ext, and the + encryption key is assumed to be set in the environment variable + ``env_name``. If ``env_name`` is not set, it falls back to + ``DOCTR_DEPLOY_ENCRYPTION_KEY`` for backwards compatibility. + + If keypath + key_ext does not exist, it falls back to + ``github_deploy_key.enc`` for backwards compatibility. + """ + key = os.environ.get(env_name, os.environ.get("DOCTR_DEPLOY_ENCRYPTION_KEY", None)) + if not key: + raise RuntimeError("{env_name} or DOCTR_DEPLOY_ENCRYPTION_KEY environment variable is not set. Make sure you followed the instructions from 'doctr configure' properly. You may need to re-run 'doctr configure' to fix this error." + .format(env_name=env_name)) + + # Legacy keyfile name + if (not os.path.isfile(keypath + key_ext) and + os.path.isfile('github_deploy_key' + key_ext)): + keypath = 'github_deploy_key' + key_filename = os.path.basename(keypath) + key = key.encode('utf-8') + self.decrypt_file(keypath + key_ext, key) + + key_path = os.path.expanduser("~/.ssh/" + key_filename) + os.makedirs(os.path.expanduser("~/.ssh"), exist_ok=True) + os.rename(keypath, key_path) + + with open(os.path.expanduser("~/.ssh/config"), 'a') as f: + f.write("Host github.com" + ' IdentityFile "%s"' + " LogLevel ERROR\n" % key_path) + + # start ssh-agent and add key to it + # info from SSH agent has to be put into the environment + agent_info = subprocess.check_output(['ssh-agent', '-s']) + agent_info = agent_info.decode('utf-8') + agent_info = agent_info.split() + + AUTH_SOCK = agent_info[0].split('=')[1][:-1] + AGENT_PID = agent_info[3].split('=')[1][:-1] + + os.putenv('SSH_AUTH_SOCK', AUTH_SOCK) + os.putenv('SSH_AGENT_PID', AGENT_PID) + + run(['ssh-add', os.path.expanduser('~/.ssh/' + key_filename)]) + + def get_current_repo(self): + """ + Get the GitHub repo name for the current directory. + + Assumes that the repo is in the ``origin`` remote. + """ + remote_url = subprocess.check_output(['git', 'config', '--get', + 'remote.origin.url']).decode('utf-8') + + # Travis uses the https clone url + _, org, git_repo = remote_url.rsplit('.git', 1)[0].rsplit('/', 2) + return (org + '/' + git_repo) + + def setup_GitHub_push(self, deploy_repo, *, auth_type='deploy_key', + full_key_path='github_deploy_key.enc', require_master=None, + branch_whitelist=None, deploy_branch='gh-pages', + env_name='DOCTR_DEPLOY_ENCRYPTION_KEY', build_tags=False): + """ + Setup the remote to push to GitHub (to be run on CI). + + ``auth_type`` should be either ``'deploy_key'`` or ``'token'``. + + For ``auth_type='token'``, this sets up the remote with the token and + checks out the gh-pages branch. The token to push to GitHub is assumed to be in the ``GH_TOKEN`` environment + variable. + + For ``auth_type='deploy_key'``, this sets up the remote with ssh access. + """ + # Set to the name of the tag for tag builds + tag = self.tag() + + if branch_whitelist is None: + branch_whitelist={'master'} + + if require_master is not None: + import warnings + warnings.warn("`setup_GitHub_push`'s `require_master` argument in favor of `branch_whitelist=['master']`", + DeprecationWarning, + stacklevel=2) + branch_whitelist.add('master') + + if auth_type not in ['deploy_key', 'token']: + raise ValueError("auth_type must be 'deploy_key' or 'token'") + + # Check if the repo is a fork + REPO_URL = 'https://api.github.com/repos/{slug}' + r = requests.get(REPO_URL.format(slug=self.repo_slug())) + fork = r.json().get('fork', False) + + canpush = self.determine_push_rights( + branch_whitelist=branch_whitelist, + branch=self.branch(), + pull_request=self.is_pull_request(), + fork=fork, + tag=tag, + build_tags=build_tags) + + print("Setting git attributes") + self.set_git_user_email() + + remotes = subprocess.check_output(['git', 'remote']).decode('utf-8').split('\n') + if 'doctr_remote' in remotes: + print("doctr_remote already exists, removing") + run(['git', 'remote', 'remove', 'doctr_remote']) + print("Adding doctr remote") + if canpush: + if auth_type == 'token': + token = get_token() + run(['git', 'remote', 'add', 'doctr_remote', + 'https://{token}@github.com/{deploy_repo}.git'.format(token=token.decode('utf-8'), + deploy_repo=deploy_repo)]) + else: + keypath, key_ext = full_key_path.rsplit('.', 1) + key_ext = '.' + key_ext + try: + self.setup_deploy_key(keypath=keypath, key_ext=key_ext, env_name=env_name) + except RuntimeError: + # Rate limits prevent this check from working every time. By default, we + # assume it isn't a fork so that things just work on non-fork builds. + if r.status_code == 403: + print(yellow("Warning: GitHub's API rate limits prevented doctr from detecting if this build is a forked repo. If it is, you may ignore the 'DOCTR_DEPLOY_ENCRYPTION_KEY environment variable is not set' error that follows. If it is not, you should re-run 'doctr configure'. Note that doctr cannot deploy from fork builds due to limitations in Travis."), file=sys.stderr) + raise + + run(['git', 'remote', 'add', 'doctr_remote', + 'git@github.com:{deploy_repo}.git'.format(deploy_repo=deploy_repo)]) + else: + print('setting a read-only GitHub doctr_remote') + run(['git', 'remote', 'add', 'doctr_remote', + 'https://github.com/{deploy_repo}.git'.format(deploy_repo=deploy_repo)]) + + + print("Fetching doctr remote") + run(['git', 'fetch', 'doctr_remote']) + + return canpush + + def set_git_user_email(self): + """ + Set global user and email for git user if not already present on system + """ + username = subprocess.run(shlex.split('git config user.name'), stdout=subprocess.PIPE).stdout.strip().decode('utf-8') + if not username or username == "Travis CI User": + run(['git', 'config', '--global', 'user.name', "Doctr (Travis CI)"]) + else: + print("Not setting git user name, as it's already set to %r" % username) + + email = subprocess.run(shlex.split('git config user.email'), stdout=subprocess.PIPE).stdout.strip().decode('utf-8') + if not email or email == "travis@example.org": + # We need a dummy email or git will fail. We use this one as per + # https://help.github.com/articles/keeping-your-email-address-private/. + run(['git', 'config', '--global', 'user.email', 'drdoctr@users.noreply.github.com']) + else: + print("Not setting git user email, as it's already set to %r" % email) + + def checkout_deploy_branch(self, deploy_branch, canpush=True): + """ + Checkout the deploy branch, creating it if it doesn't exist. + """ + # Create an empty branch with .nojekyll if it doesn't already exist + self.create_deploy_branch(deploy_branch, push=canpush) + remote_branch = "doctr_remote/{}".format(deploy_branch) + print("Checking out doctr working branch tracking", remote_branch) + self.clear_working_branch() + # If gh-pages doesn't exist the above create_deploy_branch() will create + # it we can push, but if we can't, it won't and the --track would fail. + if run(['git', 'rev-parse', '--verify', remote_branch], exit=False) == 0: + extra_args = ['--track', remote_branch] + else: + extra_args = [] + run(['git', 'checkout', '-b', DOCTR_WORKING_BRANCH] + extra_args) + print("Done") + + return canpush + + def clear_working_branch(self): + local_branch_names = subprocess.check_output(['git', 'branch']).decode('utf-8').split() + if DOCTR_WORKING_BRANCH in local_branch_names: + run(['git', 'branch', '-D', DOCTR_WORKING_BRANCH]) + + def deploy_branch_exists(self, deploy_branch): + """ + Check if there is a remote branch with name specified in ``deploy_branch``. + + Note that default ``deploy_branch`` is ``gh-pages`` for regular repos and + ``master`` for ``github.io`` repos. + + This isn't completely robust. If there are multiple remotes and you have a + ``deploy_branch`` branch on the non-default remote, this won't see it. + """ + remote_name = 'doctr_remote' + branch_names = subprocess.check_output(['git', 'branch', '-r']).decode('utf-8').split() + + return '{}/{}'.format(remote_name, deploy_branch) in branch_names + + def create_deploy_branch(self, deploy_branch, push=True): + """ + If there is no remote branch with name specified in ``deploy_branch``, + create one. + + Note that default ``deploy_branch`` is ``gh-pages`` for regular + repos and ``master`` for ``github.io`` repos. + + Return True if ``deploy_branch`` was created, False if not. + """ + if not deploy_branch_exists(deploy_branch): + print("Creating {} branch on doctr_remote".format(deploy_branch)) + clear_working_branch() + run(['git', 'checkout', '--orphan', DOCTR_WORKING_BRANCH]) + # delete everything in the new ref. this is non-destructive to existing + # refs/branches, etc... + run(['git', 'rm', '-rf', '.']) + print("Adding .nojekyll file to working branch") + run(['touch', '.nojekyll']) + run(['git', 'add', '.nojekyll']) + run(['git', 'commit', '-m', 'Create new {} branch with .nojekyll'.format(deploy_branch)]) + if push: + print("Pushing working branch to remote {} branch".format(deploy_branch)) + run(['git', 'push', '-u', 'doctr_remote', '{}:{}'.format(DOCTR_WORKING_BRANCH, deploy_branch)]) + # return to master branch and clear the working branch + run(['git', 'checkout', 'master']) + run(['git', 'branch', '-D', DOCTR_WORKING_BRANCH]) + # fetch the remote so that doctr_remote/{deploy_branch} is resolved + run(['git', 'fetch', 'doctr_remote']) + + return True + return False + + + def commit_docs(self, *, added, removed): + """ + Commit the docs to the current branch + + Assumes that :func:`setup_GitHub_push`, which sets up the ``doctr_remote`` + remote, has been run. + + Returns True if changes were committed and False if no changes were + committed. + """ + commit_message = self.commit_message() + # Only commit if there were changes + if run(['git', 'diff-index', '--exit-code', '--cached', '--quiet', 'HEAD', '--'], exit=False) != 0: + print("Committing") + run(['git', 'commit', '-am', commit_message]) + return True + + return False + + def push_docs(self, deploy_branch='gh-pages', retries=5): + """ + Push the changes to the branch named ``deploy_branch``. + + Assumes that :func:`setup_GitHub_push` has been run and returned True, and + that :func:`commit_docs` has been run. Does not push anything if no changes + were made. + + """ + + code = 1 + while code and retries: + print("Pulling") + code = run(['git', 'pull', '-s', 'recursive', '-X', 'ours', + 'doctr_remote', deploy_branch], exit=False) + print("Pushing commit") + code = run(['git', 'push', '-q', 'doctr_remote', + '{}:{}'.format(DOCTR_WORKING_BRANCH, deploy_branch)], exit=False) + if code: + retries -= 1 + print("Push failed, retrying") + time.sleep(1) + else: + return + sys.exit("Giving up...") + + def last_commit_by_doctr(self): + """Check whether the author of `HEAD` is `doctr` to avoid starting an + infinite loop""" + + email = subprocess.check_output(["git", "show", "-s", "--format=%ae", "HEAD"]).decode('utf-8') + if email.strip() == "drdoctr@users.noreply.github.com": + return True + return False + + def determine_push_rights(self, *, branch_whitelist, branch, + pull_request, tag, build_tags, fork): + """Check if CI is running on ``master`` (or a whitelisted branch) to + determine if we can/should push the docs to the deploy repo + """ + canpush = True + + if tag: + if not build_tags: + print("The docs are not pushed on tag builds. To push on future tag builds, use --build-tags") + return build_tags + + if not any([re.compile(x).match(branch) for x in branch_whitelist]): + print("The docs are only pushed to gh-pages from master. To allow pushing from " + "a non-master branch, use the --no-require-master flag", file=sys.stderr) + print("This is the {branch} branch".format(branch=branch), file=sys.stderr) + canpush = False + + if pull_request: + print("The website and docs are not pushed to gh-pages on pull requests", file=sys.stderr) + canpush = False + + if fork: + print("The website and docs are not pushed to gh-pages on fork builds.", file=sys.stderr) + canpush = False + + if self.last_commit_by_doctr(): + print(red("The last commit on this branch was pushed by doctr. Not pushing to " + "avoid an infinite build-loop."), file=sys.stderr) + canpush = False + + return canpush + +def get_token(): + """ + Get the encrypted GitHub token in the CI. + + Make sure the contents this variable do not leak. The ``run()`` function + will remove this from the output, so always use it. + """ + token = os.environ.get("GH_TOKEN", None) + if not token: + token = "GH_TOKEN environment variable not set" + token = token.encode('utf-8') + return token + + +def run_command_hiding_token(args, token, shell=False): + if token: + stdout = stderr = subprocess.PIPE + else: + stdout = stderr = None + p = subprocess.run(args, stdout=stdout, stderr=stderr, shell=shell) + if token: + # XXX: Do this in a way that is streaming + out, err = p.stdout, p.stderr + out = out.replace(token, b"~"*len(token)) + err = err.replace(token, b"~"*len(token)) + if out: + print(out.decode('utf-8')) + if err: + print(err.decode('utf-8'), file=sys.stderr) + return p.returncode + +def run(args, shell=False, exit=True): + """ + Run the command ``args``. + + Automatically hides the secret GitHub token from the output. + + If shell=False (recommended for most commands), args should be a list of + strings. If shell=True, args should be a string of the command to run. + + If exit=True, it exits on nonzero returncode. Otherwise it returns the + returncode. + """ + if "GH_TOKEN" in os.environ: + token = get_token() + else: + token = b'' + + if not shell: + command = ' '.join(map(shlex.quote, args)) + else: + command = args + command = command.replace(token.decode('utf-8'), '~'*len(token)) + print(blue(command)) + sys.stdout.flush() + + returncode = run_command_hiding_token(args, token, shell=shell) + + if exit and returncode != 0: + sys.exit(red("%s failed: %s" % (command, returncode))) + return returncode + +def is_subdir(a, b): + """ + Return true if a is a subdirectory of b + """ + a, b = map(os.path.abspath, [a, b]) + + return os.path.commonpath([a, b]) == b + + +def copy_to_tmp(source): + """ + Copies ``source`` to a temporary directory, and returns the copied + location. + + If source is a file, the copied location is also a file. + """ + tmp_dir = tempfile.mkdtemp() + # Use pathlib because os.path.basename is different depending on whether + # the path ends in a / + p = pathlib.Path(source) + dirname = p.name or 'temp' + new_dir = os.path.join(tmp_dir, dirname) + if os.path.isdir(source): + shutil.copytree(source, new_dir) + else: + shutil.copy2(source, new_dir) + return new_dir + +def sync_from_log(src, dst, log_file, exclude=()): + """ + Sync the files in ``src`` to ``dst``. + + The files that are synced are logged to ``log_file``. If ``log_file`` + exists, the files in ``log_file`` are removed first. + + Returns ``(added, removed)``, where added is a list of all files synced from + ``src`` (even if it already existed in ``dst``), and ``removed`` is every + file from ``log_file`` that was removed from ``dst`` because it wasn't in + ``src``. ``added`` also includes the log file. + + ``exclude`` may be a list of paths from ``src`` that should be ignored. + Such paths are neither added nor removed, even if they are in the logfile. + """ + from os.path import join, exists, isdir + + exclude = [os.path.normpath(i) for i in exclude] + + added, removed = [], [] + + if not exists(log_file): + # Assume this is the first run + print("%s doesn't exist. Not removing any files." % log_file) + else: + with open(log_file) as f: + files = f.read().strip().split('\n') + + for new_f in files: + new_f = new_f.strip() + if any(is_subdir(new_f, os.path.join(dst, i)) for i in exclude): + pass + elif exists(new_f): + os.remove(new_f) + removed.append(new_f) + else: + print("Warning: File %s doesn't exist." % new_f, file=sys.stderr) + + if os.path.isdir(src): + if not src.endswith(os.sep): + src += os.sep + files = glob.iglob(join(src, '**'), recursive=True) + else: + files = [src] + src = os.path.dirname(src) + os.sep if os.sep in src else '' + + os.makedirs(dst, exist_ok=True) + + # sorted makes this easier to test + for f in sorted(files): + if any(is_subdir(f, os.path.join(src, i)) for i in exclude): + continue + new_f = join(dst, f[len(src):]) + + if isdir(f) or f.endswith(os.sep): + os.makedirs(new_f, exist_ok=True) + else: + shutil.copy2(f, new_f) + added.append(new_f) + if new_f in removed: + removed.remove(new_f) + + with open(log_file, 'w') as f: + f.write('\n'.join(added)) + + added.append(log_file) + + return added, removed + +def find_sphinx_build_dir(): + """ + Find build subfolder within sphinx docs directory. + + This is called by :func:`commit_docs` if keyword arg ``built_docs`` is not + specified on the command line. + """ + build = glob.glob('**/*build/html', recursive=True) + if not build: + raise RuntimeError("Could not find Sphinx build directory automatically") + build_folder = build[0] + + return build_folder diff --git a/doctr/tests/test_travis.py b/doctr/tests/test_travis.py index 81286cec..625fbcda 100644 --- a/doctr/tests/test_travis.py +++ b/doctr/tests/test_travis.py @@ -9,7 +9,8 @@ import pytest -from ..travis import sync_from_log, determine_push_rights, copy_to_tmp +from ..travis import Travis +from ..ci import sync_from_log, copy_to_tmp @pytest.mark.parametrize("src", ["src"]) @pytest.mark.parametrize("dst", ['.', 'dst']) @@ -258,61 +259,62 @@ def test_sync_from_log_file_to_dir(dst): os.chdir(old_curdir) -@pytest.mark.parametrize("""branch_whitelist, TRAVIS_BRANCH, - TRAVIS_PULL_REQUEST, TRAVIS_TAG, fork, build_tags, +@pytest.mark.parametrize("""branch_whitelist, branch, + pull_request, tag, fork, build_tags, canpush""", [ - ('master', 'doctr', 'true', "", False, False, False), - ('master', 'doctr', 'false', "", False, False, False), - ('master', 'master', 'true', "", False, False, False), - ('master', 'master', 'false', "", False, False, True), - ('doctr', 'doctr', 'True', "", False, False, False), - ('doctr', 'doctr', 'false', "", False, False, True), - ('set()', 'doctr', 'false', "", False, False, False), - - ('master', 'doctr', 'true', "tagname", False, False, False), - ('master', 'doctr', 'false', "tagname", False, False, False), - ('master', 'master', 'true', "tagname", False, False, False), - ('master', 'master', 'false', "tagname", False, False, False), - ('doctr', 'doctr', 'True', "tagname", False, False, False), - ('doctr', 'doctr', 'false', "tagname", False, False, False), - ('set()', 'doctr', 'false', "tagname", False, False, False), - - ('master', 'doctr', 'true', "", False, True, False), - ('master', 'doctr', 'false', "", False, True, False), - ('master', 'master', 'true', "", False, True, False), - ('master', 'master', 'false', "", False, True, True), - ('doctr', 'doctr', 'True', "", False, True, False), - ('doctr', 'doctr', 'false', "", False, True, True), - ('set()', 'doctr', 'false', "", False, True, False), - - ('master', 'doctr', 'true', "tagname", False, True, True), - ('master', 'doctr', 'false', "tagname", False, True, True), - ('master', 'master', 'true', "tagname", False, True, True), - ('master', 'master', 'false', "tagname", False, True, True), - ('doctr', 'doctr', 'True', "tagname", False, True, True), - ('doctr', 'doctr', 'false', "tagname", False, True, True), - ('set()', 'doctr', 'false', "tagname", False, True, True), - - ('master', 'doctr', 'true', "", True, False, False), - ('master', 'doctr', 'false', "", True, False, False), - ('master', 'master', 'true', "", True, False, False), - ('master', 'master', 'false', "", True, False, False), - ('doctr', 'doctr', 'True', "", True, False, False), - ('doctr', 'doctr', 'false', "", True, False, False), - ('set()', 'doctr', 'false', "", True, False, False), + ('master', 'doctr', True, "", False, False, False), + ('master', 'doctr', False, "", False, False, False), + ('master', 'master', True, "", False, False, False), + ('master', 'master', False, "", False, False, True), + ('doctr', 'doctr', True, "", False, False, False), + ('doctr', 'doctr', False, "", False, False, True), + ('set()', 'doctr', False, "", False, False, False), + + ('master', 'doctr', True, "tagname", False, False, False), + ('master', 'doctr', False, "tagname", False, False, False), + ('master', 'master', True, "tagname", False, False, False), + ('master', 'master', False, "tagname", False, False, False), + ('doctr', 'doctr', True, "tagname", False, False, False), + ('doctr', 'doctr', False, "tagname", False, False, False), + ('set()', 'doctr', False, "tagname", False, False, False), + + ('master', 'doctr', True, "", False, True, False), + ('master', 'doctr', False, "", False, True, False), + ('master', 'master', True, "", False, True, False), + ('master', 'master', False, "", False, True, True), + ('doctr', 'doctr', True, "", False, True, False), + ('doctr', 'doctr', False, "", False, True, True), + ('set()', 'doctr', False, "", False, True, False), + + ('master', 'doctr', True, "tagname", False, True, True), + ('master', 'doctr', False, "tagname", False, True, True), + ('master', 'master', True, "tagname", False, True, True), + ('master', 'master', False, "tagname", False, True, True), + ('doctr', 'doctr', True, "tagname", False, True, True), + ('doctr', 'doctr', False, "tagname", False, True, True), + ('set()', 'doctr', False, "tagname", False, True, True), + + ('master', 'doctr', True, "", True, False, False), + ('master', 'doctr', False, "", True, False, False), + ('master', 'master', True, "", True, False, False), + ('master', 'master', False, "", True, False, False), + ('doctr', 'doctr', True, "", True, False, False), + ('doctr', 'doctr', False, "", True, False, False), + ('set()', 'doctr', False, "", True, False, False), ]) -def test_determine_push_rights(branch_whitelist, TRAVIS_BRANCH, - TRAVIS_PULL_REQUEST, TRAVIS_TAG, build_tags, fork, canpush, monkeypatch): +def test_determine_push_rights(branch_whitelist, branch, + pull_request, tag, build_tags, fork, canpush, monkeypatch): branch_whitelist = {branch_whitelist} - assert determine_push_rights( + CI = Travis() + assert CI.determine_push_rights( branch_whitelist=branch_whitelist, - TRAVIS_BRANCH=TRAVIS_BRANCH, - TRAVIS_PULL_REQUEST=TRAVIS_PULL_REQUEST, - TRAVIS_TAG=TRAVIS_TAG, + branch=branch, + pull_request=pull_request, + tag=tag, fork=fork, build_tags=build_tags) == canpush diff --git a/doctr/travis.py b/doctr/travis.py index 0f15eca5..3966bff5 100644 --- a/doctr/travis.py +++ b/doctr/travis.py @@ -18,256 +18,36 @@ from cryptography.fernet import Fernet from .common import red, blue, yellow -DOCTR_WORKING_BRANCH = '__doctr_working_branch' - -def decrypt_file(file, key): - """ - Decrypts the file ``file``. - - The encrypted file is assumed to end with the ``.enc`` extension. The - decrypted file is saved to the same location without the ``.enc`` - extension. - - The permissions on the decrypted file are automatically set to 0o600. - - See also :func:`doctr.local.encrypt_file`. - - """ - if not file.endswith('.enc'): - raise ValueError("%s does not end with .enc" % file) - - fer = Fernet(key) - - with open(file, 'rb') as f: - decrypted_file = fer.decrypt(f.read()) - - with open(file[:-4], 'wb') as f: - f.write(decrypted_file) - - os.chmod(file[:-4], 0o600) - -def setup_deploy_key(keypath='github_deploy_key', key_ext='.enc', env_name='DOCTR_DEPLOY_ENCRYPTION_KEY'): - """ - Decrypts the deploy key and configures it with ssh - - The key is assumed to be encrypted as keypath + key_ext, and the - encryption key is assumed to be set in the environment variable - ``env_name``. If ``env_name`` is not set, it falls back to - ``DOCTR_DEPLOY_ENCRYPTION_KEY`` for backwards compatibility. - - If keypath + key_ext does not exist, it falls back to - ``github_deploy_key.enc`` for backwards compatibility. - """ - key = os.environ.get(env_name, os.environ.get("DOCTR_DEPLOY_ENCRYPTION_KEY", None)) - if not key: - raise RuntimeError("{env_name} or DOCTR_DEPLOY_ENCRYPTION_KEY environment variable is not set. Make sure you followed the instructions from 'doctr configure' properly. You may need to re-run 'doctr configure' to fix this error." - .format(env_name=env_name)) - - # Legacy keyfile name - if (not os.path.isfile(keypath + key_ext) and - os.path.isfile('github_deploy_key' + key_ext)): - keypath = 'github_deploy_key' - key_filename = os.path.basename(keypath) - key = key.encode('utf-8') - decrypt_file(keypath + key_ext, key) - - key_path = os.path.expanduser("~/.ssh/" + key_filename) - os.makedirs(os.path.expanduser("~/.ssh"), exist_ok=True) - os.rename(keypath, key_path) - - with open(os.path.expanduser("~/.ssh/config"), 'a') as f: - f.write("Host github.com" - ' IdentityFile "%s"' - " LogLevel ERROR\n" % key_path) - - # start ssh-agent and add key to it - # info from SSH agent has to be put into the environment - agent_info = subprocess.check_output(['ssh-agent', '-s']) - agent_info = agent_info.decode('utf-8') - agent_info = agent_info.split() - - AUTH_SOCK = agent_info[0].split('=')[1][:-1] - AGENT_PID = agent_info[3].split('=')[1][:-1] - - os.putenv('SSH_AUTH_SOCK', AUTH_SOCK) - os.putenv('SSH_AGENT_PID', AGENT_PID) - - run(['ssh-add', os.path.expanduser('~/.ssh/' + key_filename)]) - -def run_command_hiding_token(args, token, shell=False): - if token: - stdout = stderr = subprocess.PIPE - else: - stdout = stderr = None - p = subprocess.run(args, stdout=stdout, stderr=stderr, shell=shell) - if token: - # XXX: Do this in a way that is streaming - out, err = p.stdout, p.stderr - out = out.replace(token, b"~"*len(token)) - err = err.replace(token, b"~"*len(token)) - if out: - print(out.decode('utf-8')) - if err: - print(err.decode('utf-8'), file=sys.stderr) - return p.returncode - -def get_token(): - """ - Get the encrypted GitHub token in Travis. - - Make sure the contents this variable do not leak. The ``run()`` function - will remove this from the output, so always use it. - """ - token = os.environ.get("GH_TOKEN", None) - if not token: - token = "GH_TOKEN environment variable not set" - token = token.encode('utf-8') - return token - -def run(args, shell=False, exit=True): - """ - Run the command ``args``. - - Automatically hides the secret GitHub token from the output. - - If shell=False (recommended for most commands), args should be a list of - strings. If shell=True, args should be a string of the command to run. - - If exit=True, it exits on nonzero returncode. Otherwise it returns the - returncode. - """ - if "GH_TOKEN" in os.environ: - token = get_token() - else: - token = b'' - - if not shell: - command = ' '.join(map(shlex.quote, args)) - else: - command = args - command = command.replace(token.decode('utf-8'), '~'*len(token)) - print(blue(command)) - sys.stdout.flush() - - returncode = run_command_hiding_token(args, token, shell=shell) - - if exit and returncode != 0: - sys.exit(red("%s failed: %s" % (command, returncode))) - return returncode - -def get_current_repo(): - """ - Get the GitHub repo name for the current directory. - - Assumes that the repo is in the ``origin`` remote. - """ - remote_url = subprocess.check_output(['git', 'config', '--get', - 'remote.origin.url']).decode('utf-8') - - # Travis uses the https clone url - _, org, git_repo = remote_url.rsplit('.git', 1)[0].rsplit('/', 2) - return (org + '/' + git_repo) - -def get_travis_branch(): - """Get the name of the branch that the PR is from. - - Note that this is not simply ``$TRAVIS_BRANCH``. the ``push`` build will - use the correct branch (the branch that the PR is from) but the ``pr`` - build will use the _target_ of the PR (usually master). So instead, we ask - for ``$TRAVIS_PULL_REQUEST_BRANCH`` if it's a PR build, and - ``$TRAVIS_BRANCH`` if it's a push build. - """ - if os.environ.get("TRAVIS_PULL_REQUEST", "") == "true": - return os.environ.get("TRAVIS_PULL_REQUEST_BRANCH", "") - else: - return os.environ.get("TRAVIS_BRANCH", "") - -def setup_GitHub_push(deploy_repo, *, auth_type='deploy_key', - full_key_path='github_deploy_key.enc', require_master=None, - branch_whitelist=None, deploy_branch='gh-pages', - env_name='DOCTR_DEPLOY_ENCRYPTION_KEY', build_tags=False): - """ - Setup the remote to push to GitHub (to be run on Travis). - - ``auth_type`` should be either ``'deploy_key'`` or ``'token'``. - - For ``auth_type='token'``, this sets up the remote with the token and - checks out the gh-pages branch. The token to push to GitHub is assumed to be in the ``GH_TOKEN`` environment - variable. - - For ``auth_type='deploy_key'``, this sets up the remote with ssh access. - """ - # Set to the name of the tag for tag builds - TRAVIS_TAG = os.environ.get("TRAVIS_TAG", "") - - if branch_whitelist is None: - branch_whitelist={'master'} - - if require_master is not None: - import warnings - warnings.warn("`setup_GitHub_push`'s `require_master` argument in favor of `branch_whitelist=['master']`", - DeprecationWarning, - stacklevel=2) - branch_whitelist.add('master') - - if auth_type not in ['deploy_key', 'token']: - raise ValueError("auth_type must be 'deploy_key' or 'token'") - - TRAVIS_BRANCH = os.environ.get("TRAVIS_BRANCH", "") - TRAVIS_PULL_REQUEST = os.environ.get("TRAVIS_PULL_REQUEST", "") - - # Check if the repo is a fork - TRAVIS_REPO_SLUG = os.environ["TRAVIS_REPO_SLUG"] - REPO_URL = 'https://api.github.com/repos/{slug}' - r = requests.get(REPO_URL.format(slug=TRAVIS_REPO_SLUG)) - fork = r.json().get('fork', False) - - canpush = determine_push_rights( - branch_whitelist=branch_whitelist, - TRAVIS_BRANCH=TRAVIS_BRANCH, - TRAVIS_PULL_REQUEST=TRAVIS_PULL_REQUEST, - fork=fork, - TRAVIS_TAG=TRAVIS_TAG, - build_tags=build_tags) - - print("Setting git attributes") - set_git_user_email() - - remotes = subprocess.check_output(['git', 'remote']).decode('utf-8').split('\n') - if 'doctr_remote' in remotes: - print("doctr_remote already exists, removing") - run(['git', 'remote', 'remove', 'doctr_remote']) - print("Adding doctr remote") - if canpush: - if auth_type == 'token': - token = get_token() - run(['git', 'remote', 'add', 'doctr_remote', - 'https://{token}@github.com/{deploy_repo}.git'.format(token=token.decode('utf-8'), - deploy_repo=deploy_repo)]) +from .ci import CI + +class Travis(CI): + def branch(self): + """Get the name of the branch that the PR is from. + + Note that this is not simply ``$TRAVIS_BRANCH``. the ``push`` build will + use the correct branch (the branch that the PR is from) but the ``pr`` + build will use the _target_ of the PR (usually master). So instead, we ask + for ``$TRAVIS_PULL_REQUEST_BRANCH`` if it's a PR build, and + ``$TRAVIS_BRANCH`` if it's a push build. + """ + if os.environ.get("TRAVIS_PULL_REQUEST", "") == "true": + return os.environ.get("TRAVIS_PULL_REQUEST_BRANCH", "") else: - keypath, key_ext = full_key_path.rsplit('.', 1) - key_ext = '.' + key_ext - try: - setup_deploy_key(keypath=keypath, key_ext=key_ext, env_name=env_name) - except RuntimeError: - # Rate limits prevent this check from working every time. By default, we - # assume it isn't a fork so that things just work on non-fork builds. - if r.status_code == 403: - print(yellow("Warning: GitHub's API rate limits prevented doctr from detecting if this build is a forked repo. If it is, you may ignore the 'DOCTR_DEPLOY_ENCRYPTION_KEY environment variable is not set' error that follows. If it is not, you should re-run 'doctr configure'. Note that doctr cannot deploy from fork builds due to limitations in Travis."), file=sys.stderr) - raise - - run(['git', 'remote', 'add', 'doctr_remote', - 'git@github.com:{deploy_repo}.git'.format(deploy_repo=deploy_repo)]) - else: - print('setting a read-only GitHub doctr_remote') - run(['git', 'remote', 'add', 'doctr_remote', - 'https://github.com/{deploy_repo}.git'.format(deploy_repo=deploy_repo)]) + return os.environ.get("TRAVIS_BRANCH", "") + def tag(self): + return os.environ.get("TRAVIS_TAG", "") - print("Fetching doctr remote") - run(['git', 'fetch', 'doctr_remote']) + def is_fork(self): + # Check if the repo is a fork + TRAVIS_REPO_SLUG = os.environ["TRAVIS_REPO_SLUG"] + REPO_URL = 'https://api.github.com/repos/{slug}' + r = requests.get(REPO_URL.format(slug=TRAVIS_REPO_SLUG)) + fork = r.json().get('fork', False) + return fork - return canpush + def is_pull_request(self): + return os.environ.get("TRAVIS_PULL_REQUEST", "") != "false" def set_git_user_email(): """ @@ -287,205 +67,13 @@ def set_git_user_email(): else: print("Not setting git user email, as it's already set to %r" % email) -def checkout_deploy_branch(deploy_branch, canpush=True): - """ - Checkout the deploy branch, creating it if it doesn't exist. - """ - # Create an empty branch with .nojekyll if it doesn't already exist - create_deploy_branch(deploy_branch, push=canpush) - remote_branch = "doctr_remote/{}".format(deploy_branch) - print("Checking out doctr working branch tracking", remote_branch) - clear_working_branch() - # If gh-pages doesn't exist the above create_deploy_branch() will create - # it we can push, but if we can't, it won't and the --track would fail. - if run(['git', 'rev-parse', '--verify', remote_branch], exit=False) == 0: - extra_args = ['--track', remote_branch] - else: - extra_args = [] - run(['git', 'checkout', '-b', DOCTR_WORKING_BRANCH] + extra_args) - print("Done") - - return canpush - -def clear_working_branch(): - local_branch_names = subprocess.check_output(['git', 'branch']).decode('utf-8').split() - if DOCTR_WORKING_BRANCH in local_branch_names: - run(['git', 'branch', '-D', DOCTR_WORKING_BRANCH]) - -def deploy_branch_exists(deploy_branch): - """ - Check if there is a remote branch with name specified in ``deploy_branch``. - - Note that default ``deploy_branch`` is ``gh-pages`` for regular repos and - ``master`` for ``github.io`` repos. - - This isn't completely robust. If there are multiple remotes and you have a - ``deploy_branch`` branch on the non-default remote, this won't see it. - """ - remote_name = 'doctr_remote' - branch_names = subprocess.check_output(['git', 'branch', '-r']).decode('utf-8').split() - - return '{}/{}'.format(remote_name, deploy_branch) in branch_names - -def create_deploy_branch(deploy_branch, push=True): - """ - If there is no remote branch with name specified in ``deploy_branch``, - create one. - - Note that default ``deploy_branch`` is ``gh-pages`` for regular - repos and ``master`` for ``github.io`` repos. - - Return True if ``deploy_branch`` was created, False if not. - """ - if not deploy_branch_exists(deploy_branch): - print("Creating {} branch on doctr_remote".format(deploy_branch)) - clear_working_branch() - run(['git', 'checkout', '--orphan', DOCTR_WORKING_BRANCH]) - # delete everything in the new ref. this is non-destructive to existing - # refs/branches, etc... - run(['git', 'rm', '-rf', '.']) - print("Adding .nojekyll file to working branch") - run(['touch', '.nojekyll']) - run(['git', 'add', '.nojekyll']) - run(['git', 'commit', '-m', 'Create new {} branch with .nojekyll'.format(deploy_branch)]) - if push: - print("Pushing working branch to remote {} branch".format(deploy_branch)) - run(['git', 'push', '-u', 'doctr_remote', '{}:{}'.format(DOCTR_WORKING_BRANCH, deploy_branch)]) - # return to master branch and clear the working branch - run(['git', 'checkout', 'master']) - run(['git', 'branch', '-D', DOCTR_WORKING_BRANCH]) - # fetch the remote so that doctr_remote/{deploy_branch} is resolved - run(['git', 'fetch', 'doctr_remote']) - - return True - return False - -def find_sphinx_build_dir(): - """ - Find build subfolder within sphinx docs directory. - - This is called by :func:`commit_docs` if keyword arg ``built_docs`` is not - specified on the command line. - """ - build = glob.glob('**/*build/html', recursive=True) - if not build: - raise RuntimeError("Could not find Sphinx build directory automatically") - build_folder = build[0] - - return build_folder - # Here is the logic to get the Travis job number, to only run commit_docs in # the right build. # # TRAVIS_JOB_NUMBER = os.environ.get("TRAVIS_JOB_NUMBER", '') # ACTUAL_TRAVIS_JOB_NUMBER = TRAVIS_JOB_NUMBER.split('.')[1] -def copy_to_tmp(source): - """ - Copies ``source`` to a temporary directory, and returns the copied - location. - - If source is a file, the copied location is also a file. - """ - tmp_dir = tempfile.mkdtemp() - # Use pathlib because os.path.basename is different depending on whether - # the path ends in a / - p = pathlib.Path(source) - dirname = p.name or 'temp' - new_dir = os.path.join(tmp_dir, dirname) - if os.path.isdir(source): - shutil.copytree(source, new_dir) - else: - shutil.copy2(source, new_dir) - return new_dir - -def is_subdir(a, b): - """ - Return true if a is a subdirectory of b - """ - a, b = map(os.path.abspath, [a, b]) - - return os.path.commonpath([a, b]) == b - -def sync_from_log(src, dst, log_file, exclude=()): - """ - Sync the files in ``src`` to ``dst``. - - The files that are synced are logged to ``log_file``. If ``log_file`` - exists, the files in ``log_file`` are removed first. - - Returns ``(added, removed)``, where added is a list of all files synced from - ``src`` (even if it already existed in ``dst``), and ``removed`` is every - file from ``log_file`` that was removed from ``dst`` because it wasn't in - ``src``. ``added`` also includes the log file. - - ``exclude`` may be a list of paths from ``src`` that should be ignored. - Such paths are neither added nor removed, even if they are in the logfile. - """ - from os.path import join, exists, isdir - - exclude = [os.path.normpath(i) for i in exclude] - - added, removed = [], [] - - if not exists(log_file): - # Assume this is the first run - print("%s doesn't exist. Not removing any files." % log_file) - else: - with open(log_file) as f: - files = f.read().strip().split('\n') - - for new_f in files: - new_f = new_f.strip() - if any(is_subdir(new_f, os.path.join(dst, i)) for i in exclude): - pass - elif exists(new_f): - os.remove(new_f) - removed.append(new_f) - else: - print("Warning: File %s doesn't exist." % new_f, file=sys.stderr) - - if os.path.isdir(src): - if not src.endswith(os.sep): - src += os.sep - files = glob.iglob(join(src, '**'), recursive=True) - else: - files = [src] - src = os.path.dirname(src) + os.sep if os.sep in src else '' - - os.makedirs(dst, exist_ok=True) - - # sorted makes this easier to test - for f in sorted(files): - if any(is_subdir(f, os.path.join(src, i)) for i in exclude): - continue - new_f = join(dst, f[len(src):]) - - if isdir(f) or f.endswith(os.sep): - os.makedirs(new_f, exist_ok=True) - else: - shutil.copy2(f, new_f) - added.append(new_f) - if new_f in removed: - removed.remove(new_f) - - with open(log_file, 'w') as f: - f.write('\n'.join(added)) - - added.append(log_file) - - return added, removed - -def commit_docs(*, added, removed): - """ - Commit the docs to the current branch - - Assumes that :func:`setup_GitHub_push`, which sets up the ``doctr_remote`` - remote, has been run. - - Returns True if changes were committed and False if no changes were - committed. - """ +def commit_message(): TRAVIS_BUILD_NUMBER = os.environ.get("TRAVIS_BUILD_NUMBER", "") TRAVIS_BRANCH = os.environ.get("TRAVIS_BRANCH", "") TRAVIS_COMMIT = os.environ.get("TRAVIS_COMMIT", "") @@ -496,12 +84,6 @@ def commit_docs(*, added, removed): DOCTR_COMMAND = ' '.join(map(shlex.quote, sys.argv)) - - if added: - run(['git', 'add', *added]) - if removed: - run(['git', 'rm', *removed]) - commit_message = """\ Update docs after building Travis build {TRAVIS_BUILD_NUMBER} of {TRAVIS_REPO_SLUG} @@ -524,49 +106,7 @@ def commit_docs(*, added, removed): TRAVIS_JOB_WEB_URL=TRAVIS_JOB_WEB_URL, DOCTR_COMMAND=DOCTR_COMMAND, ) - - # Only commit if there were changes - if run(['git', 'diff-index', '--exit-code', '--cached', '--quiet', 'HEAD', '--'], exit=False) != 0: - print("Committing") - run(['git', 'commit', '-am', commit_message]) - return True - - return False - -def push_docs(deploy_branch='gh-pages', retries=5): - """ - Push the changes to the branch named ``deploy_branch``. - - Assumes that :func:`setup_GitHub_push` has been run and returned True, and - that :func:`commit_docs` has been run. Does not push anything if no changes - were made. - - """ - - code = 1 - while code and retries: - print("Pulling") - code = run(['git', 'pull', '-s', 'recursive', '-X', 'ours', - 'doctr_remote', deploy_branch], exit=False) - print("Pushing commit") - code = run(['git', 'push', '-q', 'doctr_remote', - '{}:{}'.format(DOCTR_WORKING_BRANCH, deploy_branch)], exit=False) - if code: - retries -= 1 - print("Push failed, retrying") - time.sleep(1) - else: - return - sys.exit("Giving up...") - -def last_commit_by_doctr(): - """Check whether the author of `HEAD` is `doctr` to avoid starting an - infinite loop""" - - email = subprocess.check_output(["git", "show", "-s", "--format=%ae", "HEAD"]).decode('utf-8') - if email.strip() == "drdoctr@users.noreply.github.com": - return True - return False + return commit_message def determine_push_rights(*, branch_whitelist, TRAVIS_BRANCH, TRAVIS_PULL_REQUEST, TRAVIS_TAG, build_tags, fork): From a6f8faac57d4868d76b7eb8e389a82aeb34654a8 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Wed, 11 Nov 2020 17:57:53 -0700 Subject: [PATCH 5/8] Install pynacl in Travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 5971922c..b8fe5cff 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,7 +36,7 @@ install: - conda config --add channels conda-forge # For sphinxcontrib.autoprogram - conda update -q conda - conda info -a - - conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION requests cryptography sphinx pyflakes sphinxcontrib-autoprogram pytest sphinx-issues pyyaml + - conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION requests cryptography sphinx pyflakes sphinxcontrib-autoprogram pytest sphinx-issues pyyaml pynacl - source activate test-environment script: From 681696a6148ccf0d661e58bcbad7be0968b26866 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 12 Nov 2020 14:08:29 -0700 Subject: [PATCH 6/8] Work on make doctr configure work with GitHub Actions (not tested yet) --- doctr/__main__.py | 207 +++++++++++++++++++++++++++------------- doctr/github_actions.py | 74 ++++++++++++++ doctr/local.py | 26 ++++- doctr/travis.py | 142 +++++++-------------------- 4 files changed, 274 insertions(+), 175 deletions(-) create mode 100644 doctr/github_actions.py diff --git a/doctr/__main__.py b/doctr/__main__.py index 4cb79b17..356a9375 100644 --- a/doctr/__main__.py +++ b/doctr/__main__.py @@ -34,11 +34,13 @@ from textwrap import dedent -from .local import (generate_GitHub_token, encrypt_variable_travis, encrypt_to_file, - upload_GitHub_deploy_key, generate_ssh_key, check_repo_exists, -GitHub_login, guess_github_repo, AuthenticationFailed, GitHubError, - get_travis_token) +from .local import (generate_GitHub_token, encrypt_variable_travis, + encrypt_variable_github_actions, encrypt_to_file, + upload_GitHub_deploy_key, generate_ssh_key, + check_repo_exists, GitHub_login, guess_github_repo, + AuthenticationFailed, GitHubError, get_travis_token) from .travis import Travis +from .github_actions import GitHubActions from .ci import sync_from_log, copy_to_tmp, find_sphinx_build_dir, run from .common import (red, green, blue, bold_black, BOLD_BLACK, BOLD_MAGENTA, @@ -126,11 +128,11 @@ def get_parser(config=None): subcommand = parser.add_subparsers(title='subcommand', dest='subcommand') - deploy_parser = subcommand.add_parser('deploy', help="""Deploy the docs to GitHub from Travis.""") + deploy_parser = subcommand.add_parser('deploy', help="""Deploy the docs to GitHub from CI.""") deploy_parser.set_defaults(func=deploy) deploy_parser_add_argument = make_parser_with_config_adder(deploy_parser, config) deploy_parser_add_argument('--force', action='store_true', help="""Run the deploy command even - if we do not appear to be on Travis.""") + if we do not appear to be on CI.""") deploy_parser_add_argument('deploy_directory', type=str, nargs='?', help="""Directory to deploy the html documentation to on gh-pages.""") deploy_parser_add_argument('--token', action='store_true', default=False, @@ -173,10 +175,9 @@ def get_parser(config=None): default=True, help="Run all the steps except the last push step. " "Useful for debugging") deploy_parser_add_argument('--build-tags', action='store_true', - default=False, help="""Deploy on tag builds. On a tag build, - $TRAVIS_TAG is set to the name of the tag. The default is to not - deploy on tag builds. Note that this will still build on a branch, - unless --branch-whitelist (with no arguments) is passed.""") + default=False, help="""Deploy on tag builds. The default is to not deploy on tag builds. Note that + this will still build on a branch, unless --branch-whitelist (with no + arguments) is passed.""") deploy_parser_add_argument('--gh-pages-docs', default=None, help="""!!DEPRECATED!! Directory to deploy the html documentation to on gh-pages. The default is %(default)r. The deploy directory should be passed as @@ -184,14 +185,16 @@ def get_parser(config=None): compatibility.""") deploy_parser_add_argument('--exclude', nargs='+', default=(), help="""Files and directories from --built-docs that are not copied.""") - + deploy_parser.add_help('--ci', default=None, help="""The CI system that + doctr is being run on. Should be one of ['travis', 'github-actions']. The + default is to detect automatically.""") if config: print('Warning, The following options in `.travis.yml` were not recognized:\n%s' % json.dumps(config, indent=2)) - configure_parser = subcommand.add_parser('configure', help="Configure doctr. This command should be run locally (not on Travis).") + configure_parser = subcommand.add_parser('configure', help="Configure doctr. This command should be run locally (not on CI).") configure_parser.set_defaults(func=configure) configure_parser.add_argument('--force', action='store_true', help="""Run the configure command even - if we appear to be on Travis.""") + if we appear to be on CI.""") configure_parser.add_argument('--token', action="store_true", default=False, help="""Generate a personal access token to push to GitHub. The default is to use a deploy key. WARNING: This will grant read/write access to all the @@ -212,24 +215,31 @@ def get_parser(config=None): check which the repo is activated on and ask if it is activated on both.""", choices=['c', 'com', '.com', 'travis-ci.com', 'o', 'org', '.org', 'travis-ci.org']) + configure_parser.add_argument('--ci-service', default=None, help="""The CI service to configure doctr for. Must be one of ['travis', + 'github-actions']. The default is to infer automatically based on which is + enabled, and ask if more than one is.""") return parser -def get_config(): +def get_config(ci_type): """ This load some configuration from the ``.travis.yml``, if file is present, ``doctr`` key if present. """ - p = Path('.travis.yml') - if not p.exists(): - return {} - with p.open() as f: - travis_config = yaml.safe_load(f.read()) + if ci_type == 'travis': + p = Path('.travis.yml') + if not p.exists(): + return {} + with p.open() as f: + travis_config = yaml.safe_load(f.read()) - config = travis_config.get('doctr', {}) + config = travis_config.get('doctr', {}) - if not isinstance(config, dict): - raise ValueError('config is not a dict: {}'.format(config)) + if not isinstance(config, dict): + raise ValueError('config is not a dict: {}'.format(config)) + else: + # GitHub Actions level config is not yet supported + config = {} return config def get_deploy_key_repo(deploy_repo, keypath, key_ext=''): @@ -265,14 +275,27 @@ def process_args(parser): def on_travis(): return os.environ.get("TRAVIS_JOB_NUMBER", '') +def on_github_actions(): + return os.environ.get("GITHUB_ACTIONS", "") == "true" + def deploy(args, parser): print("Running doctr deploy, version", __version__) - if not args.force and not on_travis(): - parser.error("doctr does not appear to be running on Travis. Use " - "doctr deploy --force to run anyway.") - - config = get_config() + if not args.force and not (on_travis() or on_github_actions()): + parser.error("doctr does not appear to be running on CI. Use " + "doctr deploy --force --ci to run anyway.") + + if args.ci: + if args.ci not in ['travis', 'github-actions']: + parser.error("--ci must be one of ['travis', 'github-actions']") + ci_type = args.ci + elif on_travis(): + ci_type = 'travis' + elif on_github_actions(): + ci_type = 'github-actions' + else: + parser.error("Could not detect the CI type (Travis or GitHub Actions). Make sure doctr deploy is being run on CI. You can set the CI type manually using the --ci flag.") + config = get_config(ci_type) if args.tmp_dir: parser.error("The --tmp-dir flag has been removed (doctr no longer uses a temporary directory when deploying).") @@ -288,7 +311,11 @@ def deploy(args, parser): deploy_dir = args.gh_pages_docs or args.deploy_directory - CI = Travis() + if ci_type == 'travis': + CI = Travis() + elif ci_type == 'github-actions': + CI = GitHubActions() + build_repo = CI.get_current_repo() deploy_repo = args.deploy_repo or build_repo @@ -383,6 +410,10 @@ def configure(args, parser): parser.error(red("doctr appears to be running on Travis. Use " "doctr configure --force to run anyway.")) + if not args.force and on_github_actions(): + parser.error(red("doctr appears to be running on GitHub Actions. Use " + "doctr configure --force to run anyway.")) + if not args.authenticate: args.upload_key = False @@ -396,7 +427,7 @@ def configure(args, parser): Welcome to Doctr. We need to ask you a few questions to get you on your way to automatically - deploying from Travis CI to GitHub pages. + deploying from CI to GitHub pages. """))) login_kwargs = {} @@ -427,21 +458,50 @@ def configure(args, parser): if is_private and not args.authenticate: sys.exit(red("--no-authenticate is not supported for private repositories.")) - headers = {} - travis_token = None - if is_private: - if args.token: - GitHub_token = generate_GitHub_token(note="Doctr token for pushing to gh-pages from Travis (for {build_repo}).".format(build_repo=build_repo), - scopes=["read:org", "user:email", "repo"], **login_kwargs)['token'] - travis_token = get_travis_token(GitHub_token=GitHub_token, **login_kwargs) - headers['Authorization'] = "token {}".format(travis_token) - - service = args.travis_tld if args.travis_tld else 'travis' - c = check_repo_exists(build_repo, service=service, ask=True, headers=headers) - tld = c['service'][-4:] - is_private = c['private'] or is_private - if is_private and not args.authenticate: - sys.exit(red("--no-authenticate is not supported for private repos.")) + + if args.ci_service: + service = args.ci_service + else: + headers = {} + travis_token = None + if is_private: + if args.token: + GitHub_token = generate_GitHub_token(note="Doctr token for pushing to gh-pages from Travis (for {build_repo}).".format(build_repo=build_repo), + scopes=["read:org", "user:email", "repo"], **login_kwargs)['token'] + travis_token = get_travis_token(GitHub_token=GitHub_token, **login_kwargs) + headers['Authorization'] = "token {}".format(travis_token) + + travis_service = args.travis_tld if args.travis_tld else 'travis' + c = check_repo_exists(build_repo, service=travis_service, + ask=True, headers=headers, raise_=False) + if not c: + travis = False + else: + tld = c['service'][-4:] + is_private = c['private'] or is_private + if is_private and not args.authenticate: + sys.exit(red("--no-authenticate is not supported for private repos.")) + travis = travis_service + + github_actions = check_repo_exists(build_repo, service='github actions') + + if travis and github_actions: + while True: + print(green("{build_repo} appears to exist on both Travis CI and GitHub Actions.".format(build_repo=build_repo))) + preferred = input("Which do you want to use? [{default}/GitHub Actions] ".format(default=blue("Travis CI"))) + preferred = preferred.lower().strip() + if preferred in ['t', 'travis', 'travis ci']: + service = travis + break + elif preferred in ['g', 'github', 'github actions']: + service = github_actions['service'] + break + else: + print(red("Please type 'Travis CI' or 'GitHub Actions'.")) + elif not (travis or github_actions): + raise RuntimeError("{build_repo} does not appear to have Travis CI or GitHub Actions enabled. Enable the one you want to use doctr with and run 'doctr configure' again.".format(build_repo=build_repo)) + else: + service = travis or github_actions['service'] get_build_repo = True except GitHubError: @@ -472,8 +532,12 @@ def configure(args, parser): if args.token: if not GitHub_token: GitHub_token = generate_GitHub_token(**login_kwargs)['token'] - encrypted_variable = encrypt_variable_travis("GH_TOKEN={GitHub_token}".format(GitHub_token=GitHub_token).encode('utf-8'), - build_repo=build_repo, tld=tld, travis_token=travis_token, **login_kwargs) + if 'travis' in service: + encrypted_variable = encrypt_variable_travis("GH_TOKEN={GitHub_token}".format(GitHub_token=GitHub_token).encode('utf-8'), + build_repo=build_repo, tld=tld, travis_token=travis_token, **login_kwargs) + else: + encrypted_variable = encrypt_variable_github_actions("GH_TOKEN={GitHub_token}".format(GitHub_token=GitHub_token).encode('utf-8'), + build_repo=build_repo, **login_kwargs) print(dedent(""" A personal access token for doctr has been created. @@ -487,9 +551,12 @@ def configure(args, parser): key = encrypt_to_file(private_ssh_key, keypath + '.enc') del private_ssh_key # Prevent accidental use below public_ssh_key = public_ssh_key.decode('ASCII') - encrypted_variable = encrypt_variable_travis(env_name.encode('utf-8') + b"=" + key, + if 'travis' in service: + encrypted_variable = encrypt_variable_travis(env_name.encode('utf-8') + b"=" + key, build_repo=build_repo, tld=tld, travis_token=travis_token, **login_kwargs) - + else: + encrypted_variable = encrypt_variable_github_actions(env_name.encode('utf-8') + b"=" + key, + build_repo=build_repo, **login_kwargs) deploy_keys_url = 'https://github.com/{deploy_repo}/settings/keys'.format(deploy_repo=deploy_key_repo) if args.upload_key: @@ -531,23 +598,33 @@ def configure(args, parser): options += ' --token' key_type = "personal access token" - print(dedent("""\ - {N}. {BOLD_MAGENTA}Add these lines to your `.travis.yml` file:{RESET} - - env: - global: - # Doctr {key_type} for {deploy_repo} - - secure: "{encrypted_variable}" - - script: - - set -e - - {BOLD_BLACK}{RESET} - - pip install doctr - - doctr deploy {options} {BOLD_BLACK}{RESET} - """.format(options=options, N=N, key_type=key_type, - encrypted_variable=encrypted_variable.decode('utf-8'), - deploy_repo=deploy_repo, BOLD_MAGENTA=BOLD_MAGENTA, - BOLD_BLACK=BOLD_BLACK, RESET=RESET))) + if 'travis' in service: + print(dedent("""\ + {N}. {BOLD_MAGENTA}Add these lines to your `.travis.yml` file:{RESET} + + env: + global: + # Doctr {key_type} for {deploy_repo} + - secure: "{encrypted_variable}" + + script: + - set -e + - {BOLD_BLACK}{RESET} + - pip install doctr + - doctr deploy {options} {BOLD_BLACK}{RESET} + """.format(options=options, N=N, key_type=key_type, + encrypted_variable=encrypted_variable.decode('utf-8'), + deploy_repo=deploy_repo, BOLD_MAGENTA=BOLD_MAGENTA, + BOLD_BLACK=BOLD_BLACK, RESET=RESET))) + else: + print(dedent("""\ + {N}. {BOLD_MAGENTA}Add these lines to your github actions yaml file:{RESET} + + TODO: Write this part. + """.format(options=options, N=N, key_type=key_type, + encrypted_variable=encrypted_variable.decode('utf-8'), + deploy_repo=deploy_repo, BOLD_MAGENTA=BOLD_MAGENTA, + BOLD_BLACK=BOLD_BLACK, RESET=RESET))) print(dedent("""\ Replace the text in {BOLD_BLACK}{RESET} with the relevant diff --git a/doctr/github_actions.py b/doctr/github_actions.py new file mode 100644 index 00000000..59fd43c3 --- /dev/null +++ b/doctr/github_actions.py @@ -0,0 +1,74 @@ +import os +import shlex +import sys + +from .ci import CI + +class GitHubActions(CI): + def branch(self): + """Get the name of the branch that the PR is from. + + Note that this is not simply ``$TRAVIS_BRANCH``. the ``push`` build will + use the correct branch (the branch that the PR is from) but the ``pr`` + build will use the _target_ of the PR (usually master). So instead, we ask + for ``$TRAVIS_PULL_REQUEST_BRANCH`` if it's a PR build, and + ``$TRAVIS_BRANCH`` if it's a push build. + """ + head_ref = os.environ.get("GITHUB_HEAD_REF", "") + if head_ref: + return head_ref + GITHUB_REF = os.environ.get("GITHUB_REF", "") + if "refs/heads/" in GITHUB_REF: + return GITHUB_REF[len('refs/heads/'):] + return "" + + def tag(self): + GITHUB_REF = os.environ.get("GITHUB_REF", "") + if "refs/tags/" in GITHUB_REF: + return GITHUB_REF[len('refs/tags/'):] + return "" + + def repo_slug(self): + return os.environ["GITHUB_REPOSITORY"] + + def is_pull_request(self): + return os.environ.get("GITHUB_EVENT_NAME", "") != "pull_request" + + + def commit_message(self): + GITHUB_RUN_ID = os.environ.get("GITHUB_RUN_ID", "") + GITHUB_RUN_NUMBER = os.environ.get("GITHUB_RUN_NUMBER", "") + branch = self.branch() or "" + tag = self.tag() + typ = 'branch' if branch else 'tag' + branch_or_tag = branch or tag + GITHUB_SHA = os.environ.get("GITHUB_SHA", "") + repo_slug = self.repo_slug + web_url = (os.environ.get("GITHUB_SERVER_URL", "") + '/' + + repo_slug + '/runs/' + GITHUB_RUN_NUMBER) + + DOCTR_COMMAND = ' '.join(map(shlex.quote, sys.argv)) + + commit_message = """\ + Update docs after building GitHub Actions build {GITHUB_RUN_ID} ({GITHUB_RUN_NUMBER}) of {repo_slug} + + The docs were built from the {typ} '{branch_or_tag}' against the commit + {GITHUB_SHA}. + + The GitHub Actions build that generated this commit is at + {web_url}. + + The doctr command that was run is + + {DOCTR_COMMAND} + """.format( + GITHUB_RUN_ID=GITHUB_RUN_ID, + GITHUB_RUN_NUMBER=GITHUB_RUN_NUMBER, + repo_slug=repo_slug, + typ=typ, + branch_or_tag=branch_or_tag, + GITHUB_SHA=GITHUB_SHA, + web_url=web_url, + DOCTR_COMMAND=DOCTR_COMMAND, + ) + return commit_message diff --git a/doctr/local.py b/doctr/local.py index be949d0e..66fc1207 100644 --- a/doctr/local.py +++ b/doctr/local.py @@ -389,14 +389,18 @@ def generate_ssh_key(): return private_key, public_key def check_repo_exists(deploy_repo, service='github', *, auth=None, - headers=None, ask=False): + headers=None, ask=False, raise_=True): """ Checks that the repository exists on GitHub. This should be done before attempting generate a key to deploy to that repo. - Raises ``RuntimeError`` if the repo is not valid. + 'service' should be one of 'github', 'travis', 'travis-ci.org', + 'travis-ci.com', or 'github actions'. + + If the repo is not valid and raise_ is True, raises ``RuntimeError``, + otherwise returns False. Returns a dictionary with the following keys: @@ -425,8 +429,10 @@ def check_repo_exists(deploy_repo, service='github', *, auth=None, elif service == 'travis-ci.org': REPO_URL = 'https://api.travis-ci.org/repo/{user}%2F{repo}' headers = {**headers, **Travis_APIv3} + elif service == 'github actions': + REPO_URL = 'https://api.github.com/repos/{owner}/{repo}/actions/permissions' else: - raise RuntimeError('Invalid service specified for repo check (should be one of {"github", "travis", "travis-ci.com", "travis-ci.org"}') + raise RuntimeError('Invalid service specified for repo check (should be one of {"github", "travis", "travis-ci.com", "travis-ci.org", "github actions"}') wiki = False if repo.endswith('.wiki') and service == 'github': @@ -446,7 +452,12 @@ def _try(url): r = _try(REPO_URL.format(user=urllib.parse.quote(user), repo=urllib.parse.quote(repo))) - r_active = r and (service == 'github' or r.json().get('active', False)) + if not r: + r_active = False + elif 'travis' in service: + r_active = r.json().get('active', False) + elif service == 'github actions': + r_active = r.json().get('enabled', False) if service == 'travis': REPO_URL = 'https://api.travis-ci.org/repo/{user}%2F{repo}' @@ -455,6 +466,8 @@ def _try(url): r_org_active = r_org and r_org.json().get('active', False) if not r_active: if not r_org_active: + if not raise_: + return False raise RuntimeError('"{user}/{repo}" not found on travis-ci.org or travis-ci.com'.format(user=user, repo=repo)) r = r_org r_active = r_org_active @@ -483,11 +496,14 @@ def _try(url): if not r_active: msg = '' if auth else '. If the repo is private, then you need to authenticate.' + if not raise_: + return False raise RuntimeError('"{user}/{repo}" not found on {service}{msg}'.format(user=user, repo=repo, service=service, msg=msg)) + # TODO: Handle private repos for GitHub actions private = r.json().get('private', False) if wiki and not private: @@ -495,6 +511,8 @@ def _try(url): p = subprocess.run(['git', 'ls-remote', '-h', 'https://github.com/{user}/{repo}.wiki'.format( user=user, repo=repo)], stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False) if p.stderr or p.returncode: + if not raise_: + return False raise RuntimeError('Wiki not found. Please create a wiki') return { diff --git a/doctr/travis.py b/doctr/travis.py index 3966bff5..05ccc259 100644 --- a/doctr/travis.py +++ b/doctr/travis.py @@ -4,20 +4,12 @@ import os import shlex -import shutil import subprocess import sys -import glob import re -import pathlib -import tempfile -import time import requests -from cryptography.fernet import Fernet - -from .common import red, blue, yellow from .ci import CI class Travis(CI): @@ -38,105 +30,43 @@ def branch(self): def tag(self): return os.environ.get("TRAVIS_TAG", "") - def is_fork(self): - # Check if the repo is a fork - TRAVIS_REPO_SLUG = os.environ["TRAVIS_REPO_SLUG"] - REPO_URL = 'https://api.github.com/repos/{slug}' - r = requests.get(REPO_URL.format(slug=TRAVIS_REPO_SLUG)) - fork = r.json().get('fork', False) - return fork + def repo_slug(self): + return os.environ["TRAVIS_REPO_SLUG"] def is_pull_request(self): return os.environ.get("TRAVIS_PULL_REQUEST", "") != "false" -def set_git_user_email(): - """ - Set global user and email for git user if not already present on system - """ - username = subprocess.run(shlex.split('git config user.name'), stdout=subprocess.PIPE).stdout.strip().decode('utf-8') - if not username or username == "Travis CI User": - run(['git', 'config', '--global', 'user.name', "Doctr (Travis CI)"]) - else: - print("Not setting git user name, as it's already set to %r" % username) - - email = subprocess.run(shlex.split('git config user.email'), stdout=subprocess.PIPE).stdout.strip().decode('utf-8') - if not email or email == "travis@example.org": - # We need a dummy email or git will fail. We use this one as per - # https://help.github.com/articles/keeping-your-email-address-private/. - run(['git', 'config', '--global', 'user.email', 'drdoctr@users.noreply.github.com']) - else: - print("Not setting git user email, as it's already set to %r" % email) - -# Here is the logic to get the Travis job number, to only run commit_docs in -# the right build. -# -# TRAVIS_JOB_NUMBER = os.environ.get("TRAVIS_JOB_NUMBER", '') -# ACTUAL_TRAVIS_JOB_NUMBER = TRAVIS_JOB_NUMBER.split('.')[1] - -def commit_message(): - TRAVIS_BUILD_NUMBER = os.environ.get("TRAVIS_BUILD_NUMBER", "") - TRAVIS_BRANCH = os.environ.get("TRAVIS_BRANCH", "") - TRAVIS_COMMIT = os.environ.get("TRAVIS_COMMIT", "") - TRAVIS_REPO_SLUG = os.environ.get("TRAVIS_REPO_SLUG", "") - TRAVIS_JOB_WEB_URL = os.environ.get("TRAVIS_JOB_WEB_URL", "") - TRAVIS_TAG = os.environ.get("TRAVIS_TAG", "") - branch = "tag" if TRAVIS_TAG else "branch" - - DOCTR_COMMAND = ' '.join(map(shlex.quote, sys.argv)) - - commit_message = """\ -Update docs after building Travis build {TRAVIS_BUILD_NUMBER} of -{TRAVIS_REPO_SLUG} - -The docs were built from the {branch} '{TRAVIS_BRANCH}' against the commit -{TRAVIS_COMMIT}. - -The Travis build that generated this commit is at -{TRAVIS_JOB_WEB_URL}. - -The doctr command that was run is - - {DOCTR_COMMAND} -""".format( - branch=branch, - TRAVIS_BUILD_NUMBER=TRAVIS_BUILD_NUMBER, - TRAVIS_BRANCH=TRAVIS_BRANCH, - TRAVIS_COMMIT=TRAVIS_COMMIT, - TRAVIS_REPO_SLUG=TRAVIS_REPO_SLUG, - TRAVIS_JOB_WEB_URL=TRAVIS_JOB_WEB_URL, - DOCTR_COMMAND=DOCTR_COMMAND, - ) - return commit_message - -def determine_push_rights(*, branch_whitelist, TRAVIS_BRANCH, - TRAVIS_PULL_REQUEST, TRAVIS_TAG, build_tags, fork): - """Check if Travis is running on ``master`` (or a whitelisted branch) to - determine if we can/should push the docs to the deploy repo - """ - canpush = True - - if TRAVIS_TAG: - if not build_tags: - print("The docs are not pushed on tag builds. To push on future tag builds, use --build-tags") - return build_tags - - if not any([re.compile(x).match(TRAVIS_BRANCH) for x in branch_whitelist]): - print("The docs are only pushed to gh-pages from master. To allow pushing from " - "a non-master branch, use the --no-require-master flag", file=sys.stderr) - print("This is the {TRAVIS_BRANCH} branch".format(TRAVIS_BRANCH=TRAVIS_BRANCH), file=sys.stderr) - canpush = False - - if TRAVIS_PULL_REQUEST != "false": - print("The website and docs are not pushed to gh-pages on pull requests", file=sys.stderr) - canpush = False - - if fork: - print("The website and docs are not pushed to gh-pages on fork builds.", file=sys.stderr) - canpush = False - - if last_commit_by_doctr(): - print(red("The last commit on this branch was pushed by doctr. Not pushing to " - "avoid an infinite build-loop."), file=sys.stderr) - canpush = False - - return canpush + def commit_message(self): + TRAVIS_BUILD_NUMBER = os.environ.get("TRAVIS_BUILD_NUMBER", "") + TRAVIS_BRANCH = os.environ.get("TRAVIS_BRANCH", "") + TRAVIS_COMMIT = os.environ.get("TRAVIS_COMMIT", "") + TRAVIS_REPO_SLUG = os.environ.get("TRAVIS_REPO_SLUG", "") + TRAVIS_JOB_WEB_URL = os.environ.get("TRAVIS_JOB_WEB_URL", "") + TRAVIS_TAG = os.environ.get("TRAVIS_TAG", "") + branch = "tag" if TRAVIS_TAG else "branch" + + DOCTR_COMMAND = ' '.join(map(shlex.quote, sys.argv)) + + commit_message = """\ + Update docs after building Travis build {TRAVIS_BUILD_NUMBER} of + {TRAVIS_REPO_SLUG} + + The docs were built from the {branch} '{TRAVIS_BRANCH}' against the commit + {TRAVIS_COMMIT}. + + The Travis build that generated this commit is at + {TRAVIS_JOB_WEB_URL}. + + The doctr command that was run is + + {DOCTR_COMMAND} + """.format( + branch=branch, + TRAVIS_BUILD_NUMBER=TRAVIS_BUILD_NUMBER, + TRAVIS_BRANCH=TRAVIS_BRANCH, + TRAVIS_COMMIT=TRAVIS_COMMIT, + TRAVIS_REPO_SLUG=TRAVIS_REPO_SLUG, + TRAVIS_JOB_WEB_URL=TRAVIS_JOB_WEB_URL, + DOCTR_COMMAND=DOCTR_COMMAND, + ) + return commit_message From c43a4680bdb94c3733f5932226d3e15f793f091f Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 12 Nov 2020 14:11:27 -0700 Subject: [PATCH 7/8] Fix pyflakes errors --- doctr/__main__.py | 7 +++---- doctr/ci.py | 4 ++-- doctr/travis.py | 4 ---- 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/doctr/__main__.py b/doctr/__main__.py index 356a9375..9903d390 100644 --- a/doctr/__main__.py +++ b/doctr/__main__.py @@ -621,10 +621,9 @@ def configure(args, parser): {N}. {BOLD_MAGENTA}Add these lines to your github actions yaml file:{RESET} TODO: Write this part. - """.format(options=options, N=N, key_type=key_type, - encrypted_variable=encrypted_variable.decode('utf-8'), - deploy_repo=deploy_repo, BOLD_MAGENTA=BOLD_MAGENTA, - BOLD_BLACK=BOLD_BLACK, RESET=RESET))) + """.format(N=N, + BOLD_MAGENTA=BOLD_MAGENTA, + RESET=RESET))) print(dedent("""\ Replace the text in {BOLD_BLACK}{RESET} with the relevant diff --git a/doctr/ci.py b/doctr/ci.py index 2fedb238..c9293240 100644 --- a/doctr/ci.py +++ b/doctr/ci.py @@ -259,9 +259,9 @@ def create_deploy_branch(self, deploy_branch, push=True): Return True if ``deploy_branch`` was created, False if not. """ - if not deploy_branch_exists(deploy_branch): + if not self.deploy_branch_exists(deploy_branch): print("Creating {} branch on doctr_remote".format(deploy_branch)) - clear_working_branch() + self.clear_working_branch() run(['git', 'checkout', '--orphan', DOCTR_WORKING_BRANCH]) # delete everything in the new ref. this is non-destructive to existing # refs/branches, etc... diff --git a/doctr/travis.py b/doctr/travis.py index 05ccc259..0cdf8108 100644 --- a/doctr/travis.py +++ b/doctr/travis.py @@ -4,11 +4,7 @@ import os import shlex -import subprocess import sys -import re - -import requests from .ci import CI From b1a2a0f3f44850161d186059b14aec4afab4efcb Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 12 Nov 2020 15:47:29 -0700 Subject: [PATCH 8/8] Fix docs build --- doctr/__main__.py | 6 +++--- doctr/local.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/doctr/__main__.py b/doctr/__main__.py index 9903d390..db7d9f33 100644 --- a/doctr/__main__.py +++ b/doctr/__main__.py @@ -185,9 +185,9 @@ def get_parser(config=None): compatibility.""") deploy_parser_add_argument('--exclude', nargs='+', default=(), help="""Files and directories from --built-docs that are not copied.""") - deploy_parser.add_help('--ci', default=None, help="""The CI system that - doctr is being run on. Should be one of ['travis', 'github-actions']. The - default is to detect automatically.""") + deploy_parser.add_argument('--ci', default=None, help="""The CI system + that doctr is being run on. Should be one of ['travis', + 'github-actions']. The default is to detect automatically.""") if config: print('Warning, The following options in `.travis.yml` were not recognized:\n%s' % json.dumps(config, indent=2)) diff --git a/doctr/local.py b/doctr/local.py index 66fc1207..051b0ed4 100644 --- a/doctr/local.py +++ b/doctr/local.py @@ -399,7 +399,7 @@ def check_repo_exists(deploy_repo, service='github', *, auth=None, 'service' should be one of 'github', 'travis', 'travis-ci.org', 'travis-ci.com', or 'github actions'. - If the repo is not valid and raise_ is True, raises ``RuntimeError``, + If the repo is not valid and ``raise_`` is True, raises ``RuntimeError``, otherwise returns False. Returns a dictionary with the following keys: