From dd3be612e3df1450de40a74fadc6df3c2885e9b1 Mon Sep 17 00:00:00 2001 From: Marc Cornellà Date: Wed, 20 Dec 2023 10:21:31 +0100 Subject: ci(dependencies): add automation for updating external dependencies (#12109) --- .github/workflows/dependencies/updater.py | 450 ++++++++++++++++++++++++++++++ 1 file changed, 450 insertions(+) create mode 100644 .github/workflows/dependencies/updater.py (limited to '.github/workflows/dependencies/updater.py') diff --git a/.github/workflows/dependencies/updater.py b/.github/workflows/dependencies/updater.py new file mode 100644 index 000000000..5af19d14c --- /dev/null +++ b/.github/workflows/dependencies/updater.py @@ -0,0 +1,450 @@ +import os +import subprocess +import sys +import requests +import shutil +import yaml +from copy import deepcopy +from typing import Optional, TypedDict + +# Get TMP_DIR variable from environment +TMP_DIR = os.path.join(os.environ.get("TMP_DIR", "/tmp"), "ohmyzsh") +# Relative path to dependencies.yml file +DEPS_YAML_FILE = ".github/dependencies.yml" +# Dry run flag +DRY_RUN = os.environ.get("DRY_RUN", "0") == "1" + +import timeit +class CodeTimer: + def __init__(self, name=None): + self.name = " '" + name + "'" if name else '' + + def __enter__(self): + self.start = timeit.default_timer() + + def __exit__(self, exc_type, exc_value, traceback): + self.took = (timeit.default_timer() - self.start) * 1000.0 + print('Code block' + self.name + ' took: ' + str(self.took) + ' ms') + + +### YAML representation +def str_presenter(dumper, data): + """ + Configures yaml for dumping multiline strings + Ref: https://stackoverflow.com/a/33300001 + """ + if len(data.splitlines()) > 1: # check for multiline string + return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='|') + return dumper.represent_scalar('tag:yaml.org,2002:str', data) + +yaml.add_representer(str, str_presenter) +yaml.representer.SafeRepresenter.add_representer(str, str_presenter) + + +# Types +class DependencyDict(TypedDict): + repo: str + branch: str + version: str + precopy: Optional[str] + postcopy: Optional[str] + +class DependencyYAML(TypedDict): + dependencies: dict[str, DependencyDict] + +class UpdateStatus(TypedDict): + has_updates: bool + version: Optional[str] + compare_url: Optional[str] + head_ref: Optional[str] + head_url: Optional[str] + + +class CommandRunner: + class Exception(Exception): + def __init__(self, message, returncode, stage, stdout, stderr): + super().__init__(message) + self.returncode = returncode + self.stage = stage + self.stdout = stdout + self.stderr = stderr + + @staticmethod + def run_or_fail(command: list[str], stage: str, *args, **kwargs): + if DRY_RUN and command[0] == "gh": + command.insert(0, "echo") + + result = subprocess.run(command, *args, capture_output=True, **kwargs) + + if result.returncode != 0: + raise CommandRunner.Exception( + f"{stage} command failed with exit code {result.returncode}", returncode=result.returncode, + stage=stage, + stdout=result.stdout.decode("utf-8"), + stderr=result.stderr.decode("utf-8") + ) + + return result + + +class DependencyStore: + store: DependencyYAML = { + "dependencies": {} + } + + @staticmethod + def set(data: DependencyYAML): + DependencyStore.store = data + + @staticmethod + def update_dependency_version(path: str, version: str) -> DependencyYAML: + with CodeTimer(f"store deepcopy: {path}"): + store_copy = deepcopy(DependencyStore.store) + + dependency = store_copy["dependencies"].get(path, {}) + dependency["version"] = version + store_copy["dependencies"][path] = dependency + + return store_copy + + @staticmethod + def write_store(file: str, data: DependencyYAML): + with open(file, "w") as yaml_file: + yaml.safe_dump(data, yaml_file, sort_keys=False) + + +class Dependency: + def __init__(self, path: str, values: DependencyDict): + self.path = path + self.values = values + + self.name: str = "" + self.desc: str = "" + self.kind: str = "" + + match path.split("/"): + case ["plugins", name]: + self.name = name + self.kind = "plugin" + self.desc = f"{name} plugin" + case ["themes", name]: + self.name = name.replace(".zsh-theme", "") + self.kind = "theme" + self.desc = f"{self.name} theme" + case _: + self.name = self.desc = path + + def __str__(self): + output: str = "" + for key in DependencyDict.__dict__['__annotations__'].keys(): + if key not in self.values: + output += f"{key}: None\n" + continue + + value = self.values[key] + if "\n" not in value: + output += f"{key}: {value}\n" + else: + output += f"{key}:\n " + output += value.replace("\n", "\n ", value.count("\n") - 1) + return output + + def update_or_notify(self): + # Print dependency settings + print(f"Processing {self.desc}...", file=sys.stderr) + print(self, file=sys.stderr) + + # Check for updates + repo = self.values["repo"] + remote_branch = self.values["branch"] + version = self.values["version"] + is_tag = version.startswith("tag:") + + try: + with CodeTimer(f"update check: {repo}"): + if is_tag: + status = GitHub.check_newer_tag(repo, version.replace("tag:", "")) + else: + status = GitHub.check_updates(repo, remote_branch, version) + + if status["has_updates"]: + short_sha = status["head_ref"][:8] + new_version = status["version"] if is_tag else short_sha + + try: + # Create new branch + branch = Git.create_branch(self.path, new_version) + + # Update dependencies.yml file + self.__update_yaml(f"tag:{new_version}" if is_tag else status["version"]) + + # Update dependency files + self.__apply_upstream_changes() + + # Add all changes and commit + Git.add_and_commit(self.name, short_sha) + + # Push changes to remote + Git.push(branch) + + # Create GitHub PR + GitHub.create_pr( + branch, + f"feat({self.name}): update to version {new_version}", + f"""## Description + +Update for **{self.desc}**: update to version [{new_version}]({status['head_url']}). +Check out the [list of changes]({status['compare_url']}). +""" + ) + + # Clean up repository + Git.clean_repo() + except (CommandRunner.Exception, shutil.Error) as e: + # Handle exception on automatic update + match type(e): + case CommandRunner.Exception: + # Print error message + print(f"Error running {e.stage} command: {e.returncode}", file=sys.stderr) + print(e.stderr, file=sys.stderr) + case shutil.Error: + print(f"Error copying files: {e}", file=sys.stderr) + + try: + Git.clean_repo() + except CommandRunner.Exception as e: + print(f"Error reverting repository to clean state: {e}", file=sys.stderr) + sys.exit(1) + + # Create a GitHub issue to notify maintainer + title = f"{self.path}: update to {new_version}" + body = ( + f"""## Description + +There is a new version of `{self.name}` {self.kind} available. + +New version: [{new_version}]({status['head_url']}) +Check out the [list of changes]({status['compare_url']}). +""" + ) + + print(f"Creating GitHub issue", file=sys.stderr) + print(f"{title}\n\n{body}", file=sys.stderr) + GitHub.create_issue(title, body) + except Exception as e: + print(e, file=sys.stderr) + + def __update_yaml(self, new_version: str) -> None: + dep_yaml = DependencyStore.update_dependency_version(self.path, new_version) + DependencyStore.write_store(DEPS_YAML_FILE, dep_yaml) + + def __apply_upstream_changes(self) -> None: + # Patterns to ignore in copying files from upstream repo + GLOBAL_IGNORE = [ + ".git", + ".github", + ".gitignore" + ] + + path = os.path.abspath(self.path) + precopy = self.values.get("precopy") + postcopy = self.values.get("postcopy") + + repo = self.values["repo"] + branch = self.values["branch"] + remote_url = f"https://github.com/{repo}.git" + repo_dir = os.path.join(TMP_DIR, repo) + + # Clone repository + Git.clone(remote_url, branch, repo_dir, reclone=True) + + # Run precopy on tmp repo + if precopy is not None: + print("Running precopy script:", end="\n ", file=sys.stderr) + print(precopy.replace("\n", "\n ", precopy.count("\n") - 1), file=sys.stderr) + CommandRunner.run_or_fail(["bash", "-c", precopy], cwd=repo_dir, stage="Precopy") + + # Copy files from upstream repo + print(f"Copying files from {repo_dir} to {path}", file=sys.stderr) + shutil.copytree(repo_dir, path, dirs_exist_ok=True, ignore=shutil.ignore_patterns(*GLOBAL_IGNORE)) + + # Run postcopy on our repository + if postcopy is not None: + print("Running postcopy script:", end="\n ", file=sys.stderr) + print(postcopy.replace("\n", "\n ", postcopy.count("\n") - 1), file=sys.stderr) + CommandRunner.run_or_fail(["bash", "-c", postcopy], cwd=path, stage="Postcopy") + + +class Git: + default_branch = "master" + + @staticmethod + def clone(remote_url: str, branch: str, repo_dir: str, reclone=False): + # If repo needs to be fresh + if reclone and os.path.exists(repo_dir): + shutil.rmtree(repo_dir) + + # Clone repo in tmp directory and checkout branch + if not os.path.exists(repo_dir): + print(f"Cloning {remote_url} to {repo_dir} and checking out {branch}", file=sys.stderr) + CommandRunner.run_or_fail(["git", "clone", "--depth=1", "-b", branch, remote_url, repo_dir], stage="Clone") + + @staticmethod + def create_branch(path: str, version: str): + # Get current branch name + result = CommandRunner.run_or_fail(["git", "rev-parse", "--abbrev-ref", "HEAD"], stage="GetDefaultBranch") + Git.default_branch = result.stdout.decode("utf-8").strip() + + # Create new branch and return created branch name + branch_name = f"update/{path}/{version}" + CommandRunner.run_or_fail(["git", "checkout", "-b", branch_name], stage="CreateBranch") + return branch_name + + @staticmethod + def add_and_commit(scope: str, version: str): + user_name = "ohmyzsh" + user_email = "bot@ohmyz.sh" + + # Add all files to git staging + CommandRunner.run_or_fail(["git", "add", "-A", "-v"], stage="AddFiles") + + # Reset environment and git config + clean_env = os.environ.copy() + clean_env["LANG"]="C.UTF-8" + clean_env["GIT_CONFIG_GLOBAL"]="/dev/null" + clean_env["GIT_CONFIG_NOSYSTEM"]="1" + + # Commit with settings above + CommandRunner.run_or_fail([ + "git", + "-c", f"user.name={user_name}", + "-c", f"user.email={user_email}", + "commit", + "-m", f"feat({scope}): update to {version}" + ], stage="CreateCommit", env=clean_env) + + @staticmethod + def push(branch: str): + CommandRunner.run_or_fail(["git", "push", "-u", "origin", branch], stage="PushBranch") + + @staticmethod + def clean_repo(): + CommandRunner.run_or_fail(["git", "reset", "--hard", "HEAD"], stage="ResetRepository") + CommandRunner.run_or_fail(["git", "checkout", Git.default_branch], stage="CheckoutDefaultBranch") + + +class GitHub: + @staticmethod + def check_newer_tag(repo, current_tag) -> UpdateStatus: + # GET /repos/:owner/:repo/git/refs/tags + url = f"https://api.github.com/repos/{repo}/git/refs/tags" + + # Send a GET request to the GitHub API + response = requests.get(url) + + # If the request was successful + if response.status_code == 200: + # Parse the JSON response + data = response.json() + + if len(data) == 0: + return { + "has_updates": False, + } + + latest_ref = data[-1] + latest_tag = latest_ref["ref"].replace("refs/tags/", "") + + if latest_tag == current_tag: + return { + "has_updates": False, + } + + return { + "has_updates": True, + "version": latest_tag, + "compare_url": f"https://github.com/{repo}/compare/{current_tag}...{latest_tag}", + "head_ref": latest_ref["object"]["sha"], + "head_url": f"https://github.com/{repo}/releases/tag/{latest_tag}", + } + else: + # If the request was not successful, raise an exception + raise Exception(f"GitHub API request failed with status code {response.status_code}: {response.json()}") + + @staticmethod + def check_updates(repo, branch, version) -> UpdateStatus: + # TODO: add support for semver updating (based on tags) + # Check if upstream github repo has a new version + # GitHub API URL for comparing two commits + url = f"https://api.github.com/repos/{repo}/compare/{version}...{branch}" + + # Send a GET request to the GitHub API + response = requests.get(url) + + # If the request was successful + if response.status_code == 200: + # Parse the JSON response + data = response.json() + + # If the base is behind the head, there is a newer version + has_updates = data["status"] != "identical" + + if not has_updates: + return { + "has_updates": False, + } + + return { + "has_updates": data["status"] != "identical", + "version": data["commits"][-1]["sha"], + "compare_url": data["permalink_url"], + "head_ref": data["commits"][-1]["sha"], + "head_url": data["commits"][-1]["html_url"] + } + else: + # If the request was not successful, raise an exception + raise Exception(f"GitHub API request failed with status code {response.status_code}: {response.json()}") + + @staticmethod + def create_issue(title: str, body: str) -> None: + cmd = [ + "gh", + "issue", + "create", + "-t", title, + "-b", body + ] + CommandRunner.run_or_fail(cmd, stage="CreateIssue") + + @staticmethod + def create_pr(branch: str, title: str, body: str) -> None: + cmd = [ + "gh", + "pr", + "create", + "-B", Git.default_branch, + "-H", branch, + "-t", title, + "-b", body + ] + CommandRunner.run_or_fail(cmd, stage="CreatePullRequest") + + +def main(): + # Load the YAML file + with open(DEPS_YAML_FILE, "r") as yaml_file: + data: DependencyYAML = yaml.safe_load(yaml_file) + + if "dependencies" not in data: + raise Exception(f"dependencies.yml not properly formatted") + + # Cache YAML version + DependencyStore.set(data) + + dependencies = data["dependencies"] + for path in dependencies: + dependency = Dependency(path, dependencies[path]) + dependency.update_or_notify() + +if __name__ == "__main__": + main() -- cgit v1.2.3-70-g09d2 From 46b24d409909cdc00ac441448c0aa31cc35d12c9 Mon Sep 17 00:00:00 2001 From: Carlo Sala Date: Thu, 28 Dec 2023 21:20:28 +0100 Subject: ci(dependencies): fix some envs and add requirements --- .github/workflows/dependencies.yml | 11 +++++++---- .github/workflows/dependencies/requirements.txt | 2 ++ .github/workflows/dependencies/updater.py | 4 ++-- 3 files changed, 11 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/dependencies/requirements.txt (limited to '.github/workflows/dependencies/updater.py') diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yml index aab740126..2e2217e1c 100644 --- a/.github/workflows/dependencies.yml +++ b/.github/workflows/dependencies.yml @@ -8,19 +8,22 @@ jobs: check: name: Check for updates runs-on: ubuntu-latest + if: github.repository == 'ohmyzsh/ohmyzsh' steps: - name: Checkout - if: github.repository == 'ohmyzsh/ohmyzsh' uses: actions/checkout@v4 - name: Authenticate as @ohmyzsh + id: generate_token uses: ohmyzsh/github-app-token@v2 with: app_id: ${{ secrets.OHMYZSH_APP_ID }} private_key: ${{ secrets.OHMYZSH_APP_PRIVATE_KEY }} - name: Process dependencies env: - GITHUB_TOKEN: ${{ steps.generate_token.outputs.token }} - TMP_DIR: ${{ env.RUNNER_TEMP }} + GH_TOKEN: ${{ steps.generate_token.outputs.token }} + GIT_APP_NAME: ohmyzsh[bot] + GIT_APP_EMAIL: 54982679+ohmyzsh[bot]@users.noreply.github.com + TMP_DIR: ${{ runner.temp }} run: | - gh auth login --with-token <<< "${GITHUB_TOKEN}" + pip install -r .github/workflows/dependencies/requirements.txt python3 .github/workflows/dependencies/updater.py diff --git a/.github/workflows/dependencies/requirements.txt b/.github/workflows/dependencies/requirements.txt new file mode 100644 index 000000000..3c4c149ea --- /dev/null +++ b/.github/workflows/dependencies/requirements.txt @@ -0,0 +1,2 @@ +PyYAML~=6.0.1 +requests~=2.31.0 diff --git a/.github/workflows/dependencies/updater.py b/.github/workflows/dependencies/updater.py index 5af19d14c..f54d316f9 100644 --- a/.github/workflows/dependencies/updater.py +++ b/.github/workflows/dependencies/updater.py @@ -302,8 +302,8 @@ class Git: @staticmethod def add_and_commit(scope: str, version: str): - user_name = "ohmyzsh" - user_email = "bot@ohmyz.sh" + user_name = os.environ.get("GIT_APP_NAME") + user_email = os.environ.get("GIT_APP_EMAIL") # Add all files to git staging CommandRunner.run_or_fail(["git", "add", "-A", "-v"], stage="AddFiles") -- cgit v1.2.3-70-g09d2 From 13c8a10e39d60c078193d403b253270cdc4abc80 Mon Sep 17 00:00:00 2001 From: Carlo Sala Date: Thu, 9 May 2024 17:20:09 +0200 Subject: style(dependencies): run `ruff` formatter --- .editorconfig | 3 + .github/workflows/dependencies/updater.py | 788 ++++++++++++++++-------------- 2 files changed, 420 insertions(+), 371 deletions(-) (limited to '.github/workflows/dependencies/updater.py') diff --git a/.editorconfig b/.editorconfig index b5321de59..b349bcc42 100644 --- a/.editorconfig +++ b/.editorconfig @@ -6,3 +6,6 @@ insert_final_newline = true charset = utf-8 indent_size = 2 indent_style = space + +[*.py] +indent_size = 4 diff --git a/.github/workflows/dependencies/updater.py b/.github/workflows/dependencies/updater.py index f54d316f9..765cb26fe 100644 --- a/.github/workflows/dependencies/updater.py +++ b/.github/workflows/dependencies/updater.py @@ -1,12 +1,14 @@ import os +import shutil import subprocess import sys -import requests -import shutil -import yaml +import timeit from copy import deepcopy from typing import Optional, TypedDict +import requests +import yaml + # Get TMP_DIR variable from environment TMP_DIR = os.path.join(os.environ.get("TMP_DIR", "/tmp"), "ohmyzsh") # Relative path to dependencies.yml file @@ -14,28 +16,29 @@ DEPS_YAML_FILE = ".github/dependencies.yml" # Dry run flag DRY_RUN = os.environ.get("DRY_RUN", "0") == "1" -import timeit + class CodeTimer: - def __init__(self, name=None): - self.name = " '" + name + "'" if name else '' + def __init__(self, name=None): + self.name = " '" + name + "'" if name else "" - def __enter__(self): - self.start = timeit.default_timer() + def __enter__(self): + self.start = timeit.default_timer() - def __exit__(self, exc_type, exc_value, traceback): - self.took = (timeit.default_timer() - self.start) * 1000.0 - print('Code block' + self.name + ' took: ' + str(self.took) + ' ms') + def __exit__(self, exc_type, exc_value, traceback): + self.took = (timeit.default_timer() - self.start) * 1000.0 + print("Code block" + self.name + " took: " + str(self.took) + " ms") ### YAML representation def str_presenter(dumper, data): - """ - Configures yaml for dumping multiline strings - Ref: https://stackoverflow.com/a/33300001 - """ - if len(data.splitlines()) > 1: # check for multiline string - return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='|') - return dumper.represent_scalar('tag:yaml.org,2002:str', data) + """ + Configures yaml for dumping multiline strings + Ref: https://stackoverflow.com/a/33300001 + """ + if len(data.splitlines()) > 1: # check for multiline string + return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|") + return dumper.represent_scalar("tag:yaml.org,2002:str", data) + yaml.add_representer(str, str_presenter) yaml.representer.SafeRepresenter.add_representer(str, str_presenter) @@ -43,408 +46,451 @@ yaml.representer.SafeRepresenter.add_representer(str, str_presenter) # Types class DependencyDict(TypedDict): - repo: str - branch: str - version: str - precopy: Optional[str] - postcopy: Optional[str] + repo: str + branch: str + version: str + precopy: Optional[str] + postcopy: Optional[str] + class DependencyYAML(TypedDict): - dependencies: dict[str, DependencyDict] + dependencies: dict[str, DependencyDict] + class UpdateStatus(TypedDict): - has_updates: bool - version: Optional[str] - compare_url: Optional[str] - head_ref: Optional[str] - head_url: Optional[str] + has_updates: bool + version: Optional[str] + compare_url: Optional[str] + head_ref: Optional[str] + head_url: Optional[str] class CommandRunner: - class Exception(Exception): - def __init__(self, message, returncode, stage, stdout, stderr): - super().__init__(message) - self.returncode = returncode - self.stage = stage - self.stdout = stdout - self.stderr = stderr + class Exception(Exception): + def __init__(self, message, returncode, stage, stdout, stderr): + super().__init__(message) + self.returncode = returncode + self.stage = stage + self.stdout = stdout + self.stderr = stderr - @staticmethod - def run_or_fail(command: list[str], stage: str, *args, **kwargs): - if DRY_RUN and command[0] == "gh": - command.insert(0, "echo") + @staticmethod + def run_or_fail(command: list[str], stage: str, *args, **kwargs): + if DRY_RUN and command[0] == "gh": + command.insert(0, "echo") - result = subprocess.run(command, *args, capture_output=True, **kwargs) + result = subprocess.run(command, *args, capture_output=True, **kwargs) - if result.returncode != 0: - raise CommandRunner.Exception( - f"{stage} command failed with exit code {result.returncode}", returncode=result.returncode, - stage=stage, - stdout=result.stdout.decode("utf-8"), - stderr=result.stderr.decode("utf-8") - ) + if result.returncode != 0: + raise CommandRunner.Exception( + f"{stage} command failed with exit code {result.returncode}", + returncode=result.returncode, + stage=stage, + stdout=result.stdout.decode("utf-8"), + stderr=result.stderr.decode("utf-8"), + ) - return result + return result class DependencyStore: - store: DependencyYAML = { - "dependencies": {} - } + store: DependencyYAML = {"dependencies": {}} - @staticmethod - def set(data: DependencyYAML): - DependencyStore.store = data + @staticmethod + def set(data: DependencyYAML): + DependencyStore.store = data - @staticmethod - def update_dependency_version(path: str, version: str) -> DependencyYAML: - with CodeTimer(f"store deepcopy: {path}"): - store_copy = deepcopy(DependencyStore.store) + @staticmethod + def update_dependency_version(path: str, version: str) -> DependencyYAML: + with CodeTimer(f"store deepcopy: {path}"): + store_copy = deepcopy(DependencyStore.store) - dependency = store_copy["dependencies"].get(path, {}) - dependency["version"] = version - store_copy["dependencies"][path] = dependency + dependency = store_copy["dependencies"].get(path, {}) + dependency["version"] = version + store_copy["dependencies"][path] = dependency - return store_copy + return store_copy - @staticmethod - def write_store(file: str, data: DependencyYAML): - with open(file, "w") as yaml_file: - yaml.safe_dump(data, yaml_file, sort_keys=False) + @staticmethod + def write_store(file: str, data: DependencyYAML): + with open(file, "w") as yaml_file: + yaml.safe_dump(data, yaml_file, sort_keys=False) class Dependency: - def __init__(self, path: str, values: DependencyDict): - self.path = path - self.values = values - - self.name: str = "" - self.desc: str = "" - self.kind: str = "" - - match path.split("/"): - case ["plugins", name]: - self.name = name - self.kind = "plugin" - self.desc = f"{name} plugin" - case ["themes", name]: - self.name = name.replace(".zsh-theme", "") - self.kind = "theme" - self.desc = f"{self.name} theme" - case _: - self.name = self.desc = path - - def __str__(self): - output: str = "" - for key in DependencyDict.__dict__['__annotations__'].keys(): - if key not in self.values: - output += f"{key}: None\n" - continue - - value = self.values[key] - if "\n" not in value: - output += f"{key}: {value}\n" - else: - output += f"{key}:\n " - output += value.replace("\n", "\n ", value.count("\n") - 1) - return output - - def update_or_notify(self): - # Print dependency settings - print(f"Processing {self.desc}...", file=sys.stderr) - print(self, file=sys.stderr) - - # Check for updates - repo = self.values["repo"] - remote_branch = self.values["branch"] - version = self.values["version"] - is_tag = version.startswith("tag:") - - try: - with CodeTimer(f"update check: {repo}"): - if is_tag: - status = GitHub.check_newer_tag(repo, version.replace("tag:", "")) - else: - status = GitHub.check_updates(repo, remote_branch, version) - - if status["has_updates"]: - short_sha = status["head_ref"][:8] - new_version = status["version"] if is_tag else short_sha + def __init__(self, path: str, values: DependencyDict): + self.path = path + self.values = values + + self.name: str = "" + self.desc: str = "" + self.kind: str = "" + + match path.split("/"): + case ["plugins", name]: + self.name = name + self.kind = "plugin" + self.desc = f"{name} plugin" + case ["themes", name]: + self.name = name.replace(".zsh-theme", "") + self.kind = "theme" + self.desc = f"{self.name} theme" + case _: + self.name = self.desc = path + + def __str__(self): + output: str = "" + for key in DependencyDict.__dict__["__annotations__"].keys(): + if key not in self.values: + output += f"{key}: None\n" + continue + + value = self.values[key] + if "\n" not in value: + output += f"{key}: {value}\n" + else: + output += f"{key}:\n " + output += value.replace("\n", "\n ", value.count("\n") - 1) + return output + + def update_or_notify(self): + # Print dependency settings + print(f"Processing {self.desc}...", file=sys.stderr) + print(self, file=sys.stderr) + + # Check for updates + repo = self.values["repo"] + remote_branch = self.values["branch"] + version = self.values["version"] + is_tag = version.startswith("tag:") try: - # Create new branch - branch = Git.create_branch(self.path, new_version) + with CodeTimer(f"update check: {repo}"): + if is_tag: + status = GitHub.check_newer_tag(repo, version.replace("tag:", "")) + else: + status = GitHub.check_updates(repo, remote_branch, version) - # Update dependencies.yml file - self.__update_yaml(f"tag:{new_version}" if is_tag else status["version"]) + if status["has_updates"]: + short_sha = status["head_ref"][:8] + new_version = status["version"] if is_tag else short_sha - # Update dependency files - self.__apply_upstream_changes() + try: + # Create new branch + branch = Git.create_branch(self.path, new_version) - # Add all changes and commit - Git.add_and_commit(self.name, short_sha) + # Update dependencies.yml file + self.__update_yaml( + f"tag:{new_version}" if is_tag else status["version"] + ) - # Push changes to remote - Git.push(branch) + # Update dependency files + self.__apply_upstream_changes() - # Create GitHub PR - GitHub.create_pr( - branch, - f"feat({self.name}): update to version {new_version}", - f"""## Description + # Add all changes and commit + Git.add_and_commit(self.name, short_sha) + + # Push changes to remote + Git.push(branch) + + # Create GitHub PR + GitHub.create_pr( + branch, + f"feat({self.name}): update to version {new_version}", + f"""## Description Update for **{self.desc}**: update to version [{new_version}]({status['head_url']}). Check out the [list of changes]({status['compare_url']}). -""" - ) - - # Clean up repository - Git.clean_repo() - except (CommandRunner.Exception, shutil.Error) as e: - # Handle exception on automatic update - match type(e): - case CommandRunner.Exception: - # Print error message - print(f"Error running {e.stage} command: {e.returncode}", file=sys.stderr) - print(e.stderr, file=sys.stderr) - case shutil.Error: - print(f"Error copying files: {e}", file=sys.stderr) - - try: - Git.clean_repo() - except CommandRunner.Exception as e: - print(f"Error reverting repository to clean state: {e}", file=sys.stderr) - sys.exit(1) - - # Create a GitHub issue to notify maintainer - title = f"{self.path}: update to {new_version}" - body = ( - f"""## Description +""", + ) + + # Clean up repository + Git.clean_repo() + except (CommandRunner.Exception, shutil.Error) as e: + # Handle exception on automatic update + match type(e): + case CommandRunner.Exception: + # Print error message + print( + f"Error running {e.stage} command: {e.returncode}", + file=sys.stderr, + ) + print(e.stderr, file=sys.stderr) + case shutil.Error: + print(f"Error copying files: {e}", file=sys.stderr) + + try: + Git.clean_repo() + except CommandRunner.Exception as e: + print( + f"Error reverting repository to clean state: {e}", + file=sys.stderr, + ) + sys.exit(1) + + # Create a GitHub issue to notify maintainer + title = f"{self.path}: update to {new_version}" + body = f"""## Description There is a new version of `{self.name}` {self.kind} available. New version: [{new_version}]({status['head_url']}) Check out the [list of changes]({status['compare_url']}). """ - ) - - print(f"Creating GitHub issue", file=sys.stderr) - print(f"{title}\n\n{body}", file=sys.stderr) - GitHub.create_issue(title, body) - except Exception as e: - print(e, file=sys.stderr) - - def __update_yaml(self, new_version: str) -> None: - dep_yaml = DependencyStore.update_dependency_version(self.path, new_version) - DependencyStore.write_store(DEPS_YAML_FILE, dep_yaml) - - def __apply_upstream_changes(self) -> None: - # Patterns to ignore in copying files from upstream repo - GLOBAL_IGNORE = [ - ".git", - ".github", - ".gitignore" - ] - - path = os.path.abspath(self.path) - precopy = self.values.get("precopy") - postcopy = self.values.get("postcopy") - - repo = self.values["repo"] - branch = self.values["branch"] - remote_url = f"https://github.com/{repo}.git" - repo_dir = os.path.join(TMP_DIR, repo) - - # Clone repository - Git.clone(remote_url, branch, repo_dir, reclone=True) - - # Run precopy on tmp repo - if precopy is not None: - print("Running precopy script:", end="\n ", file=sys.stderr) - print(precopy.replace("\n", "\n ", precopy.count("\n") - 1), file=sys.stderr) - CommandRunner.run_or_fail(["bash", "-c", precopy], cwd=repo_dir, stage="Precopy") - - # Copy files from upstream repo - print(f"Copying files from {repo_dir} to {path}", file=sys.stderr) - shutil.copytree(repo_dir, path, dirs_exist_ok=True, ignore=shutil.ignore_patterns(*GLOBAL_IGNORE)) - - # Run postcopy on our repository - if postcopy is not None: - print("Running postcopy script:", end="\n ", file=sys.stderr) - print(postcopy.replace("\n", "\n ", postcopy.count("\n") - 1), file=sys.stderr) - CommandRunner.run_or_fail(["bash", "-c", postcopy], cwd=path, stage="Postcopy") + + print("Creating GitHub issue", file=sys.stderr) + print(f"{title}\n\n{body}", file=sys.stderr) + GitHub.create_issue(title, body) + except Exception as e: + print(e, file=sys.stderr) + + def __update_yaml(self, new_version: str) -> None: + dep_yaml = DependencyStore.update_dependency_version(self.path, new_version) + DependencyStore.write_store(DEPS_YAML_FILE, dep_yaml) + + def __apply_upstream_changes(self) -> None: + # Patterns to ignore in copying files from upstream repo + GLOBAL_IGNORE = [".git", ".github", ".gitignore"] + + path = os.path.abspath(self.path) + precopy = self.values.get("precopy") + postcopy = self.values.get("postcopy") + + repo = self.values["repo"] + branch = self.values["branch"] + remote_url = f"https://github.com/{repo}.git" + repo_dir = os.path.join(TMP_DIR, repo) + + # Clone repository + Git.clone(remote_url, branch, repo_dir, reclone=True) + + # Run precopy on tmp repo + if precopy is not None: + print("Running precopy script:", end="\n ", file=sys.stderr) + print( + precopy.replace("\n", "\n ", precopy.count("\n") - 1), file=sys.stderr + ) + CommandRunner.run_or_fail( + ["bash", "-c", precopy], cwd=repo_dir, stage="Precopy" + ) + + # Copy files from upstream repo + print(f"Copying files from {repo_dir} to {path}", file=sys.stderr) + shutil.copytree( + repo_dir, + path, + dirs_exist_ok=True, + ignore=shutil.ignore_patterns(*GLOBAL_IGNORE), + ) + + # Run postcopy on our repository + if postcopy is not None: + print("Running postcopy script:", end="\n ", file=sys.stderr) + print( + postcopy.replace("\n", "\n ", postcopy.count("\n") - 1), + file=sys.stderr, + ) + CommandRunner.run_or_fail( + ["bash", "-c", postcopy], cwd=path, stage="Postcopy" + ) class Git: - default_branch = "master" - - @staticmethod - def clone(remote_url: str, branch: str, repo_dir: str, reclone=False): - # If repo needs to be fresh - if reclone and os.path.exists(repo_dir): - shutil.rmtree(repo_dir) - - # Clone repo in tmp directory and checkout branch - if not os.path.exists(repo_dir): - print(f"Cloning {remote_url} to {repo_dir} and checking out {branch}", file=sys.stderr) - CommandRunner.run_or_fail(["git", "clone", "--depth=1", "-b", branch, remote_url, repo_dir], stage="Clone") - - @staticmethod - def create_branch(path: str, version: str): - # Get current branch name - result = CommandRunner.run_or_fail(["git", "rev-parse", "--abbrev-ref", "HEAD"], stage="GetDefaultBranch") - Git.default_branch = result.stdout.decode("utf-8").strip() - - # Create new branch and return created branch name - branch_name = f"update/{path}/{version}" - CommandRunner.run_or_fail(["git", "checkout", "-b", branch_name], stage="CreateBranch") - return branch_name - - @staticmethod - def add_and_commit(scope: str, version: str): - user_name = os.environ.get("GIT_APP_NAME") - user_email = os.environ.get("GIT_APP_EMAIL") - - # Add all files to git staging - CommandRunner.run_or_fail(["git", "add", "-A", "-v"], stage="AddFiles") - - # Reset environment and git config - clean_env = os.environ.copy() - clean_env["LANG"]="C.UTF-8" - clean_env["GIT_CONFIG_GLOBAL"]="/dev/null" - clean_env["GIT_CONFIG_NOSYSTEM"]="1" - - # Commit with settings above - CommandRunner.run_or_fail([ - "git", - "-c", f"user.name={user_name}", - "-c", f"user.email={user_email}", - "commit", - "-m", f"feat({scope}): update to {version}" - ], stage="CreateCommit", env=clean_env) - - @staticmethod - def push(branch: str): - CommandRunner.run_or_fail(["git", "push", "-u", "origin", branch], stage="PushBranch") - - @staticmethod - def clean_repo(): - CommandRunner.run_or_fail(["git", "reset", "--hard", "HEAD"], stage="ResetRepository") - CommandRunner.run_or_fail(["git", "checkout", Git.default_branch], stage="CheckoutDefaultBranch") + default_branch = "master" + + @staticmethod + def clone(remote_url: str, branch: str, repo_dir: str, reclone=False): + # If repo needs to be fresh + if reclone and os.path.exists(repo_dir): + shutil.rmtree(repo_dir) + + # Clone repo in tmp directory and checkout branch + if not os.path.exists(repo_dir): + print( + f"Cloning {remote_url} to {repo_dir} and checking out {branch}", + file=sys.stderr, + ) + CommandRunner.run_or_fail( + ["git", "clone", "--depth=1", "-b", branch, remote_url, repo_dir], + stage="Clone", + ) + + @staticmethod + def create_branch(path: str, version: str): + # Get current branch name + result = CommandRunner.run_or_fail( + ["git", "rev-parse", "--abbrev-ref", "HEAD"], stage="GetDefaultBranch" + ) + Git.default_branch = result.stdout.decode("utf-8").strip() + + # Create new branch and return created branch name + branch_name = f"update/{path}/{version}" + CommandRunner.run_or_fail( + ["git", "checkout", "-b", branch_name], stage="CreateBranch" + ) + return branch_name + + @staticmethod + def add_and_commit(scope: str, version: str): + user_name = os.environ.get("GIT_APP_NAME") + user_email = os.environ.get("GIT_APP_EMAIL") + + # Add all files to git staging + CommandRunner.run_or_fail(["git", "add", "-A", "-v"], stage="AddFiles") + + # Reset environment and git config + clean_env = os.environ.copy() + clean_env["LANG"] = "C.UTF-8" + clean_env["GIT_CONFIG_GLOBAL"] = "/dev/null" + clean_env["GIT_CONFIG_NOSYSTEM"] = "1" + + # Commit with settings above + CommandRunner.run_or_fail( + [ + "git", + "-c", + f"user.name={user_name}", + "-c", + f"user.email={user_email}", + "commit", + "-m", + f"feat({scope}): update to {version}", + ], + stage="CreateCommit", + env=clean_env, + ) + + @staticmethod + def push(branch: str): + CommandRunner.run_or_fail( + ["git", "push", "-u", "origin", branch], stage="PushBranch" + ) + + @staticmethod + def clean_repo(): + CommandRunner.run_or_fail( + ["git", "reset", "--hard", "HEAD"], stage="ResetRepository" + ) + CommandRunner.run_or_fail( + ["git", "checkout", Git.default_branch], stage="CheckoutDefaultBranch" + ) class GitHub: - @staticmethod - def check_newer_tag(repo, current_tag) -> UpdateStatus: - # GET /repos/:owner/:repo/git/refs/tags - url = f"https://api.github.com/repos/{repo}/git/refs/tags" - - # Send a GET request to the GitHub API - response = requests.get(url) - - # If the request was successful - if response.status_code == 200: - # Parse the JSON response - data = response.json() - - if len(data) == 0: - return { - "has_updates": False, - } - - latest_ref = data[-1] - latest_tag = latest_ref["ref"].replace("refs/tags/", "") - - if latest_tag == current_tag: - return { - "has_updates": False, - } - - return { - "has_updates": True, - "version": latest_tag, - "compare_url": f"https://github.com/{repo}/compare/{current_tag}...{latest_tag}", - "head_ref": latest_ref["object"]["sha"], - "head_url": f"https://github.com/{repo}/releases/tag/{latest_tag}", - } - else: - # If the request was not successful, raise an exception - raise Exception(f"GitHub API request failed with status code {response.status_code}: {response.json()}") - - @staticmethod - def check_updates(repo, branch, version) -> UpdateStatus: - # TODO: add support for semver updating (based on tags) - # Check if upstream github repo has a new version - # GitHub API URL for comparing two commits - url = f"https://api.github.com/repos/{repo}/compare/{version}...{branch}" - - # Send a GET request to the GitHub API - response = requests.get(url) - - # If the request was successful - if response.status_code == 200: - # Parse the JSON response - data = response.json() - - # If the base is behind the head, there is a newer version - has_updates = data["status"] != "identical" - - if not has_updates: - return { - "has_updates": False, - } - - return { - "has_updates": data["status"] != "identical", - "version": data["commits"][-1]["sha"], - "compare_url": data["permalink_url"], - "head_ref": data["commits"][-1]["sha"], - "head_url": data["commits"][-1]["html_url"] - } - else: - # If the request was not successful, raise an exception - raise Exception(f"GitHub API request failed with status code {response.status_code}: {response.json()}") - - @staticmethod - def create_issue(title: str, body: str) -> None: - cmd = [ - "gh", - "issue", - "create", - "-t", title, - "-b", body - ] - CommandRunner.run_or_fail(cmd, stage="CreateIssue") - - @staticmethod - def create_pr(branch: str, title: str, body: str) -> None: - cmd = [ - "gh", - "pr", - "create", - "-B", Git.default_branch, - "-H", branch, - "-t", title, - "-b", body - ] - CommandRunner.run_or_fail(cmd, stage="CreatePullRequest") + @staticmethod + def check_newer_tag(repo, current_tag) -> UpdateStatus: + # GET /repos/:owner/:repo/git/refs/tags + url = f"https://api.github.com/repos/{repo}/git/refs/tags" + + # Send a GET request to the GitHub API + response = requests.get(url) + + # If the request was successful + if response.status_code == 200: + # Parse the JSON response + data = response.json() + + if len(data) == 0: + return { + "has_updates": False, + } + + latest_ref = data[-1] + latest_tag = latest_ref["ref"].replace("refs/tags/", "") + + if latest_tag == current_tag: + return { + "has_updates": False, + } + + return { + "has_updates": True, + "version": latest_tag, + "compare_url": f"https://github.com/{repo}/compare/{current_tag}...{latest_tag}", + "head_ref": latest_ref["object"]["sha"], + "head_url": f"https://github.com/{repo}/releases/tag/{latest_tag}", + } + else: + # If the request was not successful, raise an exception + raise Exception( + f"GitHub API request failed with status code {response.status_code}: {response.json()}" + ) + + @staticmethod + def check_updates(repo, branch, version) -> UpdateStatus: + # TODO: add support for semver updating (based on tags) + # Check if upstream github repo has a new version + # GitHub API URL for comparing two commits + url = f"https://api.github.com/repos/{repo}/compare/{version}...{branch}" + + # Send a GET request to the GitHub API + response = requests.get(url) + + # If the request was successful + if response.status_code == 200: + # Parse the JSON response + data = response.json() + + # If the base is behind the head, there is a newer version + has_updates = data["status"] != "identical" + + if not has_updates: + return { + "has_updates": False, + } + + return { + "has_updates": data["status"] != "identical", + "version": data["commits"][-1]["sha"], + "compare_url": data["permalink_url"], + "head_ref": data["commits"][-1]["sha"], + "head_url": data["commits"][-1]["html_url"], + } + else: + # If the request was not successful, raise an exception + raise Exception( + f"GitHub API request failed with status code {response.status_code}: {response.json()}" + ) + + @staticmethod + def create_issue(title: str, body: str) -> None: + cmd = ["gh", "issue", "create", "-t", title, "-b", body] + CommandRunner.run_or_fail(cmd, stage="CreateIssue") + + @staticmethod + def create_pr(branch: str, title: str, body: str) -> None: + cmd = [ + "gh", + "pr", + "create", + "-B", + Git.default_branch, + "-H", + branch, + "-t", + title, + "-b", + body, + ] + CommandRunner.run_or_fail(cmd, stage="CreatePullRequest") def main(): - # Load the YAML file - with open(DEPS_YAML_FILE, "r") as yaml_file: - data: DependencyYAML = yaml.safe_load(yaml_file) + # Load the YAML file + with open(DEPS_YAML_FILE, "r") as yaml_file: + data: DependencyYAML = yaml.safe_load(yaml_file) + + if "dependencies" not in data: + raise Exception("dependencies.yml not properly formatted") - if "dependencies" not in data: - raise Exception(f"dependencies.yml not properly formatted") + # Cache YAML version + DependencyStore.set(data) - # Cache YAML version - DependencyStore.set(data) + dependencies = data["dependencies"] + for path in dependencies: + dependency = Dependency(path, dependencies[path]) + dependency.update_or_notify() - dependencies = data["dependencies"] - for path in dependencies: - dependency = Dependency(path, dependencies[path]) - dependency.update_or_notify() if __name__ == "__main__": - main() + main() -- cgit v1.2.3-70-g09d2 From a258eb4547e610899162526f754ce1bf88027112 Mon Sep 17 00:00:00 2001 From: Carlo Sala Date: Thu, 9 May 2024 18:27:01 +0200 Subject: fix(dependencies): improve typing --- .github/workflows/dependencies/updater.py | 36 ++++++++++++++++++------------- 1 file changed, 21 insertions(+), 15 deletions(-) (limited to '.github/workflows/dependencies/updater.py') diff --git a/.github/workflows/dependencies/updater.py b/.github/workflows/dependencies/updater.py index 765cb26fe..9754cdf83 100644 --- a/.github/workflows/dependencies/updater.py +++ b/.github/workflows/dependencies/updater.py @@ -4,7 +4,7 @@ import subprocess import sys import timeit from copy import deepcopy -from typing import Optional, TypedDict +from typing import Literal, NotRequired, TypedDict import requests import yaml @@ -49,20 +49,24 @@ class DependencyDict(TypedDict): repo: str branch: str version: str - precopy: Optional[str] - postcopy: Optional[str] + precopy: NotRequired[str] + postcopy: NotRequired[str] class DependencyYAML(TypedDict): dependencies: dict[str, DependencyDict] -class UpdateStatus(TypedDict): - has_updates: bool - version: Optional[str] - compare_url: Optional[str] - head_ref: Optional[str] - head_url: Optional[str] +class UpdateStatusFalse(TypedDict): + has_updates: Literal[False] + + +class UpdateStatusTrue(TypedDict): + has_updates: Literal[True] + version: str + compare_url: str + head_ref: str + head_url: str class CommandRunner: @@ -105,7 +109,9 @@ class DependencyStore: with CodeTimer(f"store deepcopy: {path}"): store_copy = deepcopy(DependencyStore.store) - dependency = store_copy["dependencies"].get(path, {}) + dependency = store_copy["dependencies"].get(path) + if dependency is None: + raise ValueError(f"Dependency {path} {version} not found") dependency["version"] = version store_copy["dependencies"][path] = dependency @@ -171,7 +177,7 @@ class Dependency: else: status = GitHub.check_updates(repo, remote_branch, version) - if status["has_updates"]: + if status["has_updates"] is True: short_sha = status["head_ref"][:8] new_version = status["version"] if is_tag else short_sha @@ -212,10 +218,10 @@ Check out the [list of changes]({status['compare_url']}). case CommandRunner.Exception: # Print error message print( - f"Error running {e.stage} command: {e.returncode}", + f"Error running {e.stage} command: {e.returncode}", # pyright: ignore[reportAttributeAccessIssue] file=sys.stderr, ) - print(e.stderr, file=sys.stderr) + print(e.stderr, file=sys.stderr) # pyright: ignore[reportAttributeAccessIssue] case shutil.Error: print(f"Error copying files: {e}", file=sys.stderr) @@ -378,7 +384,7 @@ class Git: class GitHub: @staticmethod - def check_newer_tag(repo, current_tag) -> UpdateStatus: + def check_newer_tag(repo, current_tag) -> UpdateStatusFalse | UpdateStatusTrue: # GET /repos/:owner/:repo/git/refs/tags url = f"https://api.github.com/repos/{repo}/git/refs/tags" @@ -417,7 +423,7 @@ class GitHub: ) @staticmethod - def check_updates(repo, branch, version) -> UpdateStatus: + def check_updates(repo, branch, version) -> UpdateStatusFalse | UpdateStatusTrue: # TODO: add support for semver updating (based on tags) # Check if upstream github repo has a new version # GitHub API URL for comparing two commits -- cgit v1.2.3-70-g09d2 From 423b9a8ded10b0d4a81d6134c4045d0b7e077eb5 Mon Sep 17 00:00:00 2001 From: Carlo Sala Date: Thu, 9 May 2024 19:20:11 +0200 Subject: feat(dependencies): add support for semver tags --- .github/workflows/dependencies/requirements.txt | 1 + .github/workflows/dependencies/updater.py | 62 ++++++++++++++++++++++--- 2 files changed, 57 insertions(+), 6 deletions(-) (limited to '.github/workflows/dependencies/updater.py') diff --git a/.github/workflows/dependencies/requirements.txt b/.github/workflows/dependencies/requirements.txt index 0ef3e54ca..7e840a74c 100644 --- a/.github/workflows/dependencies/requirements.txt +++ b/.github/workflows/dependencies/requirements.txt @@ -3,4 +3,5 @@ charset-normalizer==3.3.2 idna==3.7 PyYAML==6.0.1 requests==2.31.0 +semver==3.0.2 urllib3==2.2.1 diff --git a/.github/workflows/dependencies/updater.py b/.github/workflows/dependencies/updater.py index 9754cdf83..7e8e92ca9 100644 --- a/.github/workflows/dependencies/updater.py +++ b/.github/workflows/dependencies/updater.py @@ -1,13 +1,15 @@ import os +import re import shutil import subprocess import sys import timeit from copy import deepcopy -from typing import Literal, NotRequired, TypedDict +from typing import Literal, NotRequired, Optional, TypedDict import requests import yaml +from semver import Version # Get TMP_DIR variable from environment TMP_DIR = os.path.join(os.environ.get("TMP_DIR", "/tmp"), "ohmyzsh") @@ -16,6 +18,35 @@ DEPS_YAML_FILE = ".github/dependencies.yml" # Dry run flag DRY_RUN = os.environ.get("DRY_RUN", "0") == "1" +# utils for tag comparison +BASEVERSION = re.compile( + r"""[vV]? + (?P(0|[1-9])\d*) + (\. + (?P(0|[1-9])\d*) + (\. + (?P(0|[1-9])\d*) + )? + )? + """, + re.VERBOSE, +) + + +def coerce(version: str) -> Optional[Version]: + match = BASEVERSION.search(version) + if not match: + return None + + # BASEVERSION looks for `MAJOR.minor.patch` in the string given + # it fills with None if any of them is missing (for example `2.1`) + ver = { + key: 0 if value is None else value for key, value in match.groupdict().items() + } + # Version takes `major`, `minor`, `patch` arguments + ver = Version(**ver) # pyright: ignore[reportArgumentType] + return ver + class CodeTimer: def __init__(self, name=None): @@ -390,6 +421,11 @@ class GitHub: # Send a GET request to the GitHub API response = requests.get(url) + current_version = coerce(current_tag) + if current_version is None: + raise ValueError( + f"Stored {current_version} from {repo} does not follow semver" + ) # If the request was successful if response.status_code == 200: @@ -401,10 +437,27 @@ class GitHub: "has_updates": False, } - latest_ref = data[-1] + latest_ref = None + latest_version: Optional[Version] = None + for ref in data: + # we find the tag since GitHub returns it as plain git ref + tag_version = coerce(ref["ref"].replace("refs/tags/", "")) + if tag_version is None: + # we skip every tag that is not semver-complaint + continue + if latest_version is None or tag_version.compare(latest_version) > 0: + # if we have a "greater" semver version, set it as latest + latest_version = tag_version + latest_ref = ref + + # raise if no valid semver tag is found + if latest_ref is None or latest_version is None: + raise ValueError(f"No tags following semver found in {repo}") + + # we get the tag since GitHub returns it as plain git ref latest_tag = latest_ref["ref"].replace("refs/tags/", "") - if latest_tag == current_tag: + if latest_version.compare(current_version) <= 0: return { "has_updates": False, } @@ -424,9 +477,6 @@ class GitHub: @staticmethod def check_updates(repo, branch, version) -> UpdateStatusFalse | UpdateStatusTrue: - # TODO: add support for semver updating (based on tags) - # Check if upstream github repo has a new version - # GitHub API URL for comparing two commits url = f"https://api.github.com/repos/{repo}/compare/{version}...{branch}" # Send a GET request to the GitHub API -- cgit v1.2.3-70-g09d2 From eb2ff84a2c8ae45b90d36e367c6bd6bc01f8464d Mon Sep 17 00:00:00 2001 From: Carlo Sala Date: Thu, 9 May 2024 19:52:50 +0200 Subject: fix(dependencies): avoid creating PR if it's already there --- .github/workflows/dependencies/updater.py | 42 ++++++++++++++++++++++++++----- 1 file changed, 36 insertions(+), 6 deletions(-) (limited to '.github/workflows/dependencies/updater.py') diff --git a/.github/workflows/dependencies/updater.py b/.github/workflows/dependencies/updater.py index 7e8e92ca9..6fa32e378 100644 --- a/.github/workflows/dependencies/updater.py +++ b/.github/workflows/dependencies/updater.py @@ -1,3 +1,4 @@ +import json import os import re import shutil @@ -213,8 +214,10 @@ class Dependency: new_version = status["version"] if is_tag else short_sha try: + branch_name = f"update/{self.path}/{new_version}" + # Create new branch - branch = Git.create_branch(self.path, new_version) + branch = Git.checkout_or_create_branch(branch_name) # Update dependencies.yml file self.__update_yaml( @@ -353,7 +356,7 @@ class Git: ) @staticmethod - def create_branch(path: str, version: str): + def checkout_or_create_branch(branch_name: str): # Get current branch name result = CommandRunner.run_or_fail( ["git", "rev-parse", "--abbrev-ref", "HEAD"], stage="GetDefaultBranch" @@ -361,10 +364,16 @@ class Git: Git.default_branch = result.stdout.decode("utf-8").strip() # Create new branch and return created branch name - branch_name = f"update/{path}/{version}" - CommandRunner.run_or_fail( - ["git", "checkout", "-b", branch_name], stage="CreateBranch" - ) + try: + # try to checkout already existing branch + CommandRunner.run_or_fail( + ["git", "checkout", branch_name], stage="CreateBranch" + ) + except CommandRunner.Exception: + # otherwise create new branch + CommandRunner.run_or_fail( + ["git", "checkout", "-b", branch_name], stage="CreateBranch" + ) return branch_name @staticmethod @@ -515,6 +524,27 @@ class GitHub: @staticmethod def create_pr(branch: str, title: str, body: str) -> None: + # first of all let's check if PR is already open + check_cmd = [ + "gh", + "pr", + "list", + "--state", + "open", + "--head", + branch, + "--json", + "title", + ] + # returncode is 0 also if no PRs are found + output = json.loads( + CommandRunner.run_or_fail(check_cmd, stage="CheckPullRequestOpen") + .stdout.decode("utf-8") + .strip() + ) + # we have PR in this case! + if len(output) > 0: + return cmd = [ "gh", "pr", -- cgit v1.2.3-70-g09d2 From 0493eab8ce02c4988a16cbe27ad61a20ed8a89df Mon Sep 17 00:00:00 2001 From: Carlo Sala Date: Sun, 12 May 2024 12:40:45 +0200 Subject: fix(dependencies): check if repo is clean before committing --- .github/workflows/dependencies/updater.py | 36 ++++++++++++++++++------------- 1 file changed, 21 insertions(+), 15 deletions(-) (limited to '.github/workflows/dependencies/updater.py') diff --git a/.github/workflows/dependencies/updater.py b/.github/workflows/dependencies/updater.py index 6fa32e378..4533b7aeb 100644 --- a/.github/workflows/dependencies/updater.py +++ b/.github/workflows/dependencies/updater.py @@ -390,21 +390,27 @@ class Git: clean_env["GIT_CONFIG_GLOBAL"] = "/dev/null" clean_env["GIT_CONFIG_NOSYSTEM"] = "1" - # Commit with settings above - CommandRunner.run_or_fail( - [ - "git", - "-c", - f"user.name={user_name}", - "-c", - f"user.email={user_email}", - "commit", - "-m", - f"feat({scope}): update to {version}", - ], - stage="CreateCommit", - env=clean_env, - ) + # check if repo is clean (clean => no error, no commit) + try: + CommandRunner.run_or_fail( + ["git", "diff", "--exit-code"], stage="CheckRepoClean", env=clean_env + ) + except CommandRunner.Exception: + # Commit with settings above + CommandRunner.run_or_fail( + [ + "git", + "-c", + f"user.name={user_name}", + "-c", + f"user.email={user_email}", + "commit", + "-m", + f"feat({scope}): update to {version}", + ], + stage="CreateCommit", + env=clean_env, + ) @staticmethod def push(branch: str): -- cgit v1.2.3-70-g09d2 From 0621944db50aec931850806644b673e39b650163 Mon Sep 17 00:00:00 2001 From: Carlo Sala Date: Tue, 21 May 2024 20:43:26 +0200 Subject: fix(dependencies): only open PR if there are changes --- .github/workflows/dependencies/updater.py | 78 +++++++++++++++++-------------- 1 file changed, 44 insertions(+), 34 deletions(-) (limited to '.github/workflows/dependencies/updater.py') diff --git a/.github/workflows/dependencies/updater.py b/.github/workflows/dependencies/updater.py index 4533b7aeb..e64d69939 100644 --- a/.github/workflows/dependencies/updater.py +++ b/.github/workflows/dependencies/updater.py @@ -228,21 +228,22 @@ class Dependency: self.__apply_upstream_changes() # Add all changes and commit - Git.add_and_commit(self.name, short_sha) + has_new_commit = Git.add_and_commit(self.name, short_sha) - # Push changes to remote - Git.push(branch) + if has_new_commit: + # Push changes to remote + Git.push(branch) - # Create GitHub PR - GitHub.create_pr( - branch, - f"feat({self.name}): update to version {new_version}", - f"""## Description + # Create GitHub PR + GitHub.create_pr( + branch, + f"feat({self.name}): update to version {new_version}", + f"""## Description -Update for **{self.desc}**: update to version [{new_version}]({status['head_url']}). -Check out the [list of changes]({status['compare_url']}). -""", - ) + Update for **{self.desc}**: update to version [{new_version}]({status['head_url']}). + Check out the [list of changes]({status['compare_url']}). + """, + ) # Clean up repository Git.clean_repo() @@ -377,7 +378,21 @@ class Git: return branch_name @staticmethod - def add_and_commit(scope: str, version: str): + def add_and_commit(scope: str, version: str) -> bool: + """ + Returns `True` if there were changes and were indeed commited. + Returns `False` if the repo was clean and no changes were commited. + """ + # check if repo is clean (clean => no error, no commit) + try: + CommandRunner.run_or_fail( + ["git", "diff", "--exit-code"], stage="CheckRepoClean" + ) + return False + except CommandRunner.Exception: + # if it's other kind of error just throw! + pass + user_name = os.environ.get("GIT_APP_NAME") user_email = os.environ.get("GIT_APP_EMAIL") @@ -390,27 +405,22 @@ class Git: clean_env["GIT_CONFIG_GLOBAL"] = "/dev/null" clean_env["GIT_CONFIG_NOSYSTEM"] = "1" - # check if repo is clean (clean => no error, no commit) - try: - CommandRunner.run_or_fail( - ["git", "diff", "--exit-code"], stage="CheckRepoClean", env=clean_env - ) - except CommandRunner.Exception: - # Commit with settings above - CommandRunner.run_or_fail( - [ - "git", - "-c", - f"user.name={user_name}", - "-c", - f"user.email={user_email}", - "commit", - "-m", - f"feat({scope}): update to {version}", - ], - stage="CreateCommit", - env=clean_env, - ) + # Commit with settings above + CommandRunner.run_or_fail( + [ + "git", + "-c", + f"user.name={user_name}", + "-c", + f"user.email={user_email}", + "commit", + "-m", + f"feat({scope}): update to {version}", + ], + stage="CreateCommit", + env=clean_env, + ) + return True @staticmethod def push(branch: str): -- cgit v1.2.3-70-g09d2 From 04b66b230857fb9bc28af5e7590b09ff79f888fa Mon Sep 17 00:00:00 2001 From: Carlo Sala Date: Tue, 21 May 2024 20:46:54 +0200 Subject: chore(dependencies): PR wording --- .github/workflows/dependencies/updater.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to '.github/workflows/dependencies/updater.py') diff --git a/.github/workflows/dependencies/updater.py b/.github/workflows/dependencies/updater.py index e64d69939..f85c9eda7 100644 --- a/.github/workflows/dependencies/updater.py +++ b/.github/workflows/dependencies/updater.py @@ -240,9 +240,9 @@ class Dependency: f"feat({self.name}): update to version {new_version}", f"""## Description - Update for **{self.desc}**: update to version [{new_version}]({status['head_url']}). - Check out the [list of changes]({status['compare_url']}). - """, +Update for **{self.desc}**: update to version [{new_version}]({status['head_url']}). +Check out the [list of changes]({status['compare_url']}). +""", ) # Clean up repository -- cgit v1.2.3-70-g09d2 From b3ba8da4218c3b9e1e5e45e1d4c00d312ff7226b Mon Sep 17 00:00:00 2001 From: Marc Cornellà Date: Mon, 14 Oct 2024 13:15:39 +0200 Subject: ci(dependencies): use tag version in git commit if available (#12756) Related: https://github.com/ohmyzsh/ohmyzsh/pull/12747#issuecomment-2410440748 --- .github/workflows/dependencies/updater.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to '.github/workflows/dependencies/updater.py') diff --git a/.github/workflows/dependencies/updater.py b/.github/workflows/dependencies/updater.py index f85c9eda7..02cff9030 100644 --- a/.github/workflows/dependencies/updater.py +++ b/.github/workflows/dependencies/updater.py @@ -228,7 +228,7 @@ class Dependency: self.__apply_upstream_changes() # Add all changes and commit - has_new_commit = Git.add_and_commit(self.name, short_sha) + has_new_commit = Git.add_and_commit(self.name, new_version) if has_new_commit: # Push changes to remote -- cgit v1.2.3-70-g09d2 From ca5c467db1d495a4b5c0049a4cf476ca7f34b790 Mon Sep 17 00:00:00 2001 From: Carlo Sala Date: Mon, 1 Dec 2025 09:59:09 +0100 Subject: fix(dependencies): only open PR if there are relevant changes (#13454) Fixes cases like #13453 --- .github/workflows/dependencies/updater.py | 62 +++++++++++++++++-------------- 1 file changed, 35 insertions(+), 27 deletions(-) (limited to '.github/workflows/dependencies/updater.py') diff --git a/.github/workflows/dependencies/updater.py b/.github/workflows/dependencies/updater.py index 02cff9030..aee15d9e0 100644 --- a/.github/workflows/dependencies/updater.py +++ b/.github/workflows/dependencies/updater.py @@ -219,31 +219,32 @@ class Dependency: # Create new branch branch = Git.checkout_or_create_branch(branch_name) - # Update dependencies.yml file - self.__update_yaml( - f"tag:{new_version}" if is_tag else status["version"] - ) - # Update dependency files self.__apply_upstream_changes() - # Add all changes and commit - has_new_commit = Git.add_and_commit(self.name, new_version) + if not Git.repo_is_clean(): + # Update dependencies.yml file + self.__update_yaml( + f"tag:{new_version}" if is_tag else status["version"] + ) + + # Add all changes and commit + has_new_commit = Git.add_and_commit(self.name, new_version) - if has_new_commit: - # Push changes to remote - Git.push(branch) + if has_new_commit: + # Push changes to remote + Git.push(branch) - # Create GitHub PR - GitHub.create_pr( - branch, - f"feat({self.name}): update to version {new_version}", - f"""## Description + # Create GitHub PR + GitHub.create_pr( + branch, + f"feat({self.name}): update to version {new_version}", + f"""## Description -Update for **{self.desc}**: update to version [{new_version}]({status['head_url']}). -Check out the [list of changes]({status['compare_url']}). +Update for **{self.desc}**: update to version [{new_version}]({status["head_url"]}). +Check out the [list of changes]({status["compare_url"]}). """, - ) + ) # Clean up repository Git.clean_repo() @@ -275,8 +276,8 @@ Check out the [list of changes]({status['compare_url']}). There is a new version of `{self.name}` {self.kind} available. -New version: [{new_version}]({status['head_url']}) -Check out the [list of changes]({status['compare_url']}). +New version: [{new_version}]({status["head_url"]}) +Check out the [list of changes]({status["compare_url"]}). """ print("Creating GitHub issue", file=sys.stderr) @@ -378,20 +379,27 @@ class Git: return branch_name @staticmethod - def add_and_commit(scope: str, version: str) -> bool: + def repo_is_clean() -> bool: """ - Returns `True` if there were changes and were indeed commited. - Returns `False` if the repo was clean and no changes were commited. + Returns `True` if the repo is clean. + Returns `False` if the repo is dirty. """ - # check if repo is clean (clean => no error, no commit) try: CommandRunner.run_or_fail( ["git", "diff", "--exit-code"], stage="CheckRepoClean" ) - return False + return True except CommandRunner.Exception: - # if it's other kind of error just throw! - pass + return False + + @staticmethod + def add_and_commit(scope: str, version: str) -> bool: + """ + Returns `True` if there were changes and were indeed commited. + Returns `False` if the repo was clean and no changes were commited. + """ + if Git.repo_is_clean(): + return False user_name = os.environ.get("GIT_APP_NAME") user_email = os.environ.get("GIT_APP_EMAIL") -- cgit v1.2.3-70-g09d2 From e9fc134236323ce3ce376715b1e55a54ed6ac7ac Mon Sep 17 00:00:00 2001 From: Marc Cornellà Date: Mon, 1 Dec 2025 11:22:14 +0100 Subject: ci(dependencies): update job permissions, change commits to `chore` (#13457) --- .github/workflows/dependencies.yml | 5 ++--- .github/workflows/dependencies/updater.py | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) (limited to '.github/workflows/dependencies/updater.py') diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yml index 000589087..a21803ec6 100644 --- a/.github/workflows/dependencies.yml +++ b/.github/workflows/dependencies.yml @@ -4,14 +4,13 @@ on: schedule: - cron: "0 6 * * 0" -permissions: - contents: write - jobs: check: name: Check for updates runs-on: ubuntu-latest if: github.repository == 'ohmyzsh/ohmyzsh' + permissions: + contents: write # this is needed to push commits and branches steps: - name: Harden the runner (Audit all outbound calls) uses: step-security/harden-runner@95d9a5deda9de15063e7595e9719c11c38c90ae2 # v2.13.2 diff --git a/.github/workflows/dependencies/updater.py b/.github/workflows/dependencies/updater.py index aee15d9e0..783161d6c 100644 --- a/.github/workflows/dependencies/updater.py +++ b/.github/workflows/dependencies/updater.py @@ -238,7 +238,7 @@ class Dependency: # Create GitHub PR GitHub.create_pr( branch, - f"feat({self.name}): update to version {new_version}", + f"chore({self.name}): update to version {new_version}", f"""## Description Update for **{self.desc}**: update to version [{new_version}]({status["head_url"]}). @@ -423,7 +423,7 @@ class Git: f"user.email={user_email}", "commit", "-m", - f"feat({scope}): update to {version}", + f"chore({scope}): update to {version}", ], stage="CreateCommit", env=clean_env, -- cgit v1.2.3-70-g09d2