From 8c3b76d911376706b84f127a580b74e7c3ccd1bd Mon Sep 17 00:00:00 2001 From: Giovanni Francesco Capalbo Date: Wed, 12 Apr 2023 16:20:42 +0200 Subject: [PATCH 01/10] [ADD] get_pins script --- bin/get_pins | 174 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 174 insertions(+) create mode 100755 bin/get_pins diff --git a/bin/get_pins b/bin/get_pins new file mode 100755 index 00000000..6ab6263e --- /dev/null +++ b/bin/get_pins @@ -0,0 +1,174 @@ +#!/usr/bin/env python +# Version: v.22.05.30 +# -*- coding: utf-9 -*- +import os +import argparse +from subprocess import run, PIPE +from dotenv import load_dotenv + +import yaml +from waftlib import ( + ADDONS_YAML, + PRIVATE, + REPOS_YAML, + SRC_DIR, + logger, + ODOO_VERSION, +) + +SCRIPT_PATH = os.path.abspath(os.path.dirname(__file__)) +os.environ["ODOO_WORK_DIR"] = os.path.realpath(os.path.join(SCRIPT_PATH, "../..")) +load_dotenv(os.path.join(os.environ["ODOO_WORK_DIR"], ".env-default")) +load_dotenv(os.path.join(os.environ["ODOO_WORK_DIR"], ".env-shared"), override=True) +load_dotenv(os.path.join(os.environ["ODOO_WORK_DIR"], ".env-secret"), override=True) + +parser = argparse.ArgumentParser( + prog="get_pins", + description="Fetch current pins of this instance", + epilog="Outputs pinned repo.yaml", +) + +parser.add_argument("--only-base") +args = parser.parse_args() + + +class CustomDumper(yaml.Dumper): + """ + We want order of keys intact so that the output is the same order than source. + Unfortunately PyYAML orders yaml output alphabetically on dump. + pyyaml 5.1 has an option to disable alphabetical orderng of dumps, but often our + wafts have a version < 5.1. + for this reason i made a custom dumper to have an "unordered" dump, without + affecting the default behaviour of pyyaml dump. This script supports pyYAML<5.1. + """ + + def represent_dict_preserve_order(self, data): + return self.represent_dict(data.items()) + + +def split_line(line): + splitted_lines = line.split(" ") + for sl in splitted_lines: + if sl == "": + splitted_lines.pop(splitted_lines.index(sl)) + return splitted_lines + + +def is_in_history(value, history): + for commit in history: + if commit[:8] == value[:8]: + return True + return False + + +def update_pins(): + current_heads = {} + all_commits = {} + """ + parsing directly repos.yaml, if something is not in addons.yaml, branch will still + be in folder, but may not be included in addons. Nothing changes. + """ + with open(REPOS_YAML) as yaml_file: + for doc in yaml.safe_load_all(yaml_file): + for repo in doc: + remotes = doc[repo]["remotes"].keys() + is_base_branch = True + for merge in doc[repo]["merges"]: + """ + Possible syntaxes for merges: + + merge_type=1 + merge_type=2 + merge_type=3 + merge_type=0 "invalid" + """ + index = doc[repo]["merges"].index(merge) + if index > 0 and bool(args.only_base) == True: + continue + splitted_merge = [x for x in merge.split(" ") if x != ""] + if splitted_merge[0] not in remotes: + logger.debug("Invalid Remote on line: %s" % merge) + raise ValueError + if index == 0: + base_branch = splitted_merge + merge_type = 0 # invalid + env = dict(os.environ, **doc.get("ENV", {})) + if repo in {PRIVATE, "ONLY", "ENV"}: + continue + else: + repo_path = os.path.abspath(os.path.join(SRC_DIR, repo)) + if not os.path.exists(repo_path): + logger.debug("build incomplete") + exit() + wd = os.getcwd() + os.chdir(repo_path) + allrev = run(["git", "rev-list", "HEAD"], stdout=PIPE, stderr=PIPE) + lrev = run(["git", "rev-parse", "HEAD"], stdout=PIPE, stderr=PIPE) + current_heads[repo] = ( + lrev.stdout.decode("utf-8").replace("\n", "").replace("'", "") + ) + all_commits[repo] = ( + allrev.stdout.decode("utf-8").replace("\n", ",").split(",") + ) + if len(splitted_merge) == 3: + # merge_type line + merge_type = 3 + branchname = splitted_merge[1].replace( + "${ODOO_VERSION}", ODOO_VERSION + ) + current_pin = run( + [ + "git", + "merge-base", + "HEAD", + "".join([splitted_merge[0], "/", branchname]), + ], + stdout=PIPE, + stderr=PIPE, + ) + if splitted_merge[2] != current_pin: + splitted_merge[2] = current_pin + else: + if is_in_history(splitted_merge[1], all_commits.get(repo, [])): + # merge_type line + merge_type = 2 + splitted_merge[1] = current_heads.get(repo, "") + else: + # merge_type line + merge_type = 1 + branchname = splitted_merge[1].replace( + "${ODOO_VERSION}", ODOO_VERSION + ) + current_pin = ( + run( + [ + "git", + "merge-base", + "HEAD", + "".join([splitted_merge[0], "/", branchname]), + ], + stdout=PIPE, + stderr=PIPE, + ) + .stdout.decode("utf-8") + .replace("\n", "") + ) + splitted_merge.append(current_pin) + os.chdir(wd) + if merge_type == 0: + logger.debug( + "Invalid Repo of unrecognized merge_type: %s" + % splitted_merge + ) + raise ValueError + doc[repo]["merges"][index] = " ".join(splitted_merge) + # PROPOSED ADDITIONAL FEATURE + # most recent common ancestor depth MRCAD + # if depth is insufficient we also output the new needed depth + + CustomDumper.add_representer(dict, CustomDumper.represent_dict_preserve_order) + print(yaml.dump(doc, Dumper=CustomDumper)) + + +if os.path.isfile(REPOS_YAML): + update_pins() From e2b81e7c869d8413056762fc50dfd1c43131456a Mon Sep 17 00:00:00 2001 From: Giovanni Francesco Capalbo Date: Wed, 19 Apr 2023 12:47:51 +0200 Subject: [PATCH 02/10] [REF] Refactor and added support for target pinning --- bin/get_pins | 193 ++++++++++++++++++++++++++------------------------- 1 file changed, 97 insertions(+), 96 deletions(-) diff --git a/bin/get_pins b/bin/get_pins index 6ab6263e..55fdaaf4 100755 --- a/bin/get_pins +++ b/bin/get_pins @@ -8,7 +8,6 @@ from dotenv import load_dotenv import yaml from waftlib import ( - ADDONS_YAML, PRIVATE, REPOS_YAML, SRC_DIR, @@ -30,6 +29,8 @@ parser = argparse.ArgumentParser( parser.add_argument("--only-base") args = parser.parse_args() +current_heads = {} +all_commits = {} class CustomDumper(yaml.Dumper): @@ -61,9 +62,85 @@ def is_in_history(value, history): return False +def preprocess_merge(doc, repo, merge): + remotes = doc[repo]["remotes"].keys() + splitted_merge = [x for x in merge.split(" ") if x != ""] + if splitted_merge[0] not in remotes: + logger.debug("Invalid Remote on line: %s" % merge) + raise ValueError + repo_path = os.path.abspath(os.path.join(SRC_DIR, repo)) + if not os.path.exists(repo_path): + logger.debug("build incomplete") + exit() + return repo_path, splitted_merge + + +def process_merge(doc, repo, merge, index): + """ + Possible syntaxes for merges: + + merge_type=1 + merge_type=2 + merge_type=3 + merge_type=0 "invalid" + """ + repo_path, splitted_merge = preprocess_merge(doc, repo, merge) + merge_type = 0 # invalid + env = dict(os.environ, **doc.get("ENV", {})) + wd = os.getcwd() + os.chdir(repo_path) + allrev = run(["git", "rev-list", "HEAD"], stdout=PIPE, stderr=PIPE) + lrev = run(["git", "rev-parse", "HEAD"], stdout=PIPE, stderr=PIPE) + current_heads[repo] = lrev.stdout.decode("utf-8").replace("\n", "").replace("'", "") + all_commits[repo] = allrev.stdout.decode("utf-8").replace("\n", ",").split(",") + if len(splitted_merge) == 3: + # merge_type line + merge_type = 3 + branchname = splitted_merge[1].replace("${ODOO_VERSION}", ODOO_VERSION) + current_pin = run( + [ + "git", + "merge-base", + "HEAD", + "".join([splitted_merge[0], "/", branchname]), + ], + stdout=PIPE, + stderr=PIPE, + ) + if splitted_merge[2] != current_pin: + splitted_merge[2] = current_pin + else: + if is_in_history(splitted_merge[1], all_commits.get(repo, [])): + # merge_type line + merge_type = 2 + splitted_merge[1] = current_heads.get(repo, "") + else: + # merge_type line + merge_type = 1 + branchname = splitted_merge[1].replace("${ODOO_VERSION}", ODOO_VERSION) + current_pin = ( + run( + [ + "git", + "merge-base", + "HEAD", + "".join([splitted_merge[0], "/", branchname]), + ], + stdout=PIPE, + stderr=PIPE, + ) + .stdout.decode("utf-8") + .replace("\n", "") + ) + splitted_merge.append(current_pin) + os.chdir(wd) + if merge_type == 0: + logger.debug("Invalid Repo of unrecognized merge_type: %s" % splitted_merge) + raise ValueError + return " ".join(splitted_merge) + + def update_pins(): - current_heads = {} - all_commits = {} """ parsing directly repos.yaml, if something is not in addons.yaml, branch will still be in folder, but may not be included in addons. Nothing changes. @@ -71,102 +148,26 @@ def update_pins(): with open(REPOS_YAML) as yaml_file: for doc in yaml.safe_load_all(yaml_file): for repo in doc: - remotes = doc[repo]["remotes"].keys() - is_base_branch = True + if repo in {PRIVATE, "ONLY", "ENV"}: + continue + target = doc[repo]["target"] or False + has_target = target and True or False + processed_target = False + if has_target: + doc[repo]["target"] = process_merge(doc, repo, target, 0) for merge in doc[repo]["merges"]: - """ - Possible syntaxes for merges: - - merge_type=1 - merge_type=2 - merge_type=3 - merge_type=0 "invalid" - """ index = doc[repo]["merges"].index(merge) - if index > 0 and bool(args.only_base) == True: + # if target exists and only base is selected , just pin target. + if ( + (index > 0 or (has_target and index >= 0)) + and bool(args.only_base) + ) == True: continue - splitted_merge = [x for x in merge.split(" ") if x != ""] - if splitted_merge[0] not in remotes: - logger.debug("Invalid Remote on line: %s" % merge) - raise ValueError - if index == 0: - base_branch = splitted_merge - merge_type = 0 # invalid - env = dict(os.environ, **doc.get("ENV", {})) - if repo in {PRIVATE, "ONLY", "ENV"}: - continue - else: - repo_path = os.path.abspath(os.path.join(SRC_DIR, repo)) - if not os.path.exists(repo_path): - logger.debug("build incomplete") - exit() - wd = os.getcwd() - os.chdir(repo_path) - allrev = run(["git", "rev-list", "HEAD"], stdout=PIPE, stderr=PIPE) - lrev = run(["git", "rev-parse", "HEAD"], stdout=PIPE, stderr=PIPE) - current_heads[repo] = ( - lrev.stdout.decode("utf-8").replace("\n", "").replace("'", "") - ) - all_commits[repo] = ( - allrev.stdout.decode("utf-8").replace("\n", ",").split(",") - ) - if len(splitted_merge) == 3: - # merge_type line - merge_type = 3 - branchname = splitted_merge[1].replace( - "${ODOO_VERSION}", ODOO_VERSION - ) - current_pin = run( - [ - "git", - "merge-base", - "HEAD", - "".join([splitted_merge[0], "/", branchname]), - ], - stdout=PIPE, - stderr=PIPE, - ) - if splitted_merge[2] != current_pin: - splitted_merge[2] = current_pin - else: - if is_in_history(splitted_merge[1], all_commits.get(repo, [])): - # merge_type line - merge_type = 2 - splitted_merge[1] = current_heads.get(repo, "") - else: - # merge_type line - merge_type = 1 - branchname = splitted_merge[1].replace( - "${ODOO_VERSION}", ODOO_VERSION - ) - current_pin = ( - run( - [ - "git", - "merge-base", - "HEAD", - "".join([splitted_merge[0], "/", branchname]), - ], - stdout=PIPE, - stderr=PIPE, - ) - .stdout.decode("utf-8") - .replace("\n", "") - ) - splitted_merge.append(current_pin) - os.chdir(wd) - if merge_type == 0: - logger.debug( - "Invalid Repo of unrecognized merge_type: %s" - % splitted_merge - ) - raise ValueError - doc[repo]["merges"][index] = " ".join(splitted_merge) - # PROPOSED ADDITIONAL FEATURE - # most recent common ancestor depth MRCAD - # if depth is insufficient we also output the new needed depth - - CustomDumper.add_representer(dict, CustomDumper.represent_dict_preserve_order) + doc[repo]["merges"][index] = process_merge(doc, repo, merge, index) + + CustomDumper.add_representer( + dict, CustomDumper.represent_dict_preserve_order + ) print(yaml.dump(doc, Dumper=CustomDumper)) From 67136d4330eb2909068c5999062315e21c199fe5 Mon Sep 17 00:00:00 2001 From: Giovanni Francesco Capalbo Date: Thu, 4 May 2023 12:26:11 +0200 Subject: [PATCH 03/10] [ADD] automatic needed depth determination --- bin/get_pins | 137 ++++++++++++++++++++++++++++++++++----------------- 1 file changed, 92 insertions(+), 45 deletions(-) diff --git a/bin/get_pins b/bin/get_pins index 55fdaaf4..153b881f 100755 --- a/bin/get_pins +++ b/bin/get_pins @@ -1,6 +1,5 @@ #!/usr/bin/env python # Version: v.22.05.30 -# -*- coding: utf-9 -*- import os import argparse from subprocess import run, PIPE @@ -36,11 +35,12 @@ all_commits = {} class CustomDumper(yaml.Dumper): """ We want order of keys intact so that the output is the same order than source. - Unfortunately PyYAML orders yaml output alphabetically on dump. + Unfortunately PyYAML orders yaml output alphabetically on dump. pyyaml 5.1 has an option to disable alphabetical orderng of dumps, but often our wafts have a version < 5.1. for this reason i made a custom dumper to have an "unordered" dump, without affecting the default behaviour of pyyaml dump. This script supports pyYAML<5.1. + Our output will therefore have the same order as given input. """ def represent_dict_preserve_order(self, data): @@ -75,28 +75,48 @@ def preprocess_merge(doc, repo, merge): return repo_path, splitted_merge -def process_merge(doc, repo, merge, index): +def get_branchname(splitted_merge): + return splitted_merge[1].replace("${ODOO_VERSION}", ODOO_VERSION) + + +def update_repo_commits_and_heads(repo_path, repo): + os.chdir(repo_path) + allrev = run(["git", "rev-list", "HEAD"], stdout=PIPE, stderr=PIPE) + lrev = run(["git", "rev-parse", "HEAD"], stdout=PIPE, stderr=PIPE) + # no return updating globals. + current_heads[repo] = lrev.stdout.decode("utf-8").replace("\n", "").replace("'", "") + all_commits[repo] = allrev.stdout.decode("utf-8").replace("\n", ",").split(",") + + +def get_merge_type(splitted_merge): """ Possible syntaxes for merges: merge_type=1 merge_type=2 merge_type=3 - merge_type=0 "invalid" + merge_type=0 "invalid" """ + if len(splitted_merge) == 3: + return 3 + else: + if is_in_history(splitted_merge[1], all_commits.get(repo, [])): + return 2 + else: + return 1 + return 0 # unreachable. + + +def process_merge(doc, repo, merge, index): repo_path, splitted_merge = preprocess_merge(doc, repo, merge) - merge_type = 0 # invalid + # will update all_commits[repo] and current_heads[repo] globals. + update_repo_commits_and_heads(repo_path, repo) env = dict(os.environ, **doc.get("ENV", {})) wd = os.getcwd() - os.chdir(repo_path) - allrev = run(["git", "rev-list", "HEAD"], stdout=PIPE, stderr=PIPE) - lrev = run(["git", "rev-parse", "HEAD"], stdout=PIPE, stderr=PIPE) - current_heads[repo] = lrev.stdout.decode("utf-8").replace("\n", "").replace("'", "") - all_commits[repo] = allrev.stdout.decode("utf-8").replace("\n", ",").split(",") - if len(splitted_merge) == 3: - # merge_type line - merge_type = 3 - branchname = splitted_merge[1].replace("${ODOO_VERSION}", ODOO_VERSION) + merge_type = get_merge_type(splitted_merge) + if merge_type == 3: + # merge_type #3 + branchname = get_branchname(splitted_merge) current_pin = run( [ "git", @@ -109,38 +129,36 @@ def process_merge(doc, repo, merge, index): ) if splitted_merge[2] != current_pin: splitted_merge[2] = current_pin - else: - if is_in_history(splitted_merge[1], all_commits.get(repo, [])): - # merge_type line - merge_type = 2 - splitted_merge[1] = current_heads.get(repo, "") - else: - # merge_type line - merge_type = 1 - branchname = splitted_merge[1].replace("${ODOO_VERSION}", ODOO_VERSION) - current_pin = ( - run( - [ - "git", - "merge-base", - "HEAD", - "".join([splitted_merge[0], "/", branchname]), - ], - stdout=PIPE, - stderr=PIPE, - ) - .stdout.decode("utf-8") - .replace("\n", "") + if merge_type == 2: + # merge_type line + splitted_merge[1] = current_heads.get(repo, "") + if merge_type == 1: + # merge_type line + # merge_type #1 + branchname = get_branchname(splitted_merge) + current_pin = ( + run( + [ + "git", + "merge-base", + "HEAD", + "".join([splitted_merge[0], "/", branchname]), + ], + stdout=PIPE, + stderr=PIPE, ) - splitted_merge.append(current_pin) - os.chdir(wd) - if merge_type == 0: - logger.debug("Invalid Repo of unrecognized merge_type: %s" % splitted_merge) - raise ValueError + .stdout.decode("utf-8") + .replace("\n", "") + ) + splitted_merge.append(current_pin) + os.chdir(wd) + if merge_type == 0: + logger.debug("Invalid Repo of unrecognized merge_type: %s" % splitted_merge) + raise ValueError return " ".join(splitted_merge) -def update_pins(): +def main(): """ parsing directly repos.yaml, if something is not in addons.yaml, branch will still be in folder, but may not be included in addons. Nothing changes. @@ -155,6 +173,9 @@ def update_pins(): processed_target = False if has_target: doc[repo]["target"] = process_merge(doc, repo, target, 0) + # main branch is defined as target or in absence of target, merge[0] + main_branch = target or merge[0] + main_branch = split_line(main_branch)[0] for merge in doc[repo]["merges"]: index = doc[repo]["merges"].index(merge) # if target exists and only base is selected , just pin target. @@ -164,12 +185,38 @@ def update_pins(): ) == True: continue doc[repo]["merges"][index] = process_merge(doc, repo, merge, index) - + lastrev = ( + run( + [ + "git", + "merge-base", + "HEAD", + "".join([splitted_merge[0], "/", branchname]), + ], + stdout=PIPE, + stderr=PIPE, + ) + .stdout.decode("utf-8") + .replace("\n", "") + ) + # we now calculate the needed depth of this branch + mindepth = ( + run( + ["git", "rev-list", "HEAD", "^$" + lastrev, "--count"], + stdout=PIPE, + stderr=PIPE, + ) + .stdout.decode("utf-8") + .replace("\n", "") + ) CustomDumper.add_representer( dict, CustomDumper.represent_dict_preserve_order ) print(yaml.dump(doc, Dumper=CustomDumper)) -if os.path.isfile(REPOS_YAML): - update_pins() +if os.path.isfile(REPOS_YAML) and __name__ == "main": + main() +else: + logger.debug("no %s repository file found" % REPOS_YAML) + raise ValueError From 067b945823e63df41d02c31271676d9639e3cb3f Mon Sep 17 00:00:00 2001 From: Giovanni Francesco Capalbo Date: Thu, 4 May 2023 17:38:22 +0200 Subject: [PATCH 04/10] [fix] --- bin/get_pins | 82 ++++++++++++++++++++++++++++++---------------------- 1 file changed, 47 insertions(+), 35 deletions(-) diff --git a/bin/get_pins b/bin/get_pins index 153b881f..d629a226 100755 --- a/bin/get_pins +++ b/bin/get_pins @@ -28,9 +28,11 @@ parser = argparse.ArgumentParser( parser.add_argument("--only-base") args = parser.parse_args() + + current_heads = {} all_commits = {} - +min_depth = {} class CustomDumper(yaml.Dumper): """ @@ -88,7 +90,7 @@ def update_repo_commits_and_heads(repo_path, repo): all_commits[repo] = allrev.stdout.decode("utf-8").replace("\n", ",").split(",") -def get_merge_type(splitted_merge): +def get_merge_type(splitted_merge, repo): """ Possible syntaxes for merges: @@ -106,14 +108,40 @@ def get_merge_type(splitted_merge): return 1 return 0 # unreachable. +def process_depth(splitted_merge, branchname): + lastrev = ( + run( + [ + "git", + "merge-base", + "HEAD", + "".join([splitted_merge[0], "/", branchname]), + ], + stdout=PIPE, + stderr=PIPE, + ) + .stdout.decode("utf-8") + .replace("\n", "") + ) + # we now calculate the needed depth of this branch + mindepth = ( + run( + ["git", "rev-list", "HEAD", "^$" + lastrev, "--count"], + stdout=PIPE, + stderr=PIPE, + ) + .stdout.decode("utf-8") + .replace("\n", "") + ) + return mindepth -def process_merge(doc, repo, merge, index): +def process_merge(doc, repo, merge, index, repo_min_depth): repo_path, splitted_merge = preprocess_merge(doc, repo, merge) # will update all_commits[repo] and current_heads[repo] globals. update_repo_commits_and_heads(repo_path, repo) env = dict(os.environ, **doc.get("ENV", {})) wd = os.getcwd() - merge_type = get_merge_type(splitted_merge) + merge_type = get_merge_type(splitted_merge, repo) if merge_type == 3: # merge_type #3 branchname = get_branchname(splitted_merge) @@ -126,9 +154,11 @@ def process_merge(doc, repo, merge, index): ], stdout=PIPE, stderr=PIPE, - ) + ).stdout.decode("utf-8").replace("\n", "") if splitted_merge[2] != current_pin: splitted_merge[2] = current_pin + min_depth = process_depth(splitted_merge, branchname) + repo_min_depth = min_depth and min_depth > repo_min_depth or repo_min_depth if merge_type == 2: # merge_type line splitted_merge[1] = current_heads.get(repo, "") @@ -151,11 +181,16 @@ def process_merge(doc, repo, merge, index): .replace("\n", "") ) splitted_merge.append(current_pin) + process_depth(splitted_merge, branchname) + min_depth = process_depth(splitted_merge, branchname) + print("++++++++++++" + min_depth) + repo_min_depth = min_depth and min_depth > repo_min_depth or repo_min_depth os.chdir(wd) if merge_type == 0: logger.debug("Invalid Repo of unrecognized merge_type: %s" % splitted_merge) raise ValueError - return " ".join(splitted_merge) + print("DEPTH MERGE " + str(repo_min_depth) + '---------' + str(splitted_merge)) + return " ".join(splitted_merge) , repo_min_depth def main(): @@ -165,16 +200,17 @@ def main(): """ with open(REPOS_YAML) as yaml_file: for doc in yaml.safe_load_all(yaml_file): + min_depth = 0 for repo in doc: if repo in {PRIVATE, "ONLY", "ENV"}: continue - target = doc[repo]["target"] or False + target = doc[repo].get("target") or False has_target = target and True or False processed_target = False if has_target: - doc[repo]["target"] = process_merge(doc, repo, target, 0) + doc[repo]["target"], min_depth = process_merge(doc, repo, target, 0, min_depth) # main branch is defined as target or in absence of target, merge[0] - main_branch = target or merge[0] + main_branch = target or doc[repo]["merges"][0] main_branch = split_line(main_branch)[0] for merge in doc[repo]["merges"]: index = doc[repo]["merges"].index(merge) @@ -184,38 +220,14 @@ def main(): and bool(args.only_base) ) == True: continue - doc[repo]["merges"][index] = process_merge(doc, repo, merge, index) - lastrev = ( - run( - [ - "git", - "merge-base", - "HEAD", - "".join([splitted_merge[0], "/", branchname]), - ], - stdout=PIPE, - stderr=PIPE, - ) - .stdout.decode("utf-8") - .replace("\n", "") - ) - # we now calculate the needed depth of this branch - mindepth = ( - run( - ["git", "rev-list", "HEAD", "^$" + lastrev, "--count"], - stdout=PIPE, - stderr=PIPE, - ) - .stdout.decode("utf-8") - .replace("\n", "") - ) + doc[repo]["merges"][index], min_depth = process_merge(doc, repo, merge, index, min_depth) CustomDumper.add_representer( dict, CustomDumper.represent_dict_preserve_order ) print(yaml.dump(doc, Dumper=CustomDumper)) -if os.path.isfile(REPOS_YAML) and __name__ == "main": +if os.path.isfile(REPOS_YAML) and __name__ == "__main__": main() else: logger.debug("no %s repository file found" % REPOS_YAML) From 1509bd9cdf8f251a8c9042d28bf888b10af71ef9 Mon Sep 17 00:00:00 2001 From: Giovanni Francesco Capalbo Date: Thu, 4 May 2023 19:10:27 +0200 Subject: [PATCH 05/10] f --- bin/get_pins | 128 ++++++++++++++++++++++++++++++++------------------- 1 file changed, 81 insertions(+), 47 deletions(-) diff --git a/bin/get_pins b/bin/get_pins index d629a226..383e7126 100755 --- a/bin/get_pins +++ b/bin/get_pins @@ -20,6 +20,7 @@ load_dotenv(os.path.join(os.environ["ODOO_WORK_DIR"], ".env-default")) load_dotenv(os.path.join(os.environ["ODOO_WORK_DIR"], ".env-shared"), override=True) load_dotenv(os.path.join(os.environ["ODOO_WORK_DIR"], ".env-secret"), override=True) + parser = argparse.ArgumentParser( prog="get_pins", description="Fetch current pins of this instance", @@ -32,7 +33,8 @@ args = parser.parse_args() current_heads = {} all_commits = {} -min_depth = {} +repo_min_depth = {} + class CustomDumper(yaml.Dumper): """ @@ -108,8 +110,53 @@ def get_merge_type(splitted_merge, repo): return 1 return 0 # unreachable. -def process_depth(splitted_merge, branchname): - lastrev = ( + +def process_depth(splitted_merge, branchname, main_branch, main_branch_name, repo_path): + os.chdir(repo_path) + lastrev = ( + run( + [ + "git", + "merge-base", + "".join([main_branch[0], "/", main_branch_name]), + "".join([splitted_merge[0], "/", branchname]), + ], + stdout=PIPE, + stderr=PIPE, + ) + .stdout.decode("utf-8") + .replace("\n", "") + ) + # we now calculate the needed depth of this branch + mindepth = ( + run( + [ + "git", + "rev-list", + "".join([main_branch[0], "/", main_branch_name]), + "^" + lastrev, + "--count", + ], + stdout=PIPE, + stderr=PIPE, + ) + .stdout.decode("utf-8") + .replace("\n", "") + ) + return int(mindepth) + + +def process_merge(doc, repo, merge, index): + repo_path, splitted_merge = preprocess_merge(doc, repo, merge) + # will update all_commits[repo] and current_heads[repo] globals. + update_repo_commits_and_heads(repo_path, repo) + env = dict(os.environ, **doc.get("ENV", {})) + wd = os.getcwd() + merge_type = get_merge_type(splitted_merge, repo) + if merge_type == 3: + # merge_type #3 + branchname = get_branchname(splitted_merge) + current_pin = ( run( [ "git", @@ -123,42 +170,8 @@ def process_depth(splitted_merge, branchname): .stdout.decode("utf-8") .replace("\n", "") ) - # we now calculate the needed depth of this branch - mindepth = ( - run( - ["git", "rev-list", "HEAD", "^$" + lastrev, "--count"], - stdout=PIPE, - stderr=PIPE, - ) - .stdout.decode("utf-8") - .replace("\n", "") - ) - return mindepth - -def process_merge(doc, repo, merge, index, repo_min_depth): - repo_path, splitted_merge = preprocess_merge(doc, repo, merge) - # will update all_commits[repo] and current_heads[repo] globals. - update_repo_commits_and_heads(repo_path, repo) - env = dict(os.environ, **doc.get("ENV", {})) - wd = os.getcwd() - merge_type = get_merge_type(splitted_merge, repo) - if merge_type == 3: - # merge_type #3 - branchname = get_branchname(splitted_merge) - current_pin = run( - [ - "git", - "merge-base", - "HEAD", - "".join([splitted_merge[0], "/", branchname]), - ], - stdout=PIPE, - stderr=PIPE, - ).stdout.decode("utf-8").replace("\n", "") if splitted_merge[2] != current_pin: splitted_merge[2] = current_pin - min_depth = process_depth(splitted_merge, branchname) - repo_min_depth = min_depth and min_depth > repo_min_depth or repo_min_depth if merge_type == 2: # merge_type line splitted_merge[1] = current_heads.get(repo, "") @@ -181,16 +194,11 @@ def process_merge(doc, repo, merge, index, repo_min_depth): .replace("\n", "") ) splitted_merge.append(current_pin) - process_depth(splitted_merge, branchname) - min_depth = process_depth(splitted_merge, branchname) - print("++++++++++++" + min_depth) - repo_min_depth = min_depth and min_depth > repo_min_depth or repo_min_depth os.chdir(wd) if merge_type == 0: logger.debug("Invalid Repo of unrecognized merge_type: %s" % splitted_merge) raise ValueError - print("DEPTH MERGE " + str(repo_min_depth) + '---------' + str(splitted_merge)) - return " ".join(splitted_merge) , repo_min_depth + return " ".join(splitted_merge) def main(): @@ -200,18 +208,19 @@ def main(): """ with open(REPOS_YAML) as yaml_file: for doc in yaml.safe_load_all(yaml_file): - min_depth = 0 for repo in doc: + repo_min_depth[repo] = 0 if repo in {PRIVATE, "ONLY", "ENV"}: continue target = doc[repo].get("target") or False has_target = target and True or False processed_target = False if has_target: - doc[repo]["target"], min_depth = process_merge(doc, repo, target, 0, min_depth) + doc[repo]["target"] = process_merge(doc, repo, target, 0) # main branch is defined as target or in absence of target, merge[0] main_branch = target or doc[repo]["merges"][0] - main_branch = split_line(main_branch)[0] + main_branch = split_line(main_branch) + main_branch_name = get_branchname(main_branch) for merge in doc[repo]["merges"]: index = doc[repo]["merges"].index(merge) # if target exists and only base is selected , just pin target. @@ -220,7 +229,32 @@ def main(): and bool(args.only_base) ) == True: continue - doc[repo]["merges"][index], min_depth = process_merge(doc, repo, merge, index, min_depth) + doc[repo]["merges"][index] = process_merge(doc, repo, merge, index) + repo_path, splitted_merge = preprocess_merge(doc, repo, merge) + branchname = get_branchname(splitted_merge) + min_depth = process_depth( + splitted_merge, + branchname, + main_branch, + main_branch_name, + repo_path, + ) + repo_min_depth[repo] = ( + min_depth > repo_min_depth[repo] + and min_depth + or repo_min_depth[repo] + ) + if repo_min_depth[repo] > 0: + waft_depth = doc[repo]["defaults"].get("depth") + if waft_depth == "${WAFT_DEPTH_MERGE}": + # waft_depth_merge, if not specified in env defaults to 100 + waft_depth = os.environ.get("WAFT_DEPTH_MERGE") or 100 + if waft_depth == "${WAFT_DEPTH_DEFAULT}": + waft_depth = os.environ.get("WAFT_DEPTH_DEFAULT") or 1 + waft_depth = int(waft_depth) + if repo_min_depth[repo] > waft_depth: + doc[repo]["defaults"]["depth"] = str(repo_min_depth[repo]) + CustomDumper.add_representer( dict, CustomDumper.represent_dict_preserve_order ) From 00c78f208463aa7e0b00e4b4d538032e9bea9047 Mon Sep 17 00:00:00 2001 From: Giovanni Francesco Capalbo Date: Tue, 9 May 2023 12:30:28 +0200 Subject: [PATCH 06/10] [ADD] review fixes --- bin/get_pins | 98 +++++++++++++++++++++++++++++----------------------- 1 file changed, 54 insertions(+), 44 deletions(-) diff --git a/bin/get_pins b/bin/get_pins index 383e7126..93655375 100755 --- a/bin/get_pins +++ b/bin/get_pins @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Version: v.22.05.30 +# Version: v.21.05.30 import os import argparse from subprocess import run, PIPE @@ -79,8 +79,10 @@ def preprocess_merge(doc, repo, merge): return repo_path, splitted_merge -def get_branchname(splitted_merge): - return splitted_merge[1].replace("${ODOO_VERSION}", ODOO_VERSION) +def get_branchname(splitted_merge, merge_type): + if merge_type in (1,3): + return splitted_merge[1].replace("${ODOO_VERSION}", ODOO_VERSION) + return False def update_repo_commits_and_heads(repo_path, repo): @@ -111,7 +113,8 @@ def get_merge_type(splitted_merge, repo): return 0 # unreachable. -def process_depth(splitted_merge, branchname, main_branch, main_branch_name, repo_path): +def process_depth(splitted_merge, branchname, + main_branch, main_branch_name, repo_path): os.chdir(repo_path) lastrev = ( run( @@ -155,7 +158,7 @@ def process_merge(doc, repo, merge, index): merge_type = get_merge_type(splitted_merge, repo) if merge_type == 3: # merge_type #3 - branchname = get_branchname(splitted_merge) + branchname = get_branchname(splitted_merge, merge_type) current_pin = ( run( [ @@ -178,22 +181,23 @@ def process_merge(doc, repo, merge, index): if merge_type == 1: # merge_type line # merge_type #1 - branchname = get_branchname(splitted_merge) - current_pin = ( - run( - [ - "git", - "merge-base", - "HEAD", - "".join([splitted_merge[0], "/", branchname]), - ], - stdout=PIPE, - stderr=PIPE, + branchname = get_branchname(splitted_merge, merge_type) + if branchname: + current_pin = ( + run( + [ + "git", + "merge-base", + "HEAD", + "".join([splitted_merge[0], "/", branchname]), + ], + stdout=PIPE, + stderr=PIPE, + ) + .stdout.decode("utf-8") + .replace("\n", "") ) - .stdout.decode("utf-8") - .replace("\n", "") - ) - splitted_merge.append(current_pin) + splitted_merge.append(current_pin) os.chdir(wd) if merge_type == 0: logger.debug("Invalid Repo of unrecognized merge_type: %s" % splitted_merge) @@ -209,6 +213,7 @@ def main(): with open(REPOS_YAML) as yaml_file: for doc in yaml.safe_load_all(yaml_file): for repo in doc: + print("processing repo %s" % repo) repo_min_depth[repo] = 0 if repo in {PRIVATE, "ONLY", "ENV"}: continue @@ -220,8 +225,10 @@ def main(): # main branch is defined as target or in absence of target, merge[0] main_branch = target or doc[repo]["merges"][0] main_branch = split_line(main_branch) - main_branch_name = get_branchname(main_branch) + merge_type = get_merge_type(main_branch, repo) + main_branch_name = get_branchname(main_branch, merge_type ) for merge in doc[repo]["merges"]: + print("processing merge %s of repo %s" % (merge, repo)) index = doc[repo]["merges"].index(merge) # if target exists and only base is selected , just pin target. if ( @@ -231,29 +238,32 @@ def main(): continue doc[repo]["merges"][index] = process_merge(doc, repo, merge, index) repo_path, splitted_merge = preprocess_merge(doc, repo, merge) - branchname = get_branchname(splitted_merge) - min_depth = process_depth( - splitted_merge, - branchname, - main_branch, - main_branch_name, - repo_path, - ) - repo_min_depth[repo] = ( - min_depth > repo_min_depth[repo] - and min_depth - or repo_min_depth[repo] - ) - if repo_min_depth[repo] > 0: - waft_depth = doc[repo]["defaults"].get("depth") - if waft_depth == "${WAFT_DEPTH_MERGE}": - # waft_depth_merge, if not specified in env defaults to 100 - waft_depth = os.environ.get("WAFT_DEPTH_MERGE") or 100 - if waft_depth == "${WAFT_DEPTH_DEFAULT}": - waft_depth = os.environ.get("WAFT_DEPTH_DEFAULT") or 1 - waft_depth = int(waft_depth) - if repo_min_depth[repo] > waft_depth: - doc[repo]["defaults"]["depth"] = str(repo_min_depth[repo]) + merge_type = get_merge_type(splitted_merge, repo) + branchname = get_branchname(splitted_merge, merge_type) + if branchname: + # compute depth only for merges with branchname + min_depth = process_depth( + splitted_merge, + branchname, + main_branch, + main_branch_name, + repo_path, + ) + repo_min_depth[repo] = ( + min_depth > repo_min_depth[repo] + and min_depth + or repo_min_depth[repo] + ) + if repo_min_depth[repo] > 0: + waft_depth = doc[repo]["defaults"].get("depth") + if waft_depth == "${WAFT_DEPTH_MERGE}": + # waft_depth_merge, if not specified in env defaults to 100 + waft_depth = os.environ.get("WAFT_DEPTH_MERGE") or 100 + if waft_depth == "${WAFT_DEPTH_DEFAULT}": + waft_depth = os.environ.get("WAFT_DEPTH_DEFAULT") or 1 + waft_depth = int(waft_depth) + if repo_min_depth[repo] > waft_depth: + doc[repo]["defaults"]["depth"] = str(repo_min_depth[repo]) CustomDumper.add_representer( dict, CustomDumper.represent_dict_preserve_order From 4e890ed8309ba8af7e24915d73c3b62a36fdb0d7 Mon Sep 17 00:00:00 2001 From: Giovanni Francesco Capalbo Date: Tue, 9 May 2023 12:49:20 +0200 Subject: [PATCH 07/10] [ADD] review fixes --- bin/get_pins | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/bin/get_pins b/bin/get_pins index 93655375..1447e7f4 100755 --- a/bin/get_pins +++ b/bin/get_pins @@ -174,6 +174,8 @@ def process_merge(doc, repo, merge, index): .replace("\n", "") ) if splitted_merge[2] != current_pin: + print("\t\t Updating pin on repo %s , branchname %s, from %s to %s" % + (repo , branchname, splitted_merge[2], current_pin )) splitted_merge[2] = current_pin if merge_type == 2: # merge_type line @@ -198,6 +200,9 @@ def process_merge(doc, repo, merge, index): .replace("\n", "") ) splitted_merge.append(current_pin) + print("\t\t Adding pin to previously unpinned branch: on repo %s , branchname %s, %s" % + (repo , branchname, current_pin ) + ) os.chdir(wd) if merge_type == 0: logger.debug("Invalid Repo of unrecognized merge_type: %s" % splitted_merge) @@ -213,7 +218,7 @@ def main(): with open(REPOS_YAML) as yaml_file: for doc in yaml.safe_load_all(yaml_file): for repo in doc: - print("processing repo %s" % repo) + print("===>processing repo %s" % repo) repo_min_depth[repo] = 0 if repo in {PRIVATE, "ONLY", "ENV"}: continue @@ -228,7 +233,7 @@ def main(): merge_type = get_merge_type(main_branch, repo) main_branch_name = get_branchname(main_branch, merge_type ) for merge in doc[repo]["merges"]: - print("processing merge %s of repo %s" % (merge, repo)) + print("\t===>processing merge %s of repo %s" % (merge, repo)) index = doc[repo]["merges"].index(merge) # if target exists and only base is selected , just pin target. if ( @@ -263,11 +268,16 @@ def main(): waft_depth = os.environ.get("WAFT_DEPTH_DEFAULT") or 1 waft_depth = int(waft_depth) if repo_min_depth[repo] > waft_depth: + print("\t\t Increasing depth of %s from %s to %s" % ( + repo, doc[repo]["defaults"]["depth"], + str(repo_min_depth[repo])) + ) doc[repo]["defaults"]["depth"] = str(repo_min_depth[repo]) CustomDumper.add_representer( dict, CustomDumper.represent_dict_preserve_order ) + print("===>=================SCRIPT RESULTS======================\n\n\n") print(yaml.dump(doc, Dumper=CustomDumper)) From 10894920390b6e830cc1d75ddeeae3121ce401d7 Mon Sep 17 00:00:00 2001 From: Giovanni Francesco Capalbo Date: Tue, 9 May 2023 14:33:13 +0200 Subject: [PATCH 08/10] [ADD] refactor executable --- bin/get_pins | 297 ++---------------------------------------------- bin/get_pins.py | 292 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 304 insertions(+), 285 deletions(-) create mode 100644 bin/get_pins.py diff --git a/bin/get_pins b/bin/get_pins index 1447e7f4..d8ffa4d3 100755 --- a/bin/get_pins +++ b/bin/get_pins @@ -1,288 +1,15 @@ -#!/usr/bin/env python -# Version: v.21.05.30 -import os -import argparse -from subprocess import run, PIPE -from dotenv import load_dotenv +#!/bin/sh -import yaml -from waftlib import ( - PRIVATE, - REPOS_YAML, - SRC_DIR, - logger, - ODOO_VERSION, -) +SCRIPT_PATH="$(cd "$(/usr/bin/dirname "${0}")" && /bin/pwd)" +ODOO_WORK_DIR="${SCRIPT_PATH}" +. "${ODOO_WORK_DIR}/.env-default" ; \ +. "${ODOO_WORK_DIR}/.env-shared" ; \ +. "${ODOO_WORK_DIR}/.env-secret"; cd "${ODOO_WORK_DIR}" -SCRIPT_PATH = os.path.abspath(os.path.dirname(__file__)) -os.environ["ODOO_WORK_DIR"] = os.path.realpath(os.path.join(SCRIPT_PATH, "../..")) -load_dotenv(os.path.join(os.environ["ODOO_WORK_DIR"], ".env-default")) -load_dotenv(os.path.join(os.environ["ODOO_WORK_DIR"], ".env-shared"), override=True) -load_dotenv(os.path.join(os.environ["ODOO_WORK_DIR"], ".env-secret"), override=True) +if [ "$(id -u)" -eq 0 ]; then + /bin/echo "Please dont use sudo command with script" + exit 1 +fi - -parser = argparse.ArgumentParser( - prog="get_pins", - description="Fetch current pins of this instance", - epilog="Outputs pinned repo.yaml", -) - -parser.add_argument("--only-base") -args = parser.parse_args() - - -current_heads = {} -all_commits = {} -repo_min_depth = {} - - -class CustomDumper(yaml.Dumper): - """ - We want order of keys intact so that the output is the same order than source. - Unfortunately PyYAML orders yaml output alphabetically on dump. - pyyaml 5.1 has an option to disable alphabetical orderng of dumps, but often our - wafts have a version < 5.1. - for this reason i made a custom dumper to have an "unordered" dump, without - affecting the default behaviour of pyyaml dump. This script supports pyYAML<5.1. - Our output will therefore have the same order as given input. - """ - - def represent_dict_preserve_order(self, data): - return self.represent_dict(data.items()) - - -def split_line(line): - splitted_lines = line.split(" ") - for sl in splitted_lines: - if sl == "": - splitted_lines.pop(splitted_lines.index(sl)) - return splitted_lines - - -def is_in_history(value, history): - for commit in history: - if commit[:8] == value[:8]: - return True - return False - - -def preprocess_merge(doc, repo, merge): - remotes = doc[repo]["remotes"].keys() - splitted_merge = [x for x in merge.split(" ") if x != ""] - if splitted_merge[0] not in remotes: - logger.debug("Invalid Remote on line: %s" % merge) - raise ValueError - repo_path = os.path.abspath(os.path.join(SRC_DIR, repo)) - if not os.path.exists(repo_path): - logger.debug("build incomplete") - exit() - return repo_path, splitted_merge - - -def get_branchname(splitted_merge, merge_type): - if merge_type in (1,3): - return splitted_merge[1].replace("${ODOO_VERSION}", ODOO_VERSION) - return False - - -def update_repo_commits_and_heads(repo_path, repo): - os.chdir(repo_path) - allrev = run(["git", "rev-list", "HEAD"], stdout=PIPE, stderr=PIPE) - lrev = run(["git", "rev-parse", "HEAD"], stdout=PIPE, stderr=PIPE) - # no return updating globals. - current_heads[repo] = lrev.stdout.decode("utf-8").replace("\n", "").replace("'", "") - all_commits[repo] = allrev.stdout.decode("utf-8").replace("\n", ",").split(",") - - -def get_merge_type(splitted_merge, repo): - """ - Possible syntaxes for merges: - - merge_type=1 - merge_type=2 - merge_type=3 - merge_type=0 "invalid" - """ - if len(splitted_merge) == 3: - return 3 - else: - if is_in_history(splitted_merge[1], all_commits.get(repo, [])): - return 2 - else: - return 1 - return 0 # unreachable. - - -def process_depth(splitted_merge, branchname, - main_branch, main_branch_name, repo_path): - os.chdir(repo_path) - lastrev = ( - run( - [ - "git", - "merge-base", - "".join([main_branch[0], "/", main_branch_name]), - "".join([splitted_merge[0], "/", branchname]), - ], - stdout=PIPE, - stderr=PIPE, - ) - .stdout.decode("utf-8") - .replace("\n", "") - ) - # we now calculate the needed depth of this branch - mindepth = ( - run( - [ - "git", - "rev-list", - "".join([main_branch[0], "/", main_branch_name]), - "^" + lastrev, - "--count", - ], - stdout=PIPE, - stderr=PIPE, - ) - .stdout.decode("utf-8") - .replace("\n", "") - ) - return int(mindepth) - - -def process_merge(doc, repo, merge, index): - repo_path, splitted_merge = preprocess_merge(doc, repo, merge) - # will update all_commits[repo] and current_heads[repo] globals. - update_repo_commits_and_heads(repo_path, repo) - env = dict(os.environ, **doc.get("ENV", {})) - wd = os.getcwd() - merge_type = get_merge_type(splitted_merge, repo) - if merge_type == 3: - # merge_type #3 - branchname = get_branchname(splitted_merge, merge_type) - current_pin = ( - run( - [ - "git", - "merge-base", - "HEAD", - "".join([splitted_merge[0], "/", branchname]), - ], - stdout=PIPE, - stderr=PIPE, - ) - .stdout.decode("utf-8") - .replace("\n", "") - ) - if splitted_merge[2] != current_pin: - print("\t\t Updating pin on repo %s , branchname %s, from %s to %s" % - (repo , branchname, splitted_merge[2], current_pin )) - splitted_merge[2] = current_pin - if merge_type == 2: - # merge_type line - splitted_merge[1] = current_heads.get(repo, "") - if merge_type == 1: - # merge_type line - # merge_type #1 - branchname = get_branchname(splitted_merge, merge_type) - if branchname: - current_pin = ( - run( - [ - "git", - "merge-base", - "HEAD", - "".join([splitted_merge[0], "/", branchname]), - ], - stdout=PIPE, - stderr=PIPE, - ) - .stdout.decode("utf-8") - .replace("\n", "") - ) - splitted_merge.append(current_pin) - print("\t\t Adding pin to previously unpinned branch: on repo %s , branchname %s, %s" % - (repo , branchname, current_pin ) - ) - os.chdir(wd) - if merge_type == 0: - logger.debug("Invalid Repo of unrecognized merge_type: %s" % splitted_merge) - raise ValueError - return " ".join(splitted_merge) - - -def main(): - """ - parsing directly repos.yaml, if something is not in addons.yaml, branch will still - be in folder, but may not be included in addons. Nothing changes. - """ - with open(REPOS_YAML) as yaml_file: - for doc in yaml.safe_load_all(yaml_file): - for repo in doc: - print("===>processing repo %s" % repo) - repo_min_depth[repo] = 0 - if repo in {PRIVATE, "ONLY", "ENV"}: - continue - target = doc[repo].get("target") or False - has_target = target and True or False - processed_target = False - if has_target: - doc[repo]["target"] = process_merge(doc, repo, target, 0) - # main branch is defined as target or in absence of target, merge[0] - main_branch = target or doc[repo]["merges"][0] - main_branch = split_line(main_branch) - merge_type = get_merge_type(main_branch, repo) - main_branch_name = get_branchname(main_branch, merge_type ) - for merge in doc[repo]["merges"]: - print("\t===>processing merge %s of repo %s" % (merge, repo)) - index = doc[repo]["merges"].index(merge) - # if target exists and only base is selected , just pin target. - if ( - (index > 0 or (has_target and index >= 0)) - and bool(args.only_base) - ) == True: - continue - doc[repo]["merges"][index] = process_merge(doc, repo, merge, index) - repo_path, splitted_merge = preprocess_merge(doc, repo, merge) - merge_type = get_merge_type(splitted_merge, repo) - branchname = get_branchname(splitted_merge, merge_type) - if branchname: - # compute depth only for merges with branchname - min_depth = process_depth( - splitted_merge, - branchname, - main_branch, - main_branch_name, - repo_path, - ) - repo_min_depth[repo] = ( - min_depth > repo_min_depth[repo] - and min_depth - or repo_min_depth[repo] - ) - if repo_min_depth[repo] > 0: - waft_depth = doc[repo]["defaults"].get("depth") - if waft_depth == "${WAFT_DEPTH_MERGE}": - # waft_depth_merge, if not specified in env defaults to 100 - waft_depth = os.environ.get("WAFT_DEPTH_MERGE") or 100 - if waft_depth == "${WAFT_DEPTH_DEFAULT}": - waft_depth = os.environ.get("WAFT_DEPTH_DEFAULT") or 1 - waft_depth = int(waft_depth) - if repo_min_depth[repo] > waft_depth: - print("\t\t Increasing depth of %s from %s to %s" % ( - repo, doc[repo]["defaults"]["depth"], - str(repo_min_depth[repo])) - ) - doc[repo]["defaults"]["depth"] = str(repo_min_depth[repo]) - - CustomDumper.add_representer( - dict, CustomDumper.represent_dict_preserve_order - ) - print("===>=================SCRIPT RESULTS======================\n\n\n") - print(yaml.dump(doc, Dumper=CustomDumper)) - - -if os.path.isfile(REPOS_YAML) and __name__ == "__main__": - main() -else: - logger.debug("no %s repository file found" % REPOS_YAML) - raise ValueError +. .venv/bin/activate && python waftlib/bin/get_pins.py $@ + diff --git a/bin/get_pins.py b/bin/get_pins.py new file mode 100644 index 00000000..548758d6 --- /dev/null +++ b/bin/get_pins.py @@ -0,0 +1,292 @@ +#!/usr/bin/env python +# Version: v.21.05.30 +import os +import argparse +from string import Template +from subprocess import run, PIPE +from dotenv import load_dotenv + +import yaml +from waftlib import ( + PRIVATE, + REPOS_YAML, + SRC_DIR, + logger, + ODOO_VERSION, +) + +SCRIPT_PATH = os.path.abspath(os.path.dirname(__file__)) +os.environ["ODOO_WORK_DIR"] = os.path.realpath(os.path.join(SCRIPT_PATH, "../..")) +load_dotenv(os.path.join(os.environ["ODOO_WORK_DIR"], ".env-default")) +load_dotenv(os.path.join(os.environ["ODOO_WORK_DIR"], ".env-shared"), override=True) +load_dotenv(os.path.join(os.environ["ODOO_WORK_DIR"], ".env-secret"), override=True) + + +parser = argparse.ArgumentParser( + prog="get_pins", + description="Fetch current pins of this instance", + epilog="Outputs pinned repo.yaml", +) + +parser.add_argument("--only-base") +args = parser.parse_args() + + +current_heads = {} +all_commits = {} +repo_min_depth = {} + + +class CustomDumper(yaml.Dumper): + """ + We want order of keys intact so that the output is the same order than source. + Unfortunately PyYAML orders yaml output alphabetically on dump. + pyyaml 5.1 has an option to disable alphabetical orderng of dumps, but often our + wafts have a version < 5.1. + for this reason i made a custom dumper to have an "unordered" dump, without + affecting the default behaviour of pyyaml dump. This script supports pyYAML<5.1. + Our output will therefore have the same order as given input. + """ + + def represent_dict_preserve_order(self, data): + return self.represent_dict(data.items()) + + +def split_line(line): + splitted_lines = line.split(" ") + for sl in splitted_lines: + if sl == "": + splitted_lines.pop(splitted_lines.index(sl)) + return splitted_lines + + +def is_in_history(value, history): + for commit in history: + if commit[:8] == value[:8]: + return True + return False + + +def preprocess_merge(doc, repo, merge): + remotes = doc[repo]["remotes"].keys() + splitted_merge = [x for x in merge.split(" ") if x != ""] + if splitted_merge[0] not in remotes: + logger.debug("Invalid Remote on line: %s" % merge) + raise ValueError + repo_path = os.path.abspath(os.path.join(SRC_DIR, repo)) + if not os.path.exists(repo_path): + logger.debug("build incomplete") + exit() + return repo_path, splitted_merge + + +def get_branchname(splitted_merge, merge_type): + if merge_type in (1,3): + return splitted_merge[1].replace("${ODOO_VERSION}", ODOO_VERSION) + return False + + +def update_repo_commits_and_heads(repo_path, repo): + os.chdir(repo_path) + allrev = run(["git", "rev-list", "HEAD"], stdout=PIPE, stderr=PIPE) + lrev = run(["git", "rev-parse", "HEAD"], stdout=PIPE, stderr=PIPE) + # no return updating globals. + current_heads[repo] = lrev.stdout.decode("utf-8").replace("\n", "").replace("'", "") + all_commits[repo] = allrev.stdout.decode("utf-8").replace("\n", ",").split(",") + + +def get_merge_type(splitted_merge, repo): + """ + Possible syntaxes for merges: + + merge_type=1 + merge_type=2 + merge_type=3 + merge_type=0 "invalid" + """ + if len(splitted_merge) == 3: + return 3 + else: + if is_in_history(splitted_merge[1], all_commits.get(repo, [])): + return 2 + else: + return 1 + return 0 # unreachable. + + +def process_depth(splitted_merge, branchname, + main_branch, main_branch_name, repo_path): + os.chdir(repo_path) + lastrev = ( + run( + [ + "git", + "merge-base", + "".join([main_branch[0], "/", main_branch_name]), + "".join([splitted_merge[0], "/", branchname]), + ], + stdout=PIPE, + stderr=PIPE, + ) + .stdout.decode("utf-8") + .replace("\n", "") + ) + # we now calculate the needed depth of this branch + mindepth = ( + run( + [ + "git", + "rev-list", + "".join([main_branch[0], "/", main_branch_name]), + "^" + lastrev, + "--count", + ], + stdout=PIPE, + stderr=PIPE, + ) + .stdout.decode("utf-8") + .replace("\n", "") + ) + return int(mindepth) + + +def process_merge(doc, repo, merge, index): + repo_path, splitted_merge = preprocess_merge(doc, repo, merge) + # will update all_commits[repo] and current_heads[repo] globals. + update_repo_commits_and_heads(repo_path, repo) + env = dict(os.environ, **doc.get("ENV", {})) + wd = os.getcwd() + merge_type = get_merge_type(splitted_merge, repo) + if merge_type == 3: + # merge_type #3 + branchname = get_branchname(splitted_merge, merge_type) + current_pin = ( + run( + [ + "git", + "merge-base", + "HEAD", + "".join([splitted_merge[0], "/", branchname]), + ], + stdout=PIPE, + stderr=PIPE, + ) + .stdout.decode("utf-8") + .replace("\n", "") + ) + if splitted_merge[2] != current_pin: + print("\t\t Updating pin on repo %s , branchname %s, from %s to %s" % + (repo , branchname, splitted_merge[2], current_pin )) + splitted_merge[2] = current_pin + if merge_type == 2: + # merge_type line + splitted_merge[1] = current_heads.get(repo, "") + if merge_type == 1: + # merge_type line + # merge_type #1 + branchname = get_branchname(splitted_merge, merge_type) + if branchname: + current_pin = ( + run( + [ + "git", + "merge-base", + "HEAD", + "".join([splitted_merge[0], "/", branchname]), + ], + stdout=PIPE, + stderr=PIPE, + ) + .stdout.decode("utf-8") + .replace("\n", "") + ) + splitted_merge.append(current_pin) + print("\t\t Adding pin to previously unpinned branch: on repo %s , branchname %s, %s" % + (repo , branchname, current_pin ) + ) + os.chdir(wd) + if merge_type == 0: + logger.debug("Invalid Repo of unrecognized merge_type: %s" % splitted_merge) + raise ValueError + return " ".join(splitted_merge) + + +def main(): + """ + parsing directly repos.yaml, if something is not in addons.yaml, branch will still + be in folder, but may not be included in addons. Nothing changes. + """ + with open(REPOS_YAML) as yaml_file: + yaml_substituted = Template(yaml_file.read()) + yaml_substituted = yaml_substituted.substitute(os.environ) + for doc in yaml.safe_load_all(yaml_substituted): + for repo in doc: + print("===>processing repo %s" % repo) + repo_min_depth[repo] = 0 + if repo in {PRIVATE, "ONLY", "ENV"}: + continue + target = doc[repo].get("target") or False + has_target = target and True or False + processed_target = False + if has_target: + doc[repo]["target"] = process_merge(doc, repo, target, 0) + # main branch is defined as target or in absence of target, merge[0] + main_branch = target or doc[repo]["merges"][0] + main_branch = split_line(main_branch) + merge_type = get_merge_type(main_branch, repo) + main_branch_name = get_branchname(main_branch, merge_type ) + for merge in doc[repo]["merges"]: + print("\t===>processing merge %s of repo %s" % (merge, repo)) + index = doc[repo]["merges"].index(merge) + # if target exists and only base is selected , just pin target. + if ( + (index > 0 or (has_target and index >= 0)) + and bool(args.only_base) + ) == True: + continue + doc[repo]["merges"][index] = process_merge(doc, repo, merge, index) + repo_path, splitted_merge = preprocess_merge(doc, repo, merge) + merge_type = get_merge_type(splitted_merge, repo) + branchname = get_branchname(splitted_merge, merge_type) + if branchname: + # compute depth only for merges with branchname + min_depth = process_depth( + splitted_merge, + branchname, + main_branch, + main_branch_name, + repo_path, + ) + repo_min_depth[repo] = ( + min_depth > repo_min_depth[repo] + and min_depth + or repo_min_depth[repo] + ) + if repo_min_depth[repo] > 0: + waft_depth = doc[repo]["defaults"].get("depth") + if waft_depth == "${WAFT_DEPTH_MERGE}": + # waft_depth_merge, if not specified in env defaults to 100 + waft_depth = os.environ.get("WAFT_DEPTH_MERGE") or 100 + if waft_depth == "${WAFT_DEPTH_DEFAULT}": + waft_depth = os.environ.get("WAFT_DEPTH_DEFAULT") or 1 + waft_depth = int(waft_depth) + if repo_min_depth[repo] > waft_depth: + print("\t\t Increasing depth of %s from %s to %s" % ( + repo, doc[repo]["defaults"]["depth"], + str(repo_min_depth[repo])) + ) + doc[repo]["defaults"]["depth"] = str(repo_min_depth[repo]) + + CustomDumper.add_representer( + dict, CustomDumper.represent_dict_preserve_order + ) + print("\n\n\n====================SCRIPT RESULTS======================\n\n\n") + print(yaml.dump(doc, Dumper=CustomDumper)) + + +if os.path.isfile(REPOS_YAML) and __name__ == "__main__": + main() +else: + logger.debug("no %s repository file found" % REPOS_YAML) + raise ValueError + From 987406a5257159bf0e2ab40f5a57efac372d5620 Mon Sep 17 00:00:00 2001 From: Giovanni Francesco Capalbo Date: Tue, 9 May 2023 15:20:29 +0200 Subject: [PATCH 09/10] [ADD] get pins exec in right place --- bin/get_pins => get_pins | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename bin/get_pins => get_pins (100%) diff --git a/bin/get_pins b/get_pins similarity index 100% rename from bin/get_pins rename to get_pins From 106322aa0b5d68c57edeac8b4301e9b76ca661d4 Mon Sep 17 00:00:00 2001 From: Giovanni Francesco Capalbo Date: Thu, 7 Sep 2023 16:21:54 +0200 Subject: [PATCH 10/10] fix --- bin/get_pins.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/bin/get_pins.py b/bin/get_pins.py index 548758d6..968333e1 100644 --- a/bin/get_pins.py +++ b/bin/get_pins.py @@ -147,8 +147,13 @@ def process_depth(splitted_merge, branchname, .stdout.decode("utf-8") .replace("\n", "") ) - return int(mindepth) - + try: + return int(mindepth) + except: + # if for some reason we have no common ancestor or rev-list errors out + import pudb + pudb.set_trace() + return 5000 def process_merge(doc, repo, merge, index): repo_path, splitted_merge = preprocess_merge(doc, repo, merge)