From 8faa757ae11eae8926c5386df6aa10ab488cca37 Mon Sep 17 00:00:00 2001 From: kirchsth Date: Sat, 22 Apr 2023 18:53:55 +0200 Subject: [PATCH] #275 enable semi automatic release process (#302) ./.scripts/HowToCreateANewRelease.md contains all required steps that - a new release version can be created and - deployed to plantuml-stdlib repository --- .scripts/HowToCreateANewRelease.md | 198 +++++++++++++ .scripts/plantuml_stdlib_info.txt | 2 + .scripts/readme_release_header.txt | 18 ++ .scripts/transform_files.py | 437 +++++++++++++++++++++++++++++ 4 files changed, 655 insertions(+) create mode 100644 .scripts/HowToCreateANewRelease.md create mode 100644 .scripts/plantuml_stdlib_info.txt create mode 100644 .scripts/readme_release_header.txt create mode 100644 .scripts/transform_files.py diff --git a/.scripts/HowToCreateANewRelease.md b/.scripts/HowToCreateANewRelease.md new file mode 100644 index 0000000..245218f --- /dev/null +++ b/.scripts/HowToCreateANewRelease.md @@ -0,0 +1,198 @@ +# How to create a new release (WIP) + +The idea is to +- create a new release in the "release/$release_version" branch (tagged with `$release_version` and `latest`) +- and a MR that the master branch can be updated with next beta version +- create in PlantUML/PlantUML-stdlib a MR with the released version + +## 0. Preparation + +The process requires following 3 versions: + +- **$release_version**: version which should be created (e.g. `v2.6.0`) +- **$next_version**: version of the next beta which should be stared + as soon the release is created (e.g. `v2.7.0`). + The master branch will be updated with a `beta1` of this version and C4Version() returns `2.7.0beta1`. + If it is unknown/undefined it is calculated via the release_version. It is the next patch (release patch!=0) or subversion (release patch==0). +- **$deployed_version**: this is the next "plantuml(/plantuml-stdlib)" version + which should be updated with this release (e.g. "V1.2023.2") + If it is unknown/undefined it is calculated via the running PlantUML web service + +### 0.0 Create a new issue with the title `Release $release_version` \(e.g. `Release v2.6.0`) + +and a body like in https://github.com/plantuml-stdlib/C4-PlantUML/issues/248 + +### 0.1 Check that all open issues of the related `$release_version milestone` are fixed + +### 0.2 Check that all other open changes are done + +Update copyright year, contrib files, URLS, .... if required + +### 0.* ... + +### 0.x Check which is the next released version of the PlantUML(/PlantUML-stdlib) + +it is used as $deployed_version and written in the released README.md +If it is unknown it can be calculated via `CalculateDeployedVersion` (details see below) + +## 1. create new release in branch `release/$release_version` (based on master) + +Atm following steps are semi-automated and can be executed in a bash shell: + +### 1.0. define the relevant versions as environment variabels and create the `release/$release_version` branch + +\(in following sample the `release_version` = `v2.6.0`; `next_version` = `v2.7.0` and `deployed_version` = `V1.2022.15`) + +```bash +export release_version=v2.6.0 +export next_version=v2.7.0 +export deployed_version=V1.2022.15 +``` + +If the deployed_version is unknown it can be calculate via following (don't forget to set the environment variable after the call) + +```bash +python ./.scripts/transform_files.py CalculateDeployedVersion +``` + +As soon all versions are defined the `release/$release_version` branch \(e.g. `release/v2.6.0`) can be created based on master branch + +```bash +git pull +git checkout master +git branch release/$release_version +git checkout release/$release_version +``` + +### 1.1. Update `C4Version()` in C4.puml with the new release (e.g. `2.6.0`; without `v`) + +```bash +python ./.scripts/transform_files.py UpdateC4WithReleaseVersion +``` + +### 1.2. Update all include paths and create a release version of the README.md + +Following script calls + +- Update all include paths with the release version tag based branch +- Update includes of all image urls with the release version tag based in *.md + (after that images displays the correct C4Version() number and uses only the release-tag path in all includes) +- Update README.md with the new release header and title (based on the `readme_release_header.txt` template) + +```bash +python ./.scripts/transform_files.py UpdateC4WithReleaseVersion +python ./.scripts/transform_files.py UpdateAllImages +python ./.scripts/transform_files.py ReplaceREADMEHeader +``` + +These changes can/should be checked and if everything is ok it can be committed. + +Following commit all changes and tag it locally with `$release_version` (e.g. `v2.5.0`) + +```bash +git checkout release/$release_version +git add -u **/*.md +git add -u **/*.puml +git commit -m "Create release (branch) $release_version" +git tag "$release_version" +``` + +And if everything is ok it can be pushed too + +```bash +git checkout release/$release_version +git push -u origin release/$release_version +``` + +## 2. Create `Release $release_version` \(e.g. `Release v2.6.0`) itself + +This is done manually \(incl. an additional check...) + +**Important:** As soon the release is finished check that +- 'latest' tag is re-assigned to the new release branch +- and '$release_version' tag is assigned to the new release branch + +As soon the version is released the release branch has to be write protected + +## 3. Update master branch with $next_version beta1 version + +### 3.1. create a `start-$next_version-beta1` branch \(e.g. `start-v2.7.0-beta1`) based on master branch + +```bash +git pull +git checkout master +git branch start-$next_version-beta1 +git checkout start-$next_version-beta1 +``` + +Following script update `C4Version()` in C4.puml with the next beta release (e.g. `2.7.0beta1`; without `v`) + +```bash +python ./.scripts/transform_files.py UpdateC4WithNextBeta +``` + +Following commit all changes + +```bash +git checkout start-$next_version-beta1 +git add -u C4.puml +git commit -m "Update version with first beta of $next_version ($release_version was created based on previous commit)" +``` + +And if everything is ok it can be pushed too + +```bash +git checkout start-$next_version-beta1 +git push -u origin start-$next_version-beta1 +``` + +### 3.2. Create a MR into master + +This is done manually \(incl. an additional check...) + +## 4. Create in PlantUML/PlantUML-stdlib a MR based on `release/$release_version` branch + +> !!! CHECK that the correct release branch is activated !!! + +> It is assumed that following calls are started in "C4-PlantUML repository" folder +(and not in the "plantuml-stdlib repository" folder) + +The process requires following information too: + +- **$deploy_repository_folder**: folder with the local `PlantUML/PlantUML-stdlib` git repositiory. + E.g. if it is parallel to the C4-PlantUML repository the it could be `../plantuml-stdlib`. + +### 4.1. create in a PlantUML/PlantUML-stdlib fork a `C4$release_version` branch (e.g. `C4v2.6.0`) + +```bash +export release_version=v2.6.0 +export next_version=v2.7.0 +export deployed_version=V1.2022.15 + +export deploy_repository_folder=../plantuml-stdlib + +git pull +git checkout release/$release_version + +git -C $deploy_repository_folder pull +git -C $deploy_repository_folder checkout master +git -C $deploy_repository_folder branch C4$release_version +git -C $deploy_repository_folder checkout C4$release_version +``` + +### 4.2. prepare the C4_*.puml and INFO file + +```bash +python ./.scripts/transform_files.py CreatePlantUMLStdlibC4Folder $deploy_repository_folder/C4 +``` + +### 4.3. Commit changes with comment "Update C4-PlantUML to $release_version" + +```bash +git -C $deploy_repository_folder add -u C4/** +git -C $deploy_repository_folder commit -m "Update C4-PlantUML to $release_version" +``` + +### 4.4. create a MR "Update C4-PlantUML to $next_version" + +This is done manually \(incl. an additional check...) diff --git a/.scripts/plantuml_stdlib_info.txt b/.scripts/plantuml_stdlib_info.txt new file mode 100644 index 0000000..dd71640 --- /dev/null +++ b/.scripts/plantuml_stdlib_info.txt @@ -0,0 +1,2 @@ +VERSION={release version without v} +SOURCE=https://github.com/plantuml-stdlib/C4-PlantUML diff --git a/.scripts/readme_release_header.txt b/.scripts/readme_release_header.txt new file mode 100644 index 0000000..2b74c2b --- /dev/null +++ b/.scripts/readme_release_header.txt @@ -0,0 +1,18 @@ +[![release][Release Badge]][Release Page] +[![license MIT][License Badge]][License Page] +       +[![integrated in PlantUML][Integrated Badge]][Integrated Page] +       +[![commits since][Commits Since Badge]][Commit Page] + +[Release Badge]: https://img.shields.io/badge/release-{release version}-blue +[Release Page]: https://github.com/plantuml-stdlib/C4-PlantUML/releases/{release version} +[License Badge]: https://img.shields.io/github/license/plantuml-stdlib/C4-PlantUML +[License Page]: https://github.com/plantuml-stdlib/C4-PlantUML/blob/master/LICENSE +[Integrated Badge]: https://img.shields.io/badge/C4--PlantUML%20%20{release version}%20integrated%20in%20PlantUML%20Standard%20Library-{deploy into version}-orange +[Integrated Page]: https://plantuml.com/stdlib#062f75176513a666 + +[Commits Since Badge]: https://img.shields.io/github/commits-since/plantuml-stdlib/C4-PlantUML/latest?label=new%20unreleased%20changes%20in%20master%20branch +[Commit Page]: https://github.com/plantuml-stdlib/C4-PlantUML/commits + +# C4-PlantUML ({release version}) \ No newline at end of file diff --git a/.scripts/transform_files.py b/.scripts/transform_files.py new file mode 100644 index 0000000..d29045e --- /dev/null +++ b/.scripts/transform_files.py @@ -0,0 +1,437 @@ +#!/usr/bin/python + +# If a new version is released, version number and other topics of the source +# has to be updated. +# This script simplifies the update from master to a specific release (branch/tag) + +# IMPORTANT: +# +# - It is assumed that the script is stared in repository root with relative path +# python ./.scripts/transform_files +# +# - It is assumed that +# - "release_version" (e.g. "v2.6.0"), +# - "next_version" (e.g. "v2.7.0") +# If it is undefined it will be calculated via the release_version. +# It is the next patch (release patch!=0) or subversion (release patch==0). +# - and "deployed_version" This is the next "plantuml(/plantuml-stdlib)" version +# which should be updated with this release (e.g. "V1.2023.2") +# If it is undefined it is calculated via the running PlantUML web service +# are defined as environment variable (or they will be calculated if possible) + +# Supported transformations/functions are +# +# - CalculateDeployedVersion +# +# - UpdateC4WithReleaseVersion +# - UpdateAllIncludes +# - UpdateAllImages +# - ReplaceREADMEHeader +# - UpdateC4WithNextBeta +# +# - CreatePlantUMLStdlibC4Folder [] + +import os +import re +import sys +import glob + +import zlib +import base64 +import string + +import requests + +# >>>>> plant uml decoder from ryardley/plant_uml_decoder.py +# >>>>> https://gist.github.com/ryardley/64816f5097003a35f9726aab676920d0 + +plantuml_alphabet = ( + string.digits + string.ascii_uppercase + string.ascii_lowercase + "-_" +) +base64_alphabet = string.ascii_uppercase + string.ascii_lowercase + string.digits + "+/" +b64_to_plantuml = bytes.maketrans( + base64_alphabet.encode("utf-8"), plantuml_alphabet.encode("utf-8") +) +plantuml_to_b64 = bytes.maketrans( + plantuml_alphabet.encode("utf-8"), base64_alphabet.encode("utf-8") +) + + +def plantuml_encode(plantuml_text): + """zlib compress the plantuml text and encode it for the plantuml server""" + zlibbed_str = zlib.compress(plantuml_text.encode("utf-8")) + compressed_string = zlibbed_str[2:-4] + return ( + base64.b64encode(compressed_string).translate(b64_to_plantuml).decode("utf-8") + ) + + +def plantuml_decode(plantuml_url): + """decode plantuml encoded url back to plantuml text""" + data = base64.b64decode(plantuml_url.translate(plantuml_to_b64).encode("utf-8")) + dec = zlib.decompressobj() # without check the crc. + header = b"x\x9c" + return dec.decompress(header + data).decode("utf-8") + + +# <<<<< plant uml decoder from ryardley/plant_uml_decoder.py + + +def read_environment_variable(env_var, is_required=True): + if env_var not in os.environ: + if is_required: + sys.stderr.write( + f"the required environment variable {env_var} is not defined\n" + ) + sys.exit(3) + else: + return "" + return os.environ[env_var] + + +# It is assumed that "release_version", "next_version" and "deployed_version" +# are defined as environment variable. +# If next_version is not defined then it is calculated based on release_version +# If deployed_version is not defined then it is calculated based on the running PlantUML web server +def read_environment_variables(): + global release_version + release_version = read_environment_variable("release_version") + if release_version[0] != "v": + sys.stderr.write( + f"release version {release_version} has to start with 'v' (and use a format like vX.Y.Z)" + ) + sys.exit(2) + release_match = re.search( + r"^v(?P[0-9]+)\.(?P[0-9]+)\.(?P[0-9]+)$", release_version + ) + if not release_match: + sys.stderr.write( + f"release version {release_version} has to use a format like v[0-9]+.[0-9]+.[0-9]+, e.g. v2.6.0)" + ) + sys.exit(2) + + global next_version + next_version = read_environment_variable("next_version", False) + if next_version == "": + v1 = int(release_match["v1"]) + v2 = int(release_match["v2"]) + v3 = int(release_match["v3"]) + next_version = calculate_next_version(release_version, v1, v2, v3) + + global deployed_version + deployed_version = read_environment_variable("deployed_version", False) + if deployed_version == "": + deployed_version = read_next_plantuml_version() + + +def replace_first_regex_in_file(file_path, search, replace): + r = re.compile(search) + with open(file_path, "r") as file: + filedata = file.read() + filedata = r.sub(replace, filedata, 1) + with open(file_path, "w") as file: + file.write(filedata) + + +def replace_in_file(file_path, orig, replace): + with open(file_path, "r") as file: + filedata = file.read() + filedata = filedata.replace(orig, replace) + with open(file_path, "w") as file: + file.write(filedata) + + +def replace_first_regex_copy_file( + source_path, target_path, compiled_search_regex, replace +): + with open(source_path, "r") as file: + filedata = file.read() + filedata = compiled_search_regex.sub(replace, filedata, 1) + with open(target_path, "w") as file: + file.write(filedata) + + +# Calculates the next version (inclusive starting v) based on the give version values. +# If v3 == 0 then v2 is increased otherwise v3 +def calculate_next_version(release, v1, v2, v3): + print(f"calculates the next_version based on given release_version {release} ...") + if v3 == 0: + v2 = v2 + 1 + else: + v3 = v3 + 1 + version = f"v{v1}.{v2}.{v3}" + print( + f"The calculated next_version = {version}. It can be used as next_version environment variable with following statement" + ) + print(f" export next_version={version}") + return version + + +# Calculates the next released PlantUML version that it can be used as deployed_version environment variable +# based on http://www.plantuml.com/plantuml/svg/SoWkIImgAStDuSf8JKn9BL9GBKijAixCpzFGv798pKi1oW00 response +# This function returns "V" + the parsed version number (e.g. "V1.2022.16") +def read_next_plantuml_version(): + # the idea is that the PlantUML version is extracted out of the svg result of "header %version()". + # %version() stores beta of next version. + # the returned SVG response stores the version inclusive beta in a text element; e.fg. "...1.2022.16beta2..." + # and this function returns "V" + the parsed version number (e.g. "V1.2022.16") + print( + "calculates the next deployed_version based on the running PlantUML web server response ..." + ) + resp = requests.get( + "http://www.plantuml.com/plantuml/svg/SoWkIImgAStDuSf8JKn9BL9GBKijAixCpzFGv798pKi1oW00" + ) + if resp.status_code != 200: + sys.stderr.write( + "cannot read the svg response (with the next release version) from the PlantUML web server; please check http://www.plantuml.com/plantuml/svg/SoWkIImgAStDuSf8JKn9BL9GBKijAixCpzFGv798pKi1oW00" + ) + sys.exit(4) + + # As an alternative it could be calculated via https://www.planttext.com/api/plantuml/txt/SoWkIImgIIvApSz9vL8jIoqgpipFqz3aSaZDIu680W00 + # It would return "1.2022.15beta6\n". (details see https://forum.plantuml.net/17179/ascii-art-title-produces-java-lang-illegalstateexception?show=17184#a17184) + + svgbody = resp.content + svg = svgbody.decode("utf-8") + # this regex ignore beta2 of the text section too: "]+>(?P[0-9\.]+)" + r = re.compile(r"]+>(?P[0-9\.]+)") + v = r.search(svg)["version"] + version = "V" + v + + print( + f"The next PlantUML version = {version}. It can be used as deployed_version environment variable with following statement" + ) + print(f" export deployed_version={version}") + + return version + + +def update_c4_release_version(): + # $c4Version is defined without starting 'v' + print( + f"updating C4Version() definition in C4.puml with new release_version {release_version[1:]} ..." + ) + replace_first_regex_in_file( + "C4.puml", r'!\$c4Version = ".+"', f'!$c4Version = "{release_version[1:]}"' + ) + print("C4Version() updated") + + +def update_c4_next_beta_version(): + # $c4Version is defined without starting 'v' + print( + f"updating C4Version() definition in C4.puml with new release_version {release_version[1:]} ..." + ) + replace_first_regex_in_file( + "C4.puml", r'!\$c4Version = ".+"', f'!$c4Version = "{next_version[1:]}beta1"' + ) + print("C4Version() updated") + + +def update_all_includes(): + # reference tag version is with starting 'v' + print(f"updating include paths with new tag version {release_version} ...") + files = glob.glob("./**/*", recursive=True) + for file in files: + if file.endswith(".puml") or file.endswith(".md"): + print(f" {file}") + replace_in_file( + file, + "!include https://raw.githubusercontent.com/plantuml-stdlib/C4-PlantUML/master/", + f"!include https://raw.githubusercontent.com/plantuml-stdlib/C4-PlantUML/{release_version}/", + ) + + print("include paths updated") + + +def process_url_match(m: re.Match[str]): + base = m.group("base") + out_format = m.group("format") + base64 = m.group("base64") + text = plantuml_decode(base64) + + new_path = f"!include https://raw.githubusercontent.com/plantuml-stdlib/C4-PlantUML/{release_version}/" + replaced = text.replace( + "!include https://raw.githubusercontent.com/plantuml-stdlib/C4-PlantUML/master/", + new_path, + ) + + if new_path not in replaced: + global found_not_replaced_include + found_not_replaced_include = True + print( + f"!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\ninclude could not be replaced in base64\n{base64}\nthe extracted source is\n{text}\n------------------------------------" + ) + updated_base64 = base64 + else: + updated_base64 = plantuml_encode(replaced) + + return f"{base}/{out_format}/{updated_base64}" + + +def replace_images_in_file(file_path): + # extract all base64 entries + r = re.compile( + "(?Phttps://www\\.plantuml\\.com/plantuml)/(?P(png|uml|svg))/(?P([^ )]*))" + ) + with open(file_path, "r") as file: + filedata = file.read() + filedata = r.sub(process_url_match, filedata) + with open(file_path, "w") as file: + file.write(filedata) + + +def update_all_images(): + # reference tag version is with starting 'v' + print( + f"updating include paths with new tag version {release_version} in images of all *.md files ..." + ) + + global found_not_replaced_include + found_not_replaced_include = False + + files = glob.glob("./**/*", recursive=True) + for file in files: + if file.endswith(".md"): + print(f" {file}") + replace_images_in_file(file) + + if found_not_replaced_include: + sys.stderr.write( + "!!!!!! not all images urls could be updated in the *.md files (details see log)\n" + ) + sys.exit(3) + + print("include paths in images updated") + + +def replace_readme_header(): + print(f"updating README.md with new version {release_version} and badges ...") + # remove whole part before "# C4-PlantUML" in README.md + r = re.compile(r"[^\#]+# C4-PlantUML", re.M) + with open("README.md", "r") as file: + filedata = file.read() + filedata = r.sub("# C4-PlantUML", filedata) + + with open("./.scripts/readme_release_header.txt", "r") as header_file: + header = header_file.read() + header = header.replace("{release version}", release_version).replace( + "{deploy into version}", deployed_version + ) + + filedata = filedata.replace("# C4-PlantUML", header) + + with open("README.md", "w") as file: + file.write(filedata) + + print("release README.md updated with new version and badges") + + +def create_plantuml_stdlib_c4_folder(target_path): + print( + f"prepare C4 folder of plantuml-stdlib repository in folder {target_path} ..." + ) + # remove whole begin inclusive "!endif" in the specific C4_*.puml files + inclusive_endif = re.compile(r"'[^']+!endif", re.M) + + os.makedirs(target_path, exist_ok=True) + replace_first_regex_copy_file( + "C4.puml", + os.path.join(target_path, "C4.puml"), + re.compile("DOES NOT EXIST"), + "DOES NOT EXIST", + ) + replace_first_regex_copy_file( + "C4_Component.puml", + os.path.join(target_path, "C4_Component.puml"), + inclusive_endif, + "!include ", + ) + replace_first_regex_copy_file( + "C4_Container.puml", + os.path.join(target_path, "C4_Container.puml"), + inclusive_endif, + "!include ", + ) + replace_first_regex_copy_file( + "C4_Context.puml", + os.path.join(target_path, "C4_Context.puml"), + inclusive_endif, + "!include ", + ) + replace_first_regex_copy_file( + "C4_Deployment.puml", + os.path.join(target_path, "C4_Deployment.puml"), + inclusive_endif, + "!include ", + ) + replace_first_regex_copy_file( + "C4_Dynamic.puml", + os.path.join(target_path, "C4_Dynamic.puml"), + inclusive_endif, + "!include ", + ) + + replace_first_regex_copy_file( + "./.scripts/plantuml_stdlib_info.txt", + os.path.join(target_path, "INFO"), + re.compile(r"\{release version without v\}"), + release_version[1:], + ) + + themes_path = target_path+"/themes" + os.makedirs(themes_path, exist_ok=True) + paths = glob.glob("./themes/puml-theme-C4_*.puml") + for path in paths: + file = os.path.basename(path) + if file == "puml-theme-C4_FirstTest.puml": + continue + # print(f" {path}") + replace_first_regex_copy_file( + path, + os.path.join(themes_path, file), + re.compile("DOES NOT EXIST"), + "DOES NOT EXIST", + ) + + print(f"all C4 related plantuml-stdlib files copied into {target_path}.") + + +if not ( + len(sys.argv) == 2 + or (len(sys.argv) == 3 and sys.argv[1] == "CreatePlantUMLStdlibC4Folder") +): + u = "Usage: python ./.scripts/transform_files.py \n" + sys.stderr.write("Usage: python ./.scripts/transform_files.py \n") + sys.exit(1) + +if sys.argv[0] != "./.scripts/transform_files.py": + u = "script has to be started in repository root with relative path: ./.scripts/transform_files \n" + sys.stderr.write(u) + sys.exit(1) + +if sys.argv[1] == "UpdateC4WithReleaseVersion": + read_environment_variables() + update_c4_release_version() +elif sys.argv[1] == "UpdateAllIncludes": + read_environment_variables() + update_all_includes() +elif sys.argv[1] == "UpdateAllImages": + read_environment_variables() + update_all_images() +elif sys.argv[1] == "ReplaceREADMEHeader": + read_environment_variables() + replace_readme_header() +elif sys.argv[1] == "UpdateC4WithNextBeta": + read_environment_variables() + update_c4_next_beta_version() +elif sys.argv[1] == "CalculateDeployedVersion": + calculated_deployed_version = read_next_plantuml_version() +elif sys.argv[1] == "CreatePlantUMLStdlibC4Folder": + read_environment_variables() + if len(sys.argv) == 3: + create_plantuml_stdlib_c4_folder(sys.argv[2]) + else: + create_plantuml_stdlib_c4_folder(".plantuml_stdlib_c4") +else: + sys.stderr.write(f"{sys.argv[1]} is an unsupported transformation\n") + sys.exit(1)