[Actions] Add workflow utilities scripts.

This commit is contained in:
Kuba Szczodrzyński 2021-04-04 21:39:42 +02:00
parent 5b35e3500e
commit 8f9861bac6
10 changed files with 720 additions and 0 deletions

2
.github/utils/.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
.env
__pycache__/

57
.github/utils/_get_password.py vendored Normal file
View File

@ -0,0 +1,57 @@
import base64
import secrets
from hashlib import sha256
from typing import Tuple
import mysql.connector as mysql
from Crypto.Cipher import AES
def get_password(
version_name: str,
version_code: int,
db_host: str,
db_user: str,
db_pass: str,
db_name: str,
) -> Tuple[str, bytes]:
db = mysql.connect(
host=db_host,
user=db_user,
password=db_pass,
database=db_name,
auth_plugin="mysql_native_password",
)
print(f"Generating passwords for version {version_name} ({version_code})")
password = base64.b64encode(secrets.token_bytes(16)).decode()
iv = secrets.token_bytes(16)
key = f"{version_name}.{password}.{version_code}"
key = sha256(key.encode()).digest()
data = "ThisIsOurHardWorkPleaseDoNotCopyOrSteal(c)2019.KubaSz"
data = sha256(data.encode()).digest()
data = data + (chr(16) * 16).encode()
aes = AES.new(key=key, mode=AES.MODE_CBC, iv=iv)
app_password = base64.b64encode(aes.encrypt(data)).decode()
c = db.cursor()
c.execute(
"INSERT IGNORE INTO _appPasswords (versionCode, appPassword, password, iv) VALUES (%s, %s, %s, %s);",
(version_code, app_password, password, iv),
)
db.commit()
c = db.cursor()
c.execute(
"SELECT password, iv FROM _appPasswords WHERE versionCode = %s;",
(version_code,),
)
row = c.fetchone()
db.close()
return (row[0], row[1])

142
.github/utils/_utils.py vendored Normal file
View File

@ -0,0 +1,142 @@
import re
import subprocess
import sys
from datetime import datetime
from typing import Tuple
VERSION_NAME_REGEX = r'versionName: "(.+?)"'
VERSION_CODE_REGEX = r"versionCode: ([0-9]+)"
VERSION_NAME_FORMAT = 'versionName: "{}"'
VERSION_CODE_FORMAT = "versionCode: {}"
def get_project_dir() -> str:
project_dir = sys.argv[1]
if project_dir[-1:] == "/" or project_dir[-1:] == "\\":
project_dir = project_dir[:-1]
return project_dir
def read_gradle_version(project_dir: str) -> Tuple[int, str]:
GRADLE_PATH = f"{project_dir}/build.gradle"
with open(GRADLE_PATH, "r") as f:
gradle = f.read()
version_name = re.search(VERSION_NAME_REGEX, gradle).group(1)
version_code = int(re.search(VERSION_CODE_REGEX, gradle).group(1))
return (version_code, version_name)
def write_gradle_version(project_dir: str, version_code: int, version_name: str):
GRADLE_PATH = f"{project_dir}/build.gradle"
with open(GRADLE_PATH, "r") as f:
gradle = f.read()
gradle = re.sub(
VERSION_NAME_REGEX, VERSION_NAME_FORMAT.format(version_name), gradle
)
gradle = re.sub(
VERSION_CODE_REGEX, VERSION_CODE_FORMAT.format(version_code), gradle
)
with open(GRADLE_PATH, "w") as f:
f.write(gradle)
def build_version_code(version_name: str) -> int:
version = version_name.split("+")[0].split("-")
version_base = version[0]
version_suffix = version[1] if len(version) == 2 else ""
base_parts = version_base.split(".")
major = int(base_parts[0]) or 0
minor = int(base_parts[1]) if len(base_parts) > 1 else 0
patch = int(base_parts[2]) if len(base_parts) > 2 else 0
beta = 9
rc = 9
if "dev" in version_suffix:
beta = 0
rc = 0
elif "beta." in version_suffix:
beta = int(version_suffix.split(".")[1])
rc = 0
elif "rc." in version_suffix:
beta = 0
rc = int(version_suffix.split(".")[1])
version_code = beta + rc * 10 + patch * 100 + minor * 10000 + major * 1000000
return version_code
def get_changelog(project_dir: str, format: str) -> Tuple[str, str]:
with open(
f"{project_dir}/app/src/main/assets/pl-changelog.html", "r", encoding="utf-8"
) as f:
changelog = f.read()
title = re.search(r"<h3>(.+?)</h3>", changelog).group(1)
content = re.search(r"(?s)<ul>(.+)</ul>", changelog).group(1).strip()
content = "\n".join(line.strip() for line in content.split("\n"))
if format != "html":
content = content.replace("<li>", "- ")
content = content.replace("<br>", "\n")
if format == "markdown":
content = re.sub(r"<u>(.+?)</u>", "__\\1__", content)
content = re.sub(r"<i>(.+?)</i>", "*\\1*", content)
content = re.sub(r"<b>(.+?)</b>", "**\\1**", content)
content = re.sub(r"</?.+?>", "", content)
return (title, content)
def get_commit_log(project_dir: str, format: str, max_lines: int = None) -> str:
last_tag = (
subprocess.check_output("git describe --tags --abbrev=0".split(" "))
.decode()
.strip()
)
log = subprocess.run(
args=f"git log {last_tag}..HEAD --format=%an%x00%at%x00%h%x00%s%x00%D".split(" "),
cwd=project_dir,
stdout=subprocess.PIPE,
)
log = log.stdout.strip().decode()
commits = [line.split("\x00") for line in log.split("\n")]
if max_lines:
commits = commits[:max_lines]
output = []
valid = False
for commit in commits:
if not commit[0]:
continue
if "origin/" in commit[4]:
valid = True
if not valid:
continue
date = datetime.fromtimestamp(float(commit[1]))
date = date.strftime("%Y-%m-%d %H:%M:%S")
if format == "html":
output.append(f"<li>{commit[3]} <i> - {commit[0]}</i></li>")
elif format == "markdown":
output.append(f"[{date}] {commit[0]}\n {commit[3]}")
elif format == "markdown_full":
output.append(
f"_[{date}] {commit[0]}_\n` `__`{commit[2]}`__ **{commit[3]}**"
)
elif format == "plain":
output.append(f"- {commit[3]}")
if format == "markdown":
output.insert(0, "```")
output.append("```")
return "\n".join(output)

69
.github/utils/bump_nightly.py vendored Normal file
View File

@ -0,0 +1,69 @@
import json
import os
import re
import sys
from datetime import datetime, timedelta
import requests
from _utils import (
get_commit_log,
get_project_dir,
read_gradle_version,
write_gradle_version,
)
if __name__ == "__main__":
if len(sys.argv) < 2:
print("usage: bump_nightly.py <project dir>")
exit(-1)
repo = os.getenv("GITHUB_REPOSITORY")
sha = os.getenv("GITHUB_SHA")
if not repo or not sha:
print("Missing GitHub environment variables.")
exit(-1)
with requests.get(
f"https://api.github.com/repos/{repo}/actions/runs?per_page=1&status=success"
) as r:
data = json.loads(r.text)
runs = [run for run in data["workflow_runs"] if run["head_sha"] == sha]
if runs:
print("::set-output name=hasNewChanges::false")
exit(0)
print("::set-output name=hasNewChanges::true")
project_dir = get_project_dir()
(version_code, version_name) = read_gradle_version(project_dir)
version_name = version_name.split("+")[0]
date = datetime.now()
if date.hour > 6:
version_name += "+daily." + date.strftime("%Y%m%d-%H%M")
else:
date -= timedelta(days=1)
version_name += "+nightly." + date.strftime("%Y%m%d")
print("::set-output name=appVersionName::" + version_name)
print("::set-output name=appVersionCode::" + str(version_code))
write_gradle_version(project_dir, version_code, version_name)
commit_log = get_commit_log(project_dir, format="html", max_lines=10)
with open(
f"{project_dir}/app/src/main/assets/pl-changelog.html", "r", encoding="utf-8"
) as f:
changelog = f.read()
changelog = re.sub(r"<h3>(.+?)</h3>", f"<h3>{version_name}</h3>", changelog)
changelog = re.sub(r"(?s)<ul>(.+)</ul>", f"<ul>\n{commit_log}\n</ul>", changelog)
with open(
f"{project_dir}/app/src/main/assets/pl-changelog.html", "w", encoding="utf-8"
) as f:
f.write(changelog)

41
.github/utils/bump_version.py vendored Normal file
View File

@ -0,0 +1,41 @@
import os
from dotenv import load_dotenv
from _get_password import get_password
from _utils import build_version_code, write_gradle_version
from sign import sign
if __name__ == "__main__":
version_name = input("Enter version name: ")
version_code = build_version_code(version_name)
print(f"Bumping version to {version_name} ({version_code})")
project_dir = "../.."
load_dotenv()
DB_HOST = os.getenv("DB_HOST")
DB_USER = os.getenv("DB_USER")
DB_PASS = os.getenv("DB_PASS")
DB_NAME = os.getenv("DB_NAME")
write_gradle_version(project_dir, version_code, version_name)
(password, iv) = get_password(
version_name, version_code, DB_HOST, DB_USER, DB_PASS, DB_NAME
)
sign(project_dir, version_name, version_code, password, iv, commit=False)
print("Writing mock passwords")
os.chdir(project_dir)
os.system(
"sed -i -E 's/\/\*([0-9a-f]{2} ?){16}\*\//\/*secret password - removed for source code publication*\//g' app/src/main/cpp/szkolny-signing.cpp"
)
os.system(
"sed -i -E 's/\\t0x.., 0x(.)., 0x.(.), 0x.(.), 0x.., 0x.., 0x.., 0x.(.), 0x.., 0x.(.), 0x(.)., 0x(.)., 0x.., 0x.., 0x.., 0x.(.)/\\t0x\\3\\6, 0x\\7\\4, 0x\\1\\8, 0x\\2\\5, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff /g' app/src/main/cpp/szkolny-signing.cpp"
)
os.system(
"sed -i -E 's/param1\..(.).(.).(.).(.)..(.)..(.)..(.)..(.).../param1.MTIzNDU2Nzg5MD\\5\\2\\7\\6\\1\\3\\4\8==/g' app/src/main/java/pl/szczodrzynski/edziennik/data/api/szkolny/interceptor/Signing.kt"
)
input("Press any key to finish")

59
.github/utils/extract_changelogs.py vendored Normal file
View File

@ -0,0 +1,59 @@
import os
import sys
from _utils import get_changelog, get_commit_log, get_project_dir, read_gradle_version
if __name__ == "__main__":
if len(sys.argv) < 2:
print("usage: extract_changelogs.py <project dir>")
exit(-1)
project_dir = get_project_dir()
(version_code, version_name) = read_gradle_version(project_dir)
print("::set-output name=appVersionName::" + version_name)
print("::set-output name=appVersionCode::" + str(version_code))
dir = f"{project_dir}/app/release/whatsnew-{version_name}/"
os.makedirs(dir, exist_ok=True)
print("::set-output name=changelogDir::" + dir)
(title, changelog) = get_changelog(project_dir, format="plain")
with open(dir + "whatsnew-pl-PL", "w", encoding="utf-8") as f:
f.write(changelog)
print("::set-output name=changelogPlainFile::" + dir + "whatsnew-pl-PL")
with open(dir + "whatsnew-titled.txt", "w", encoding="utf-8") as f:
f.write(title)
f.write("\n")
f.write(changelog)
print("::set-output name=changelogPlainTitledFile::" + dir + "whatsnew-titled.txt")
print("::set-output name=changelogTitle::" + title)
(_, changelog) = get_changelog(project_dir, format="markdown")
with open(dir + "whatsnew.md", "w", encoding="utf-8") as f:
f.write(changelog)
print("::set-output name=changelogMarkdownFile::" + dir + "whatsnew.md")
(_, changelog) = get_changelog(project_dir, format="html")
with open(dir + "whatsnew.html", "w", encoding="utf-8") as f:
f.write(changelog)
print("::set-output name=changelogHtmlFile::" + dir + "whatsnew.html")
changelog = get_commit_log(project_dir, format="plain", max_lines=10)
with open(dir + "commit_log.txt", "w", encoding="utf-8") as f:
f.write(changelog)
print("::set-output name=commitLogPlainFile::" + dir + "commit_log.txt")
changelog = get_commit_log(project_dir, format="markdown", max_lines=10)
with open(dir + "commit_log.md", "w", encoding="utf-8") as f:
f.write(changelog)
print("::set-output name=commitLogMarkdownFile::" + dir + "commit_log.md")
changelog = get_commit_log(project_dir, format="html", max_lines=10)
with open(dir + "commit_log.html", "w", encoding="utf-8") as f:
f.write(changelog)
print("::set-output name=commitLogHtmlFile::" + dir + "commit_log.html")

26
.github/utils/rename_artifacts.py vendored Normal file
View File

@ -0,0 +1,26 @@
import glob
import os
import sys
from _utils import get_project_dir
if __name__ == "__main__":
if len(sys.argv) < 2:
print("usage: rename_artifacts.py <project dir>")
exit(-1)
project_dir = get_project_dir()
files = glob.glob(f"{project_dir}/app/release/*.*")
for file in files:
file_relative = file.replace(os.getenv("GITHUB_WORKSPACE") + "/", "")
if "-aligned.apk" in file:
os.unlink(file)
elif "-signed.apk" in file:
new_file = file.replace("-signed.apk", ".apk")
if os.path.isfile(new_file):
os.unlink(new_file)
os.rename(file, new_file)
elif ".apk" in file or ".aab" in file:
print("::set-output name=signedReleaseFile::" + file)
print("::set-output name=signedReleaseFileRelative::" + file_relative)

122
.github/utils/save_version.py vendored Normal file
View File

@ -0,0 +1,122 @@
import glob
import os
import sys
from datetime import datetime
from time import time
import mysql.connector as mysql
from dotenv import load_dotenv
from _utils import get_changelog, get_commit_log, get_project_dir, read_gradle_version
def save_version(
project_dir: str,
db_host: str,
db_user: str,
db_pass: str,
db_name: str,
apk_server_release: str,
apk_server_nightly: str,
):
db = mysql.connect(
host=db_host,
user=db_user,
password=db_pass,
database=db_name,
auth_plugin="mysql_native_password",
)
(version_code, version_name) = read_gradle_version(project_dir)
(_, changelog) = get_changelog(project_dir, format="html")
types = ["dev", "beta", "nightly", "daily", "rc", "release"]
build_type = [x for x in types if x in version_name]
build_type = build_type[0] if build_type else "release"
if "+nightly." in version_name or "+daily." in version_name:
changelog = get_commit_log(project_dir, format="html")
build_type = "nightly"
elif "-dev" in version_name:
build_type = "dev"
elif "-beta." in version_name:
build_type = "beta"
elif "-rc." in version_name:
build_type = "rc"
build_date = int(time())
apk_name = None
bundle_name_play = None
files = glob.glob(f"{project_dir}/app/release/*.*")
output_apk = f"Edziennik_{version_name}_official.apk"
output_aab_play = f"Edziennik_{version_name}_play.aab"
for file in files:
if output_apk in file:
build_date = int(os.stat(file).st_mtime)
apk_name = output_apk
if output_aab_play in file:
build_date = int(os.stat(file).st_mtime)
bundle_name_play = output_aab_play
build_date = datetime.fromtimestamp(build_date).strftime("%Y-%m-%d %H:%M:%S")
if build_type in ["nightly", "daily"]:
download_url = apk_server_nightly + apk_name if apk_name else None
else:
download_url = apk_server_release + apk_name if apk_name else None
cols = [
"versionCode",
"versionName",
"releaseDate",
"releaseNotes",
"releaseType",
"downloadUrl",
"apkName",
"bundleNamePlay",
]
updated = {
"versionCode": version_code,
"downloadUrl": download_url,
"apkName": apk_name,
"bundleNamePlay": bundle_name_play,
}
values = [
version_code,
version_name,
build_date,
changelog,
build_type,
download_url,
apk_name,
bundle_name_play,
]
values.extend(val for val in updated.values() if val)
updated = ", ".join(f"{col} = %s" for (col, val) in updated.items() if val)
sql = f"INSERT INTO updates ({', '.join(cols)}) VALUES ({'%s, ' * (len(cols) - 1)}%s) ON DUPLICATE KEY UPDATE {updated};"
c = db.cursor()
c.execute(sql, tuple(values))
db.commit()
if __name__ == "__main__":
if len(sys.argv) < 2:
print("usage: save_version.py <project dir>")
exit(-1)
project_dir = get_project_dir()
load_dotenv()
DB_HOST = os.getenv("DB_HOST")
DB_USER = os.getenv("DB_USER")
DB_PASS = os.getenv("DB_PASS")
DB_NAME = os.getenv("DB_NAME")
APK_SERVER_RELEASE = os.getenv("APK_SERVER_RELEASE")
APK_SERVER_NIGHTLY = os.getenv("APK_SERVER_NIGHTLY")
save_version(project_dir, DB_HOST, DB_USER, DB_PASS, DB_NAME, APK_SERVER_RELEASE, APK_SERVER_NIGHTLY)

84
.github/utils/sign.py vendored Normal file
View File

@ -0,0 +1,84 @@
import os
import re
import sys
from dotenv import load_dotenv
from _get_password import get_password
from _utils import get_project_dir, read_gradle_version
def sign(
project_dir: str,
version_name: str,
version_code: int,
password: str,
iv: bytes,
commit: bool = False,
):
SIGNING_PATH = f"{project_dir}/app/src/main/java/pl/szczodrzynski/edziennik/data/api/szkolny/interceptor/Signing.kt"
CPP_PATH = f"{project_dir}/app/src/main/cpp/szkolny-signing.cpp"
with open(SIGNING_PATH, "r") as f:
signing = f.read()
with open(CPP_PATH, "r") as f:
cpp = f.read()
SIGNING_REGEX = r"\$param1\..*\.\$param2"
CPP_REGEX = r"(?s)/\*.+?toys AES_IV\[16\] = {.+?};"
SIGNING_FORMAT = "$param1.{}.$param2"
CPP_FORMAT = "/*{}*/\nstatic toys AES_IV[16] = {{\n\t{} }};"
print(f"Writing passwords for version {version_name} ({version_code})")
iv_hex = " ".join(["{:02x}".format(x) for x in iv])
iv_cpp = ", ".join(["0x{:02x}".format(x) for x in iv])
signing = re.sub(SIGNING_REGEX, SIGNING_FORMAT.format(password), signing)
cpp = re.sub(CPP_REGEX, CPP_FORMAT.format(iv_hex, iv_cpp), cpp)
with open(SIGNING_PATH, "w") as f:
f.write(signing)
with open(CPP_PATH, "w") as f:
f.write(cpp)
if commit:
os.chdir(project_dir)
os.system("git add .")
os.system(
f'git commit -m "[{version_name}] Update build.gradle, signing and changelog."'
)
if __name__ == "__main__":
if len(sys.argv) < 2:
print("usage: sign.py <project dir> [commit]")
exit(-1)
project_dir = get_project_dir()
load_dotenv()
DB_HOST = os.getenv("DB_HOST")
DB_USER = os.getenv("DB_USER")
DB_PASS = os.getenv("DB_PASS")
DB_NAME = os.getenv("DB_NAME")
(version_code, version_name) = read_gradle_version(project_dir)
(password, iv) = get_password(
version_name, version_code, DB_HOST, DB_USER, DB_PASS, DB_NAME
)
print("::set-output name=appVersionName::" + version_name)
print("::set-output name=appVersionCode::" + str(version_code))
sign(
project_dir,
version_name,
version_code,
password,
iv,
commit="commit" in sys.argv,
)

118
.github/utils/webhook_discord.py vendored Normal file
View File

@ -0,0 +1,118 @@
import os
import sys
from datetime import datetime
import requests
from dotenv import load_dotenv
from _utils import get_changelog, get_commit_log, get_project_dir, read_gradle_version
def post_webhook(
project_dir: str,
apk_file: str,
apk_server_release: str,
apk_server_nightly: str,
webhook_release: str,
webhook_testing: str,
):
(_, version_name) = read_gradle_version(project_dir)
types = ["dev", "beta", "nightly", "daily", "rc", "release"]
build_type = [x for x in types if x in version_name]
build_type = build_type[0] if build_type else None
testing = ["dev", "beta", "nightly", "daily"]
testing = build_type in testing
apk_name = os.path.basename(apk_file)
if build_type in ["nightly", "daily"]:
download_url = apk_server_nightly + apk_name
else:
download_url = apk_server_release + apk_name
if testing:
build_date = int(os.stat(apk_file).st_mtime)
if build_date:
build_date = datetime.fromtimestamp(build_date).strftime("%Y-%m-%d %H:%M")
# untagged release, get commit log
if build_type in ["nightly", "daily"]:
changelog = get_commit_log(project_dir, format="markdown", max_lines=5)
else:
changelog = get_changelog(project_dir, format="markdown")
webhook = get_webhook_testing(
version_name, build_type, changelog, download_url, build_date
)
requests.post(url=webhook_testing, json=webhook)
else:
changelog = get_changelog(project_dir, format="markdown")
webhook = get_webhook_release(changelog, download_url)
requests.post(url=webhook_release, json=webhook)
def get_webhook_release(changelog: str, download_url: str):
(title, content) = changelog
return {"content": f"__**{title}**__\n{content}\n{download_url}"}
def get_webhook_testing(
version_name: str,
build_type: str,
changelog: str,
download_url: str,
build_date: str,
):
return {
"embeds": [
{
"title": f"Nowa wersja {build_type} aplikacji Szkolny.eu",
"description": f"Dostępna jest nowa wersja testowa **{build_type}**.",
"color": 2201331,
"fields": [
{
"name": f"Wersja `{version_name}`",
"value": f"[Pobierz .APK]({download_url})"
if download_url
else "*Pobieranie niedostępne*",
"inline": False,
},
{
"name": "Data kompilacji",
"value": build_date or "-",
"inline": False,
},
{
"name": "Ostatnie zmiany",
"value": changelog or "-",
"inline": False,
},
],
}
]
}
if __name__ == "__main__":
if len(sys.argv) < 2:
print("usage: webhook_discord.py <project dir>")
exit(-1)
project_dir = get_project_dir()
load_dotenv()
APK_FILE = os.getenv("APK_FILE")
APK_SERVER_RELEASE = os.getenv("APK_SERVER_RELEASE")
APK_SERVER_NIGHTLY = os.getenv("APK_SERVER_NIGHTLY")
WEBHOOK_RELEASE = os.getenv("WEBHOOK_RELEASE")
WEBHOOK_TESTING = os.getenv("WEBHOOK_TESTING")
post_webhook(
project_dir,
APK_FILE,
APK_SERVER_RELEASE,
APK_SERVER_NIGHTLY,
WEBHOOK_RELEASE,
WEBHOOK_TESTING,
)