diff --git a/.github/utils/.gitignore b/.github/utils/.gitignore new file mode 100644 index 00000000..d50a09fc --- /dev/null +++ b/.github/utils/.gitignore @@ -0,0 +1,2 @@ +.env +__pycache__/ diff --git a/.github/utils/_get_password.py b/.github/utils/_get_password.py new file mode 100644 index 00000000..33071b6c --- /dev/null +++ b/.github/utils/_get_password.py @@ -0,0 +1,57 @@ +import base64 +import secrets +from hashlib import sha256 +from typing import Tuple + +import mysql.connector as mysql +from Crypto.Cipher import AES + + +def get_password( + version_name: str, + version_code: int, + db_host: str, + db_user: str, + db_pass: str, + db_name: str, +) -> Tuple[str, bytes]: + db = mysql.connect( + host=db_host, + user=db_user, + password=db_pass, + database=db_name, + auth_plugin="mysql_native_password", + ) + + print(f"Generating passwords for version {version_name} ({version_code})") + + password = base64.b64encode(secrets.token_bytes(16)).decode() + iv = secrets.token_bytes(16) + + key = f"{version_name}.{password}.{version_code}" + key = sha256(key.encode()).digest() + data = "ThisIsOurHardWorkPleaseDoNotCopyOrSteal(c)2019.KubaSz" + data = sha256(data.encode()).digest() + data = data + (chr(16) * 16).encode() + + aes = AES.new(key=key, mode=AES.MODE_CBC, iv=iv) + + app_password = base64.b64encode(aes.encrypt(data)).decode() + + c = db.cursor() + c.execute( + "INSERT IGNORE INTO _appPasswords (versionCode, appPassword, password, iv) VALUES (%s, %s, %s, %s);", + (version_code, app_password, password, iv), + ) + db.commit() + + c = db.cursor() + c.execute( + "SELECT password, iv FROM _appPasswords WHERE versionCode = %s;", + (version_code,), + ) + row = c.fetchone() + + db.close() + + return (row[0], row[1]) diff --git a/.github/utils/_utils.py b/.github/utils/_utils.py new file mode 100644 index 00000000..09b59f4a --- /dev/null +++ b/.github/utils/_utils.py @@ -0,0 +1,142 @@ +import re +import subprocess +import sys +from datetime import datetime +from typing import Tuple + +VERSION_NAME_REGEX = r'versionName: "(.+?)"' +VERSION_CODE_REGEX = r"versionCode: ([0-9]+)" +VERSION_NAME_FORMAT = 'versionName: "{}"' +VERSION_CODE_FORMAT = "versionCode: {}" + + +def get_project_dir() -> str: + project_dir = sys.argv[1] + if project_dir[-1:] == "/" or project_dir[-1:] == "\\": + project_dir = project_dir[:-1] + return project_dir + + +def read_gradle_version(project_dir: str) -> Tuple[int, str]: + GRADLE_PATH = f"{project_dir}/build.gradle" + + with open(GRADLE_PATH, "r") as f: + gradle = f.read() + + version_name = re.search(VERSION_NAME_REGEX, gradle).group(1) + version_code = int(re.search(VERSION_CODE_REGEX, gradle).group(1)) + + return (version_code, version_name) + + +def write_gradle_version(project_dir: str, version_code: int, version_name: str): + GRADLE_PATH = f"{project_dir}/build.gradle" + + with open(GRADLE_PATH, "r") as f: + gradle = f.read() + + gradle = re.sub( + VERSION_NAME_REGEX, VERSION_NAME_FORMAT.format(version_name), gradle + ) + gradle = re.sub( + VERSION_CODE_REGEX, VERSION_CODE_FORMAT.format(version_code), gradle + ) + + with open(GRADLE_PATH, "w") as f: + f.write(gradle) + + +def build_version_code(version_name: str) -> int: + version = version_name.split("+")[0].split("-") + version_base = version[0] + version_suffix = version[1] if len(version) == 2 else "" + + base_parts = version_base.split(".") + major = int(base_parts[0]) or 0 + minor = int(base_parts[1]) if len(base_parts) > 1 else 0 + patch = int(base_parts[2]) if len(base_parts) > 2 else 0 + + beta = 9 + rc = 9 + if "dev" in version_suffix: + beta = 0 + rc = 0 + elif "beta." in version_suffix: + beta = int(version_suffix.split(".")[1]) + rc = 0 + elif "rc." in version_suffix: + beta = 0 + rc = int(version_suffix.split(".")[1]) + + version_code = beta + rc * 10 + patch * 100 + minor * 10000 + major * 1000000 + return version_code + + +def get_changelog(project_dir: str, format: str) -> Tuple[str, str]: + with open( + f"{project_dir}/app/src/main/assets/pl-changelog.html", "r", encoding="utf-8" + ) as f: + changelog = f.read() + + title = re.search(r"

(.+?)

", changelog).group(1) + content = re.search(r"(?s)", changelog).group(1).strip() + content = "\n".join(line.strip() for line in content.split("\n")) + + if format != "html": + content = content.replace("
  • ", "- ") + content = content.replace("
    ", "\n") + if format == "markdown": + content = re.sub(r"(.+?)", "__\\1__", content) + content = re.sub(r"(.+?)", "*\\1*", content) + content = re.sub(r"(.+?)", "**\\1**", content) + content = re.sub(r"", "", content) + + return (title, content) + + +def get_commit_log(project_dir: str, format: str, max_lines: int = None) -> str: + last_tag = ( + subprocess.check_output("git describe --tags --abbrev=0".split(" ")) + .decode() + .strip() + ) + + log = subprocess.run( + args=f"git log {last_tag}..HEAD --format=%an%x00%at%x00%h%x00%s%x00%D".split(" "), + cwd=project_dir, + stdout=subprocess.PIPE, + ) + log = log.stdout.strip().decode() + + commits = [line.split("\x00") for line in log.split("\n")] + if max_lines: + commits = commits[:max_lines] + + output = [] + valid = False + + for commit in commits: + if not commit[0]: + continue + if "origin/" in commit[4]: + valid = True + if not valid: + continue + date = datetime.fromtimestamp(float(commit[1])) + date = date.strftime("%Y-%m-%d %H:%M:%S") + if format == "html": + output.append(f"
  • {commit[3]} - {commit[0]}
  • ") + elif format == "markdown": + output.append(f"[{date}] {commit[0]}\n {commit[3]}") + elif format == "markdown_full": + output.append( + f"_[{date}] {commit[0]}_\n` `__`{commit[2]}`__ **{commit[3]}**" + ) + elif format == "plain": + output.append(f"- {commit[3]}") + + if format == "markdown": + output.insert(0, "```") + output.append("```") + + return "\n".join(output) diff --git a/.github/utils/bump_nightly.py b/.github/utils/bump_nightly.py new file mode 100644 index 00000000..888a087f --- /dev/null +++ b/.github/utils/bump_nightly.py @@ -0,0 +1,69 @@ +import json +import os +import re +import sys +from datetime import datetime, timedelta + +import requests + +from _utils import ( + get_commit_log, + get_project_dir, + read_gradle_version, + write_gradle_version, +) + +if __name__ == "__main__": + if len(sys.argv) < 2: + print("usage: bump_nightly.py ") + exit(-1) + + repo = os.getenv("GITHUB_REPOSITORY") + sha = os.getenv("GITHUB_SHA") + + if not repo or not sha: + print("Missing GitHub environment variables.") + exit(-1) + + with requests.get( + f"https://api.github.com/repos/{repo}/actions/runs?per_page=1&status=success" + ) as r: + data = json.loads(r.text) + runs = [run for run in data["workflow_runs"] if run["head_sha"] == sha] + if runs: + print("::set-output name=hasNewChanges::false") + exit(0) + + print("::set-output name=hasNewChanges::true") + + project_dir = get_project_dir() + + (version_code, version_name) = read_gradle_version(project_dir) + version_name = version_name.split("+")[0] + + date = datetime.now() + if date.hour > 6: + version_name += "+daily." + date.strftime("%Y%m%d-%H%M") + else: + date -= timedelta(days=1) + version_name += "+nightly." + date.strftime("%Y%m%d") + + print("::set-output name=appVersionName::" + version_name) + print("::set-output name=appVersionCode::" + str(version_code)) + + write_gradle_version(project_dir, version_code, version_name) + + commit_log = get_commit_log(project_dir, format="html", max_lines=10) + + with open( + f"{project_dir}/app/src/main/assets/pl-changelog.html", "r", encoding="utf-8" + ) as f: + changelog = f.read() + + changelog = re.sub(r"

    (.+?)

    ", f"

    {version_name}

    ", changelog) + changelog = re.sub(r"(?s)", f"", changelog) + + with open( + f"{project_dir}/app/src/main/assets/pl-changelog.html", "w", encoding="utf-8" + ) as f: + f.write(changelog) diff --git a/.github/utils/bump_version.py b/.github/utils/bump_version.py new file mode 100644 index 00000000..80d36519 --- /dev/null +++ b/.github/utils/bump_version.py @@ -0,0 +1,41 @@ +import os + +from dotenv import load_dotenv + +from _get_password import get_password +from _utils import build_version_code, write_gradle_version +from sign import sign + +if __name__ == "__main__": + version_name = input("Enter version name: ") + version_code = build_version_code(version_name) + + print(f"Bumping version to {version_name} ({version_code})") + + project_dir = "../.." + + load_dotenv() + DB_HOST = os.getenv("DB_HOST") + DB_USER = os.getenv("DB_USER") + DB_PASS = os.getenv("DB_PASS") + DB_NAME = os.getenv("DB_NAME") + + write_gradle_version(project_dir, version_code, version_name) + (password, iv) = get_password( + version_name, version_code, DB_HOST, DB_USER, DB_PASS, DB_NAME + ) + + sign(project_dir, version_name, version_code, password, iv, commit=False) + + print("Writing mock passwords") + os.chdir(project_dir) + os.system( + "sed -i -E 's/\/\*([0-9a-f]{2} ?){16}\*\//\/*secret password - removed for source code publication*\//g' app/src/main/cpp/szkolny-signing.cpp" + ) + os.system( + "sed -i -E 's/\\t0x.., 0x(.)., 0x.(.), 0x.(.), 0x.., 0x.., 0x.., 0x.(.), 0x.., 0x.(.), 0x(.)., 0x(.)., 0x.., 0x.., 0x.., 0x.(.)/\\t0x\\3\\6, 0x\\7\\4, 0x\\1\\8, 0x\\2\\5, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff /g' app/src/main/cpp/szkolny-signing.cpp" + ) + os.system( + "sed -i -E 's/param1\..(.).(.).(.).(.)..(.)..(.)..(.)..(.).../param1.MTIzNDU2Nzg5MD\\5\\2\\7\\6\\1\\3\\4\8==/g' app/src/main/java/pl/szczodrzynski/edziennik/data/api/szkolny/interceptor/Signing.kt" + ) + input("Press any key to finish") diff --git a/.github/utils/extract_changelogs.py b/.github/utils/extract_changelogs.py new file mode 100644 index 00000000..07524bad --- /dev/null +++ b/.github/utils/extract_changelogs.py @@ -0,0 +1,59 @@ +import os +import sys + +from _utils import get_changelog, get_commit_log, get_project_dir, read_gradle_version + +if __name__ == "__main__": + if len(sys.argv) < 2: + print("usage: extract_changelogs.py ") + exit(-1) + + project_dir = get_project_dir() + + (version_code, version_name) = read_gradle_version(project_dir) + + print("::set-output name=appVersionName::" + version_name) + print("::set-output name=appVersionCode::" + str(version_code)) + + dir = f"{project_dir}/app/release/whatsnew-{version_name}/" + os.makedirs(dir, exist_ok=True) + + print("::set-output name=changelogDir::" + dir) + + (title, changelog) = get_changelog(project_dir, format="plain") + with open(dir + "whatsnew-pl-PL", "w", encoding="utf-8") as f: + f.write(changelog) + print("::set-output name=changelogPlainFile::" + dir + "whatsnew-pl-PL") + + with open(dir + "whatsnew-titled.txt", "w", encoding="utf-8") as f: + f.write(title) + f.write("\n") + f.write(changelog) + print("::set-output name=changelogPlainTitledFile::" + dir + "whatsnew-titled.txt") + + print("::set-output name=changelogTitle::" + title) + + (_, changelog) = get_changelog(project_dir, format="markdown") + with open(dir + "whatsnew.md", "w", encoding="utf-8") as f: + f.write(changelog) + print("::set-output name=changelogMarkdownFile::" + dir + "whatsnew.md") + + (_, changelog) = get_changelog(project_dir, format="html") + with open(dir + "whatsnew.html", "w", encoding="utf-8") as f: + f.write(changelog) + print("::set-output name=changelogHtmlFile::" + dir + "whatsnew.html") + + changelog = get_commit_log(project_dir, format="plain", max_lines=10) + with open(dir + "commit_log.txt", "w", encoding="utf-8") as f: + f.write(changelog) + print("::set-output name=commitLogPlainFile::" + dir + "commit_log.txt") + + changelog = get_commit_log(project_dir, format="markdown", max_lines=10) + with open(dir + "commit_log.md", "w", encoding="utf-8") as f: + f.write(changelog) + print("::set-output name=commitLogMarkdownFile::" + dir + "commit_log.md") + + changelog = get_commit_log(project_dir, format="html", max_lines=10) + with open(dir + "commit_log.html", "w", encoding="utf-8") as f: + f.write(changelog) + print("::set-output name=commitLogHtmlFile::" + dir + "commit_log.html") diff --git a/.github/utils/rename_artifacts.py b/.github/utils/rename_artifacts.py new file mode 100644 index 00000000..4eeabfaf --- /dev/null +++ b/.github/utils/rename_artifacts.py @@ -0,0 +1,26 @@ +import glob +import os +import sys + +from _utils import get_project_dir + +if __name__ == "__main__": + if len(sys.argv) < 2: + print("usage: rename_artifacts.py ") + exit(-1) + + project_dir = get_project_dir() + + files = glob.glob(f"{project_dir}/app/release/*.*") + for file in files: + file_relative = file.replace(os.getenv("GITHUB_WORKSPACE") + "/", "") + if "-aligned.apk" in file: + os.unlink(file) + elif "-signed.apk" in file: + new_file = file.replace("-signed.apk", ".apk") + if os.path.isfile(new_file): + os.unlink(new_file) + os.rename(file, new_file) + elif ".apk" in file or ".aab" in file: + print("::set-output name=signedReleaseFile::" + file) + print("::set-output name=signedReleaseFileRelative::" + file_relative) diff --git a/.github/utils/save_version.py b/.github/utils/save_version.py new file mode 100644 index 00000000..01de17a7 --- /dev/null +++ b/.github/utils/save_version.py @@ -0,0 +1,122 @@ +import glob +import os +import sys +from datetime import datetime +from time import time + +import mysql.connector as mysql +from dotenv import load_dotenv + +from _utils import get_changelog, get_commit_log, get_project_dir, read_gradle_version + + +def save_version( + project_dir: str, + db_host: str, + db_user: str, + db_pass: str, + db_name: str, + apk_server_release: str, + apk_server_nightly: str, +): + db = mysql.connect( + host=db_host, + user=db_user, + password=db_pass, + database=db_name, + auth_plugin="mysql_native_password", + ) + + (version_code, version_name) = read_gradle_version(project_dir) + (_, changelog) = get_changelog(project_dir, format="html") + + types = ["dev", "beta", "nightly", "daily", "rc", "release"] + build_type = [x for x in types if x in version_name] + build_type = build_type[0] if build_type else "release" + + if "+nightly." in version_name or "+daily." in version_name: + changelog = get_commit_log(project_dir, format="html") + build_type = "nightly" + elif "-dev" in version_name: + build_type = "dev" + elif "-beta." in version_name: + build_type = "beta" + elif "-rc." in version_name: + build_type = "rc" + + build_date = int(time()) + apk_name = None + bundle_name_play = None + + files = glob.glob(f"{project_dir}/app/release/*.*") + output_apk = f"Edziennik_{version_name}_official.apk" + output_aab_play = f"Edziennik_{version_name}_play.aab" + for file in files: + if output_apk in file: + build_date = int(os.stat(file).st_mtime) + apk_name = output_apk + if output_aab_play in file: + build_date = int(os.stat(file).st_mtime) + bundle_name_play = output_aab_play + + build_date = datetime.fromtimestamp(build_date).strftime("%Y-%m-%d %H:%M:%S") + + if build_type in ["nightly", "daily"]: + download_url = apk_server_nightly + apk_name if apk_name else None + else: + download_url = apk_server_release + apk_name if apk_name else None + + cols = [ + "versionCode", + "versionName", + "releaseDate", + "releaseNotes", + "releaseType", + "downloadUrl", + "apkName", + "bundleNamePlay", + ] + updated = { + "versionCode": version_code, + "downloadUrl": download_url, + "apkName": apk_name, + "bundleNamePlay": bundle_name_play, + } + + values = [ + version_code, + version_name, + build_date, + changelog, + build_type, + download_url, + apk_name, + bundle_name_play, + ] + values.extend(val for val in updated.values() if val) + + updated = ", ".join(f"{col} = %s" for (col, val) in updated.items() if val) + + sql = f"INSERT INTO updates ({', '.join(cols)}) VALUES ({'%s, ' * (len(cols) - 1)}%s) ON DUPLICATE KEY UPDATE {updated};" + + c = db.cursor() + c.execute(sql, tuple(values)) + db.commit() + + +if __name__ == "__main__": + if len(sys.argv) < 2: + print("usage: save_version.py ") + exit(-1) + + project_dir = get_project_dir() + + load_dotenv() + DB_HOST = os.getenv("DB_HOST") + DB_USER = os.getenv("DB_USER") + DB_PASS = os.getenv("DB_PASS") + DB_NAME = os.getenv("DB_NAME") + APK_SERVER_RELEASE = os.getenv("APK_SERVER_RELEASE") + APK_SERVER_NIGHTLY = os.getenv("APK_SERVER_NIGHTLY") + + save_version(project_dir, DB_HOST, DB_USER, DB_PASS, DB_NAME, APK_SERVER_RELEASE, APK_SERVER_NIGHTLY) diff --git a/.github/utils/sign.py b/.github/utils/sign.py new file mode 100644 index 00000000..026d819b --- /dev/null +++ b/.github/utils/sign.py @@ -0,0 +1,84 @@ +import os +import re +import sys + +from dotenv import load_dotenv + +from _get_password import get_password +from _utils import get_project_dir, read_gradle_version + + +def sign( + project_dir: str, + version_name: str, + version_code: int, + password: str, + iv: bytes, + commit: bool = False, +): + SIGNING_PATH = f"{project_dir}/app/src/main/java/pl/szczodrzynski/edziennik/data/api/szkolny/interceptor/Signing.kt" + CPP_PATH = f"{project_dir}/app/src/main/cpp/szkolny-signing.cpp" + + with open(SIGNING_PATH, "r") as f: + signing = f.read() + + with open(CPP_PATH, "r") as f: + cpp = f.read() + + SIGNING_REGEX = r"\$param1\..*\.\$param2" + CPP_REGEX = r"(?s)/\*.+?toys AES_IV\[16\] = {.+?};" + + SIGNING_FORMAT = "$param1.{}.$param2" + CPP_FORMAT = "/*{}*/\nstatic toys AES_IV[16] = {{\n\t{} }};" + + print(f"Writing passwords for version {version_name} ({version_code})") + + iv_hex = " ".join(["{:02x}".format(x) for x in iv]) + iv_cpp = ", ".join(["0x{:02x}".format(x) for x in iv]) + + signing = re.sub(SIGNING_REGEX, SIGNING_FORMAT.format(password), signing) + cpp = re.sub(CPP_REGEX, CPP_FORMAT.format(iv_hex, iv_cpp), cpp) + + with open(SIGNING_PATH, "w") as f: + f.write(signing) + + with open(CPP_PATH, "w") as f: + f.write(cpp) + + if commit: + os.chdir(project_dir) + os.system("git add .") + os.system( + f'git commit -m "[{version_name}] Update build.gradle, signing and changelog."' + ) + + +if __name__ == "__main__": + if len(sys.argv) < 2: + print("usage: sign.py [commit]") + exit(-1) + + project_dir = get_project_dir() + + load_dotenv() + DB_HOST = os.getenv("DB_HOST") + DB_USER = os.getenv("DB_USER") + DB_PASS = os.getenv("DB_PASS") + DB_NAME = os.getenv("DB_NAME") + + (version_code, version_name) = read_gradle_version(project_dir) + (password, iv) = get_password( + version_name, version_code, DB_HOST, DB_USER, DB_PASS, DB_NAME + ) + + print("::set-output name=appVersionName::" + version_name) + print("::set-output name=appVersionCode::" + str(version_code)) + + sign( + project_dir, + version_name, + version_code, + password, + iv, + commit="commit" in sys.argv, + ) diff --git a/.github/utils/webhook_discord.py b/.github/utils/webhook_discord.py new file mode 100644 index 00000000..3c1404ae --- /dev/null +++ b/.github/utils/webhook_discord.py @@ -0,0 +1,118 @@ +import os +import sys +from datetime import datetime + +import requests +from dotenv import load_dotenv + +from _utils import get_changelog, get_commit_log, get_project_dir, read_gradle_version + + +def post_webhook( + project_dir: str, + apk_file: str, + apk_server_release: str, + apk_server_nightly: str, + webhook_release: str, + webhook_testing: str, +): + (_, version_name) = read_gradle_version(project_dir) + + types = ["dev", "beta", "nightly", "daily", "rc", "release"] + build_type = [x for x in types if x in version_name] + build_type = build_type[0] if build_type else None + + testing = ["dev", "beta", "nightly", "daily"] + testing = build_type in testing + + apk_name = os.path.basename(apk_file) + if build_type in ["nightly", "daily"]: + download_url = apk_server_nightly + apk_name + else: + download_url = apk_server_release + apk_name + + if testing: + build_date = int(os.stat(apk_file).st_mtime) + if build_date: + build_date = datetime.fromtimestamp(build_date).strftime("%Y-%m-%d %H:%M") + + # untagged release, get commit log + if build_type in ["nightly", "daily"]: + changelog = get_commit_log(project_dir, format="markdown", max_lines=5) + else: + changelog = get_changelog(project_dir, format="markdown") + + webhook = get_webhook_testing( + version_name, build_type, changelog, download_url, build_date + ) + requests.post(url=webhook_testing, json=webhook) + else: + changelog = get_changelog(project_dir, format="markdown") + webhook = get_webhook_release(changelog, download_url) + requests.post(url=webhook_release, json=webhook) + + +def get_webhook_release(changelog: str, download_url: str): + (title, content) = changelog + return {"content": f"__**{title}**__\n{content}\n{download_url}"} + + +def get_webhook_testing( + version_name: str, + build_type: str, + changelog: str, + download_url: str, + build_date: str, +): + return { + "embeds": [ + { + "title": f"Nowa wersja {build_type} aplikacji Szkolny.eu", + "description": f"Dostępna jest nowa wersja testowa **{build_type}**.", + "color": 2201331, + "fields": [ + { + "name": f"Wersja `{version_name}`", + "value": f"[Pobierz .APK]({download_url})" + if download_url + else "*Pobieranie niedostępne*", + "inline": False, + }, + { + "name": "Data kompilacji", + "value": build_date or "-", + "inline": False, + }, + { + "name": "Ostatnie zmiany", + "value": changelog or "-", + "inline": False, + }, + ], + } + ] + } + + +if __name__ == "__main__": + if len(sys.argv) < 2: + print("usage: webhook_discord.py ") + exit(-1) + + project_dir = get_project_dir() + + load_dotenv() + APK_FILE = os.getenv("APK_FILE") + APK_SERVER_RELEASE = os.getenv("APK_SERVER_RELEASE") + APK_SERVER_NIGHTLY = os.getenv("APK_SERVER_NIGHTLY") + WEBHOOK_RELEASE = os.getenv("WEBHOOK_RELEASE") + WEBHOOK_TESTING = os.getenv("WEBHOOK_TESTING") + + post_webhook( + project_dir, + APK_FILE, + APK_SERVER_RELEASE, + APK_SERVER_NIGHTLY, + WEBHOOK_RELEASE, + WEBHOOK_TESTING, + )