From b5b22c7123cdc35c555549179c5f6ff01caccd78 Mon Sep 17 00:00:00 2001 From: Philip Nagler-Frank Date: Fri, 18 Feb 2022 13:28:46 +0100 Subject: [PATCH] implement automatic updater --- FDroid/.version_code | 1 + FDroidPrivilegedExtension/.version_code | 1 + FakeStore/.version_code | 1 + GmsCore/.version_code | 1 + GsfProxy/.version_code | 1 + IchnaeaNlpBackend/.version_code | 1 + NominatimGeocoderBackend/.version_code | 1 + updater/.gitignore | 3 + updater/Pipfile | 12 +++ updater/Pipfile.lock | 107 ++++++++++++++++++++++++ updater/certificates.py | 11 +++ updater/git.py | 17 ++++ updater/main.py | 48 +++++++++++ updater/sources.py | 38 +++++++++ 14 files changed, 243 insertions(+) create mode 100644 FDroid/.version_code create mode 100644 FDroidPrivilegedExtension/.version_code create mode 100644 FakeStore/.version_code create mode 100644 GmsCore/.version_code create mode 100644 GsfProxy/.version_code create mode 100644 IchnaeaNlpBackend/.version_code create mode 100644 NominatimGeocoderBackend/.version_code create mode 100644 updater/.gitignore create mode 100644 updater/Pipfile create mode 100644 updater/Pipfile.lock create mode 100644 updater/certificates.py create mode 100644 updater/git.py create mode 100644 updater/main.py create mode 100644 updater/sources.py diff --git a/FDroid/.version_code b/FDroid/.version_code new file mode 100644 index 0000000..db03913 --- /dev/null +++ b/FDroid/.version_code @@ -0,0 +1 @@ +1014050 \ No newline at end of file diff --git a/FDroidPrivilegedExtension/.version_code b/FDroidPrivilegedExtension/.version_code new file mode 100644 index 0000000..f91847e --- /dev/null +++ b/FDroidPrivilegedExtension/.version_code @@ -0,0 +1 @@ +2130 \ No newline at end of file diff --git a/FakeStore/.version_code b/FakeStore/.version_code new file mode 100644 index 0000000..8fdd954 --- /dev/null +++ b/FakeStore/.version_code @@ -0,0 +1 @@ +22 \ No newline at end of file diff --git a/GmsCore/.version_code b/GmsCore/.version_code new file mode 100644 index 0000000..7d53558 --- /dev/null +++ b/GmsCore/.version_code @@ -0,0 +1 @@ +214816048 \ No newline at end of file diff --git a/GsfProxy/.version_code b/GsfProxy/.version_code new file mode 100644 index 0000000..301160a --- /dev/null +++ b/GsfProxy/.version_code @@ -0,0 +1 @@ +8 \ No newline at end of file diff --git a/IchnaeaNlpBackend/.version_code b/IchnaeaNlpBackend/.version_code new file mode 100644 index 0000000..f14154f --- /dev/null +++ b/IchnaeaNlpBackend/.version_code @@ -0,0 +1 @@ +20033 \ No newline at end of file diff --git a/NominatimGeocoderBackend/.version_code b/NominatimGeocoderBackend/.version_code new file mode 100644 index 0000000..046def1 --- /dev/null +++ b/NominatimGeocoderBackend/.version_code @@ -0,0 +1 @@ +20042 \ No newline at end of file diff --git a/updater/.gitignore b/updater/.gitignore new file mode 100644 index 0000000..25422a3 --- /dev/null +++ b/updater/.gitignore @@ -0,0 +1,3 @@ +/.idea +/*.iml +__pycache__ diff --git a/updater/Pipfile b/updater/Pipfile new file mode 100644 index 0000000..90dc377 --- /dev/null +++ b/updater/Pipfile @@ -0,0 +1,12 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +requests-cache = "==0.9.1" + +[dev-packages] + +[requires] +python_version = "3.9" diff --git a/updater/Pipfile.lock b/updater/Pipfile.lock new file mode 100644 index 0000000..8791c11 --- /dev/null +++ b/updater/Pipfile.lock @@ -0,0 +1,107 @@ +{ + "_meta": { + "hash": { + "sha256": "43267acd3e0a2938456d3924e25339a1da500cb295a1afee2e8a41843ba8af63" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.9" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "appdirs": { + "hashes": [ + "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41", + "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128" + ], + "version": "==1.4.4" + }, + "attrs": { + "hashes": [ + "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4", + "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==21.4.0" + }, + "cattrs": { + "hashes": [ + "sha256:211800f725cdecedcbcf4c753bbd22d248312b37d130f06045434acb7d9b34e1", + "sha256:35dd9063244263e63bd0bd24ea61e3015b00272cead084b2c40d788b0f857c46" + ], + "markers": "python_version >= '3.7' and python_version < '4.0'", + "version": "==1.10.0" + }, + "certifi": { + "hashes": [ + "sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872", + "sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569" + ], + "version": "==2021.10.8" + }, + "charset-normalizer": { + "hashes": [ + "sha256:2842d8f5e82a1f6aa437380934d5e1cd4fcf2003b06fed6940769c164a480a45", + "sha256:98398a9d69ee80548c762ba991a4728bfc3836768ed226b3945908d1a688371c" + ], + "markers": "python_version >= '3'", + "version": "==2.0.11" + }, + "idna": { + "hashes": [ + "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff", + "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d" + ], + "markers": "python_version >= '3'", + "version": "==3.3" + }, + "requests": { + "hashes": [ + "sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61", + "sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", + "version": "==2.27.1" + }, + "requests-cache": { + "hashes": [ + "sha256:3e3384c48dca231ee4c49e7ba53162bc0d99e16721baf085bfba1552d065d151", + "sha256:7737f83f0f48481a904bb9a9402233db5090931e46f9644c502646a573848d35" + ], + "index": "pypi", + "version": "==0.9.1" + }, + "six": { + "hashes": [ + "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", + "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.16.0" + }, + "url-normalize": { + "hashes": [ + "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2", + "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", + "version": "==1.4.3" + }, + "urllib3": { + "hashes": [ + "sha256:000ca7f471a233c2251c6c7023ee85305721bfdf18621ebff4fd17a8653427ed", + "sha256:0e7c33d9a63e7ddfcb86780aac87befc2fbddf46c58dbb487e0855f7ceec283c" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4.0'", + "version": "==1.26.8" + } + }, + "develop": {} +} diff --git a/updater/certificates.py b/updater/certificates.py new file mode 100644 index 0000000..547e4e7 --- /dev/null +++ b/updater/certificates.py @@ -0,0 +1,11 @@ +import subprocess + + +def get_apk_certificate(file: str): + output = subprocess.check_output(['keytool', '-printcert', '-rfc', '-jarfile', file], text=True) + lines = output.split("\n") + return '\n'.join(lines[ + lines.index('-----BEGIN CERTIFICATE-----'): + (lines.index('-----END CERTIFICATE-----')+1) + ]) + diff --git a/updater/git.py b/updater/git.py new file mode 100644 index 0000000..1c74b32 --- /dev/null +++ b/updater/git.py @@ -0,0 +1,17 @@ +import subprocess + + +user_name = 'Updater Robot' +user_email = 'robot@nowhere.invalid' + + +def add_commit_push(directory: str, message: str): + diff = subprocess.run(['git', 'diff', '--cached', '--exit-code'], capture_output=True, text=True) + if diff.returncode != 0: + status = subprocess.run(['git', 'status'], capture_output=True, text=True) + raise Exception('Unknown staged changes found: {}'.format(status.stdout)) + + subprocess.run(['git', 'add', '--all', directory], check=True) + subprocess.run(['git', '-c', 'user.name={}'.format(user_name), '-c', 'user.email={}'.format(user_email), + 'commit', '--message', message]) + subprocess.run(['git', 'push']) diff --git a/updater/main.py b/updater/main.py new file mode 100644 index 0000000..7452a74 --- /dev/null +++ b/updater/main.py @@ -0,0 +1,48 @@ +import urllib.request +from os import path + +import certificates +import git +from sources import ApkRelease, fdroid_recommended_release + + +def update_if_needed(module: str, release: ApkRelease): + module_dir = path.abspath(path.join(path.dirname(__file__), '..', module)) + with open(path.join(module_dir, '.version_code'), 'r+') as version_code_file: + version_code = int(version_code_file.read()) + if version_code < release.version_code: + print('updating {} to {}'.format(module, release.version_name)) + apk_filename = path.join(module_dir, '{}.apk'.format(module)) + + old_sig = certificates.get_apk_certificate(apk_filename) + + print('downloading {} ...'.format(release.download_url)) + urllib.request.urlretrieve(release.download_url, apk_filename) + + new_sig = certificates.get_apk_certificate(apk_filename) + if old_sig != new_sig: + raise Exception('Signature mismatch for {} old sig: {} new sig: {}'.format(module, old_sig, new_sig)) + + version_code_file.seek(0) + version_code_file.write(str(release.version_code)) + version_code_file.truncate() + version_code_file.close() + + print('commit and push...') + git.add_commit_push(module_dir, 'Update {} to {}'.format(module, release.version_name)) + + elif version_code > release.version_code: + print('{} ahead of suggested version ({} > {})'.format(module, version_code, release.version_code)) + elif version_code == release.version_code: + print('{} up to date.'.format(module)) + +fdroid_main_repo = 'https://www.f-droid.org/repo' +fdroid_microg_repo = 'https://microg.org/fdroid/repo' + +update_if_needed('FakeStore', fdroid_recommended_release(fdroid_microg_repo, 'com.android.vending')) +update_if_needed('FDroid', fdroid_recommended_release(fdroid_main_repo, 'org.fdroid.fdroid')) +update_if_needed('FDroidPrivilegedExtension', fdroid_recommended_release(fdroid_main_repo, 'org.fdroid.fdroid.privileged')) +update_if_needed('GmsCore', fdroid_recommended_release(fdroid_microg_repo, 'com.google.android.gms')) +update_if_needed('GsfProxy', fdroid_recommended_release(fdroid_microg_repo, 'com.google.android.gsf')) +update_if_needed('IchnaeaNlpBackend', fdroid_recommended_release(fdroid_main_repo, 'org.microg.nlp.backend.ichnaea')) +update_if_needed('NominatimGeocoderBackend', fdroid_recommended_release(fdroid_main_repo, 'org.microg.nlp.backend.nominatim')) diff --git a/updater/sources.py b/updater/sources.py new file mode 100644 index 0000000..51c0bbf --- /dev/null +++ b/updater/sources.py @@ -0,0 +1,38 @@ +from xml.dom import minidom, pulldom + +import requests_cache + +requests_session = requests_cache.CachedSession('updater', backend='memory') + + +class ApkRelease: + version_name: str + version_code: int + download_url: str + + def __init__(self, version_name: str, version_code: int, download_url: str): + self.version_name = version_name + self.version_code = version_code + self.download_url = download_url + + +def _child_el_content(el: minidom.Element, tag_name: str): + return el.getElementsByTagName(tag_name).item(0).firstChild.data + + +def fdroid_recommended_release(repo: str, application_id: str): + with requests_session.get('{}/index.xml'.format(repo)) as r: + doc = pulldom.parseString(r.text) + for event, node in doc: + if event == pulldom.START_ELEMENT and node.tagName == 'application': + if node.getAttribute('id') == application_id: + doc.expandNode(node) + marketvercode = _child_el_content(node, 'marketvercode') + for p in node.getElementsByTagName('package'): + if _child_el_content(p, 'versioncode') == marketvercode: + return ApkRelease( + _child_el_content(p, 'version'), + int(marketvercode), + '{}/{}'.format(repo, _child_el_content(p, 'apkname')) + ) + raise Exception('Did not find {} in repo {}'.format(application_id, repo))