Merge pull request #4 from lineageos4microg/automate_updates

Automate updates

.github/workflows/updater.yml

@@ -0,0 +1,24 @@
+name: Updater
+on:
+  schedule:
+    - cron:  '37 13 * * *'
+  workflow_dispatch:
+jobs:
+  Update-Apks:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-java@v2
+        with:
+          distribution: 'temurin'
+          java-version: '17'
+      - run: pipx install pipenv
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.9'
+          cache: 'pipenv'
+          cache-dependency-path: 'updater/Pipfile.lock'
+      - run: pipenv install
+        working-directory: updater
+      - run: pipenv run python main.py
+        working-directory: updater

FDroid/.version_code

@@ -0,0 +1 @@
+1014050

FDroid/Android.mk

@@ -2,7 +2,7 @@ LOCAL_PATH := $(call my-dir)
 include $(CLEAR_VARS)
 LOCAL_MODULE_TAGS := optional
 LOCAL_MODULE := FDroid
-LOCAL_SRC_FILES := org.fdroid.fdroid_1014050.apk
+LOCAL_SRC_FILES := FDroid.apk
 LOCAL_MODULE_CLASS := APPS
 LOCAL_MODULE_SUFFIX := $(COMMON_ANDROID_PACKAGE_SUFFIX)
 LOCAL_CERTIFICATE := PRESIGNED

FDroidPrivilegedExtension/.version_code

@@ -0,0 +1 @@
+2130

FDroidPrivilegedExtension/Android.mk

@@ -11,7 +11,7 @@ include $(BUILD_PREBUILT)
 include $(CLEAR_VARS)
 LOCAL_MODULE_TAGS := optional
 LOCAL_MODULE := FDroidPrivilegedExtension
-LOCAL_SRC_FILES := org.fdroid.fdroid.privileged_2130.apk
+LOCAL_SRC_FILES := FDroidPrivilegedExtension.apk
 LOCAL_MODULE_CLASS := APPS
 LOCAL_PRIVILEGED_MODULE := true
 LOCAL_MODULE_SUFFIX := $(COMMON_ANDROID_PACKAGE_SUFFIX)

FakeStore/.version_code

@@ -0,0 +1 @@
+22

FakeStore/Android.mk

@@ -11,7 +11,7 @@ include $(BUILD_PREBUILT)
 include $(CLEAR_VARS)
 LOCAL_MODULE_TAGS := optional
 LOCAL_MODULE := FakeStore
-LOCAL_SRC_FILES := com.android.vending-22.apk
+LOCAL_SRC_FILES := FakeStore.apk
 LOCAL_MODULE_CLASS := APPS
 LOCAL_PRIVILEGED_MODULE := true
 LOCAL_MODULE_SUFFIX := $(COMMON_ANDROID_PACKAGE_SUFFIX)

GmsCore/.version_code

@@ -0,0 +1 @@
+214816048

GmsCore/Android.mk

@@ -19,7 +19,7 @@ include $(BUILD_PREBUILT)
 include $(CLEAR_VARS)
 LOCAL_MODULE_TAGS := optional
 LOCAL_MODULE := GmsCore
-LOCAL_SRC_FILES := com.google.android.gms-214816048.apk
+LOCAL_SRC_FILES := GmsCore.apk
 LOCAL_MODULE_CLASS := APPS
 LOCAL_PRIVILEGED_MODULE := true
 LOCAL_MODULE_SUFFIX := $(COMMON_ANDROID_PACKAGE_SUFFIX)

GsfProxy/.version_code

@@ -0,0 +1 @@
+8

GsfProxy/Android.mk

@@ -2,7 +2,7 @@ LOCAL_PATH := $(call my-dir)
 include $(CLEAR_VARS)
 LOCAL_MODULE_TAGS := optional
 LOCAL_MODULE := GsfProxy
-LOCAL_SRC_FILES := com.google.android.gsf-8.apk
+LOCAL_SRC_FILES := GsfProxy.apk
 LOCAL_MODULE_CLASS := APPS
 LOCAL_MODULE_SUFFIX := $(COMMON_ANDROID_PACKAGE_SUFFIX)
 LOCAL_CERTIFICATE := PRESIGNED

IchnaeaNlpBackend/.version_code

@@ -0,0 +1 @@
+20033

IchnaeaNlpBackend/Android.mk

@@ -2,7 +2,7 @@ LOCAL_PATH := $(call my-dir)
 include $(CLEAR_VARS)
 LOCAL_MODULE_TAGS := optional
 LOCAL_MODULE := IchnaeaNlpBackend
-LOCAL_SRC_FILES := org.microg.nlp.backend.ichnaea_20033.apk
+LOCAL_SRC_FILES := IchnaeaNlpBackend.apk
 LOCAL_MODULE_CLASS := APPS
 LOCAL_MODULE_SUFFIX := $(COMMON_ANDROID_PACKAGE_SUFFIX)
 LOCAL_CERTIFICATE := PRESIGNED

NominatimGeocoderBackend/.version_code

@@ -0,0 +1 @@
+20042

NominatimGeocoderBackend/Android.mk

@@ -2,7 +2,7 @@ LOCAL_PATH := $(call my-dir)
 include $(CLEAR_VARS)
 LOCAL_MODULE_TAGS := optional
 LOCAL_MODULE := NominatimGeocoderBackend
-LOCAL_SRC_FILES := org.microg.nlp.backend.nominatim_20042.apk
+LOCAL_SRC_FILES := NominatimGeocoderBackend.apk
 LOCAL_MODULE_CLASS := APPS
 LOCAL_MODULE_SUFFIX := $(COMMON_ANDROID_PACKAGE_SUFFIX)
 LOCAL_CERTIFICATE := PRESIGNED

updater/.gitignore

@@ -0,0 +1,3 @@
+/.idea
+/*.iml
+__pycache__

updater/Pipfile

@@ -0,0 +1,12 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+requests-cache = "==0.9.1"
+
+[dev-packages]
+
+[requires]
+python_version = "3.9"

updater/Pipfile.lock

@@ -0,0 +1,107 @@
+{
+    "_meta": {
+        "hash": {
+            "sha256": "43267acd3e0a2938456d3924e25339a1da500cb295a1afee2e8a41843ba8af63"
+        },
+        "pipfile-spec": 6,
+        "requires": {
+            "python_version": "3.9"
+        },
+        "sources": [
+            {
+                "name": "pypi",
+                "url": "https://pypi.org/simple",
+                "verify_ssl": true
+            }
+        ]
+    },
+    "default": {
+        "appdirs": {
+            "hashes": [
+                "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41",
+                "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128"
+            ],
+            "version": "==1.4.4"
+        },
+        "attrs": {
+            "hashes": [
+                "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4",
+                "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
+            "version": "==21.4.0"
+        },
+        "cattrs": {
+            "hashes": [
+                "sha256:211800f725cdecedcbcf4c753bbd22d248312b37d130f06045434acb7d9b34e1",
+                "sha256:35dd9063244263e63bd0bd24ea61e3015b00272cead084b2c40d788b0f857c46"
+            ],
+            "markers": "python_version >= '3.7' and python_version < '4.0'",
+            "version": "==1.10.0"
+        },
+        "certifi": {
+            "hashes": [
+                "sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872",
+                "sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569"
+            ],
+            "version": "==2021.10.8"
+        },
+        "charset-normalizer": {
+            "hashes": [
+                "sha256:2842d8f5e82a1f6aa437380934d5e1cd4fcf2003b06fed6940769c164a480a45",
+                "sha256:98398a9d69ee80548c762ba991a4728bfc3836768ed226b3945908d1a688371c"
+            ],
+            "markers": "python_version >= '3'",
+            "version": "==2.0.11"
+        },
+        "idna": {
+            "hashes": [
+                "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff",
+                "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"
+            ],
+            "markers": "python_version >= '3'",
+            "version": "==3.3"
+        },
+        "requests": {
+            "hashes": [
+                "sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61",
+                "sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'",
+            "version": "==2.27.1"
+        },
+        "requests-cache": {
+            "hashes": [
+                "sha256:3e3384c48dca231ee4c49e7ba53162bc0d99e16721baf085bfba1552d065d151",
+                "sha256:7737f83f0f48481a904bb9a9402233db5090931e46f9644c502646a573848d35"
+            ],
+            "index": "pypi",
+            "version": "==0.9.1"
+        },
+        "six": {
+            "hashes": [
+                "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
+                "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==1.16.0"
+        },
+        "url-normalize": {
+            "hashes": [
+                "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2",
+                "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'",
+            "version": "==1.4.3"
+        },
+        "urllib3": {
+            "hashes": [
+                "sha256:000ca7f471a233c2251c6c7023ee85305721bfdf18621ebff4fd17a8653427ed",
+                "sha256:0e7c33d9a63e7ddfcb86780aac87befc2fbddf46c58dbb487e0855f7ceec283c"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4.0'",
+            "version": "==1.26.8"
+        }
+    },
+    "develop": {}
+}

updater/certificates.py

@@ -0,0 +1,11 @@
+import subprocess
+
+
+def get_apk_certificate(file: str):
+    output = subprocess.check_output(['keytool', '-printcert', '-rfc', '-jarfile', file], text=True)
+    lines = output.split("\n")
+    return '\n'.join(lines[
+        lines.index('-----BEGIN CERTIFICATE-----'):
+        (lines.index('-----END CERTIFICATE-----')+1)
+    ])
+

updater/git.py

@@ -0,0 +1,17 @@
+import subprocess
+
+
+user_name = 'Updater Robot'
+user_email = 'robot@nowhere.invalid'
+
+
+def add_commit_push(directory: str, message: str):
+    diff = subprocess.run(['git', 'diff', '--cached', '--exit-code'], capture_output=True, text=True)
+    if diff.returncode != 0:
+        status = subprocess.run(['git', 'status'], capture_output=True, text=True)
+        raise Exception('Unknown staged changes found: {}'.format(status.stdout))
+
+    subprocess.run(['git', 'add', '--all', directory], check=True)
+    subprocess.run(['git', '-c', 'user.name={}'.format(user_name), '-c', 'user.email={}'.format(user_email),
+                    'commit', '--message', message])
+    subprocess.run(['git', 'push'])

updater/main.py

@@ -0,0 +1,48 @@
+import urllib.request
+from os import path
+
+import certificates
+import git
+from sources import ApkRelease, fdroid_recommended_release
+
+
+def update_if_needed(module: str, release: ApkRelease):
+    module_dir = path.abspath(path.join(path.dirname(__file__), '..', module))
+    with open(path.join(module_dir, '.version_code'), 'r+') as version_code_file:
+        version_code = int(version_code_file.read())
+        if version_code < release.version_code:
+            print('updating {} to {}'.format(module, release.version_name))
+            apk_filename = path.join(module_dir, '{}.apk'.format(module))
+
+            old_sig = certificates.get_apk_certificate(apk_filename)
+
+            print('downloading {} ...'.format(release.download_url))
+            urllib.request.urlretrieve(release.download_url, apk_filename)
+
+            new_sig = certificates.get_apk_certificate(apk_filename)
+            if old_sig != new_sig:
+                raise Exception('Signature mismatch for {} old sig: {} new sig: {}'.format(module, old_sig, new_sig))
+
+            version_code_file.seek(0)
+            version_code_file.write(str(release.version_code))
+            version_code_file.truncate()
+            version_code_file.close()
+
+            print('commit and push...')
+            git.add_commit_push(module_dir, 'Update {} to {}'.format(module, release.version_name))
+
+        elif version_code > release.version_code:
+            print('{} ahead of suggested version ({} > {})'.format(module, version_code, release.version_code))
+        elif version_code == release.version_code:
+            print('{} up to date.'.format(module))
+
+fdroid_main_repo = 'https://www.f-droid.org/repo'
+fdroid_microg_repo = 'https://microg.org/fdroid/repo'
+
+update_if_needed('FakeStore', fdroid_recommended_release(fdroid_microg_repo, 'com.android.vending'))
+update_if_needed('FDroid', fdroid_recommended_release(fdroid_main_repo, 'org.fdroid.fdroid'))
+update_if_needed('FDroidPrivilegedExtension', fdroid_recommended_release(fdroid_main_repo, 'org.fdroid.fdroid.privileged'))
+update_if_needed('GmsCore', fdroid_recommended_release(fdroid_microg_repo, 'com.google.android.gms'))
+update_if_needed('GsfProxy', fdroid_recommended_release(fdroid_microg_repo, 'com.google.android.gsf'))
+update_if_needed('IchnaeaNlpBackend', fdroid_recommended_release(fdroid_main_repo, 'org.microg.nlp.backend.ichnaea'))
+update_if_needed('NominatimGeocoderBackend', fdroid_recommended_release(fdroid_main_repo, 'org.microg.nlp.backend.nominatim'))

updater/sources.py

@@ -0,0 +1,38 @@
+from xml.dom import minidom, pulldom
+
+import requests_cache
+
+requests_session = requests_cache.CachedSession('updater', backend='memory')
+
+
+class ApkRelease:
+    version_name: str
+    version_code: int
+    download_url: str
+
+    def __init__(self, version_name: str, version_code: int, download_url: str):
+        self.version_name = version_name
+        self.version_code = version_code
+        self.download_url = download_url
+
+
+def _child_el_content(el: minidom.Element, tag_name: str):
+    return el.getElementsByTagName(tag_name).item(0).firstChild.data
+
+
+def fdroid_recommended_release(repo: str, application_id: str):
+    with requests_session.get('{}/index.xml'.format(repo)) as r:
+        doc = pulldom.parseString(r.text)
+        for event, node in doc:
+            if event == pulldom.START_ELEMENT and node.tagName == 'application':
+                if node.getAttribute('id') == application_id:
+                    doc.expandNode(node)
+                    marketvercode = _child_el_content(node, 'marketvercode')
+                    for p in node.getElementsByTagName('package'):
+                        if _child_el_content(p, 'versioncode') == marketvercode:
+                            return ApkRelease(
+                                _child_el_content(p, 'version'),
+                                int(marketvercode),
+                                '{}/{}'.format(repo, _child_el_content(p, 'apkname'))
+                            )
+        raise Exception('Did not find {} in repo {}'.format(application_id, repo))