Add fastlane metadata generation for Android builds

This is currently done on the signing machines as part of the F-Droid
nightly pipeline, but should rather happen as part of the build process
in the future.

Compared to the binary factory script this has a few extensions already:
- Besides recovering information from APKs we can now consume appdata
  files directly, or scan the entire source dir.
- Screenshots from appdata files are downloaded.
- The 'x-test' language is ignored.
- Donation and translation information are added.
- Add links to the source code repository, if we can determine that.

The result is put into a single archive per APK, so we can easily transfer
that to the signing machine via Jenkins alongside the APK.

2 files changed, 337 insertions, 0 deletions

diff --git a/toolchain/ECMAndroidDeployQt.cmake b/toolchain/ECMAndroidDeployQt.cmake
index 74f4c550..544f5a8d 100644
--- a/toolchain/ECMAndroidDeployQt.cmake
+++ b/toolchain/ECMAndroidDeployQt.cmake
@@ -1,5 +1,6 @@
 cmake_minimum_required (VERSION 3.7 FATAL_ERROR)
 find_package(Qt5Core REQUIRED)
+find_package(Python3 COMPONENTS Interpreter REQUIRED)
 
 function(ecm_androiddeployqt QTANDROID_EXPORTED_TARGET ECM_ADDITIONAL_FIND_ROOT_PATH)
     set(EXPORT_DIR "${CMAKE_BINARY_DIR}/${QTANDROID_EXPORTED_TARGET}_build_apk/")
@@ -64,6 +65,12 @@ function(ecm_androiddeployqt QTANDROID_EXPORTED_TARGET ECM_ADDITIONAL_FIND_ROOT_
 
     if (NOT TARGET create-apk)
         add_custom_target(create-apk)
+        if (NOT DEFINED ANDROID_FASTLANE_METADATA_OUTPUT_DIR)
+            set(ANDROID_FASTLANE_METADATA_OUTPUT_DIR ${CMAKE_BINARY_DIR}/fastlane)
+        endif()
+        add_custom_target(create-fastlane
+            COMMAND Python3::Interpreter ${CMAKE_CURRENT_LIST_DIR}/generate-fastlane-metadata.py --output ${ANDROID_FASTLANE_METADATA_OUTPUT_DIR} --source ${CMAKE_SOURCE_DIR}
+        )
     endif()
 
     if (NOT DEFINED ANDROID_APK_OUTPUT_DIR)
@@ -88,4 +95,5 @@ function(ecm_androiddeployqt QTANDROID_EXPORTED_TARGET ECM_ADDITIONAL_FIND_ROOT_
         COMMAND adb install -r "${ANDROID_APK_OUTPUT_DIR}/${QTANDROID_EXPORTED_TARGET}-${CMAKE_ANDROID_ARCH_ABI}.apk"
     )
     add_dependencies(create-apk ${CREATEAPK_TARGET_NAME})
+    add_dependencies(${CREATEAPK_TARGET_NAME} create-fastlane)
 endfunction()
diff --git a/toolchain/generate-fastlane-metadata.py b/toolchain/generate-fastlane-metadata.py
new file mode 100755
index 00000000..d1f6da67
--- /dev/null
+++ b/toolchain/generate-fastlane-metadata.py
@@ -0,0 +1,329 @@
+#!/usr/bin/env python3
+#
+# SPDX-FileCopyrightText: 2018-2020 Aleix Pol Gonzalez <aleixpol@kde.org>
+# SPDX-FileCopyrightText: 2019-2020 Ben Cooksley <bcooksley@kde.org>
+# SPDX-FileCopyrightText: 2020 Volker Krause <vkrause@kde.org>
+#
+# SPDX-License-Identifier: GPL-2.0-or-later
+#
+# Generates fastlane metadata for Android apps from appstream files.
+#
+
+import argparse
+import glob
+import io
+import os
+import re
+import requests
+import subprocess
+import sys
+import tempfile
+import xdg.DesktopEntry
+import xml.etree.ElementTree as ET
+import yaml
+import zipfile
+
+# Constants used in this script
+languageMap = {
+    None: "en-US",
+    "ca": "ca-ES"
+}
+
+# Android appdata.xml textual item parser
+# This function handles reading standard text entries within an Android appdata.xml file
+# In particular, it handles splitting out the various translations, and converts some HTML to something which F-Droid can make use of
+def readText(elem, found):
+    # Determine the language this entry is in
+    lang = elem.get('{http://www.w3.org/XML/1998/namespace}lang')
+    if lang == 'x-test':
+        return
+
+    # Do we have any text for this language yet?
+    # If not, get everything setup
+    if not lang in found:
+        found[lang] = ""
+
+    # Do we have a HTML List Item?
+    if elem.tag == 'li':
+        found[lang] += "· "
+
+    # If there is text available, we'll want to extract it
+    # Additionally, if this element has any children, make sure we read those as well
+    # It isn't clear if it is possible for an item to not have text, but to have children which do have text
+    # The code will currently skip these if they're encountered
+    if elem.text:
+        found[lang] += elem.text
+        for child in elem:
+            readText(child, found)
+
+    # Finally, if this element is a HTML Paragraph (p) or HTML List Item (li) make sure we add a new line for presentation purposes
+    if elem.tag == 'li' or elem.tag == 'p':
+        found[lang] += "\n"
+
+
+# Create the various Fastlane format files per the information we've previously extracted
+# These files are laid out following the Fastlane specification (links below)
+# https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots
+# https://docs.fastlane.tools/actions/supply/
+def createFastlaneFile( applicationName, filenameToPopulate, fileContent ):
+    # Go through each language and content pair we've been given
+    for lang, text in fileContent.items():
+        # First, do we need to amend the language id, to turn the Android language ID into something more F-Droid/Fastlane friendly?
+        languageCode = languageMap.get(lang, lang)
+
+        # Next we need to determine the path to the directory we're going to be writing the data into
+        repositoryBasePath = arguments.output
+        path = os.path.join( repositoryBasePath, 'metadata',  applicationName, languageCode )
+
+        # Make sure the directory exists
+        os.makedirs(path, exist_ok=True)
+
+        # Now write out file contents!
+        with open(path + '/' + filenameToPopulate, 'w') as f:
+            f.write(text)
+
+# Create the summary appname.yml file used by F-Droid to summarise this particular entry in the repository
+# see https://f-droid.org/en/docs/Build_Metadata_Reference/
+def createYml(appname, data):
+    # Prepare to retrieve the existing information
+    info = {}
+
+    # Determine the path to the appname.yml file
+    repositoryBasePath = arguments.output
+    path = os.path.join( repositoryBasePath, 'metadata', appname + '.yml' )
+
+    # Update the categories first
+    # Now is also a good time to add 'KDE' to the list of categories as well
+    info['Categories'] = data['categories'][None] + ['KDE']
+
+    # Update the general sumamry as well
+    info['Summary'] = data['summary'][None]
+
+    # Check to see if we have a Homepage...
+    if 'url-homepage' in data:
+        info['WebSite'] = data['url-homepage'][None]
+
+    # What about a bug tracker?
+    if 'url-bugtracker' in data:
+        info['IssueTracker'] = data['url-bugtracker'][None]
+
+    if 'project_license' in data:
+        info["License"] = data['project_license'][None]
+
+    if 'source-repo' in data:
+        info['SourceCode'] = data['source-repo']
+
+    # static data
+    info['Donate'] = 'https://kde.org/community/donations/'
+    info['Translation'] = 'https://l10n.kde.org/'
+
+    # Finally, with our updates completed, we can save the updated appname.yml file back to disk
+    with open(path, 'w') as output:
+        yaml.dump(info, output, default_flow_style=False)
+
+# Download screenshots referenced in the appstream data
+# see https://f-droid.org/en/docs/All_About_Descriptions_Graphics_and_Screenshots/
+def downloadScreenshots(applicationName, data):
+    if not 'screenshots' in data:
+        return
+
+    basePath = arguments.output
+    path = os.path.join(basePath, 'metadata',  applicationName, 'en-US', 'images', 'phoneScreenshots')
+    os.makedirs(path, exist_ok=True)
+
+    for screenshot in data['screenshots']:
+        fileName = screenshot[screenshot.rindex('/') + 1:]
+        r = requests.get(screenshot)
+        with open(os.path.join(path, fileName), 'wb') as f:
+            f.write(r.content)
+
+# Put all metadata for the given application name into an archive
+# We need this to easily transfer the entire metadata to the signing machine for integration
+# into the F-Droid nightly repository
+def createMetadataArchive(applicationName):
+    srcPath = os.path.join(arguments.output, 'metadata')
+    archive = zipfile.ZipFile(os.path.join(srcPath, 'fastlane-' + applicationName + '.zip'), 'w')
+    archive.write(os.path.join(srcPath, applicationName + '.yml'), applicationName + '.yml')
+
+    oldcwd = os.getcwd()
+    os.chdir(srcPath)
+    for file in glob.iglob(applicationName + '/**', recursive=True):
+        archive.write(file, file)
+    os.chdir(oldcwd)
+
+# Main function for extracting metadata from APK files
+def processApkFile(apkFilepath):
+    # First, determine the name of the application we have here
+    # This is needed in order to locate the metadata files within the APK that have the information we need
+
+    # Prepare the aapt (Android SDK command) to inspect the provided APK
+    commandToRun = "aapt dump badging %s" % (apkFilepath)
+    manifest = subprocess.check_output( commandToRun, shell=True ).decode('utf-8')
+    # Search through the aapt output for the name of the application
+    result = re.search(' name=\'([^\']*)\'', manifest)
+    applicationName = result.group(1)
+
+    # Attempt to look within the APK provided for the metadata information we will need
+    with zipfile.ZipFile(apkFilepath, 'r') as contents:
+        appdataFile = contents.open("assets/share/metainfo/%s.appdata.xml" % applicationName)
+        desktopFileContent = None
+        try:
+            desktopFileContent = contents.read("assets/share/applications/%s.desktop" % applicationName)
+        except:
+            None
+        processAppstreamData(applicationName, appdataFile.read(), desktopFileContent)
+
+# Extract meta data from appstream/desktop file contents
+def processAppstreamData(applicationName, appstreamData, desktopData):
+    data = {}
+    # Within this file we look at every entry, and where possible try to export it's content so we can use it later
+    root = ET.fromstring(appstreamData)
+    for child in root:
+        # Make sure we start with a blank slate for this entry
+        output = {}
+
+        # Grab the name of this particular attribute we're looking at
+        # Within the Fastlane specification, it is possible to have several items with the same name but as different types
+        # We therefore include this within our extracted name for the attribute to differentiate them
+        tag = child.tag
+        if 'type' in child.attrib:
+            tag += '-' + child.attrib['type']
+
+        # Have we found some information already for this particular attribute?
+        if tag in data:
+            output = data[tag]
+
+        # Are we dealing with category information here?
+        # If so, then we need to look into this items children to find out all the categories this APK belongs in
+        if tag == 'categories':
+            cats = []
+            for x in child:
+                cats.append(x.text)
+            output = { None: cats }
+
+        # screenshot links
+        elif tag == 'screenshots':
+            output = []
+            for screenshot in child:
+                if screenshot.tag == 'screenshot':
+                    for image in screenshot:
+                        if image.tag == 'image':
+                            output.append(image.text)
+
+        # Otherwise this is just textual information we need to extract
+        else:
+            readText(child, output)
+
+        # Save the information we've gathered!
+        data[tag] = output
+
+    # Did we find any categories?
+    # Sometimes we don't find any within the Fastlane information, but without categories the F-Droid store isn't of much use
+    # In the event this happens, fallback to the *.desktop file for the application to see if it can provide any insight.
+    if not 'categories' in data and desktopData:
+        # The Python XDG extension/wrapper requires that it be able to read the file itself
+        # To ensure it is able to do this, we transfer the content of the file from the APK out to a temporary file to keep it happy
+        (fd, path) = tempfile.mkstemp(suffix=name + ".desktop")
+        handle = open(fd, "wb")
+        handle.write(desktopFileContents.read())
+        handle.close()
+        # Parse the XDG format *.desktop file, and extract the categories within it
+        desktopFile = xdg.DesktopEntry.DesktopEntry(path)
+        data['categories'] = { None: desktopFile.getCategories() }
+
+    # Try to figure out the source repository
+    if arguments.source and os.path.exists(os.path.join(arguments.source, '.git')):
+        output = subprocess.check_output('git remote show -n origin', shell=True, cwd = arguments.source).decode('utf-8')
+        result = re.search(' Fetch URL: (.*)\n', output)
+        data['source-repo'] = result.group(1)
+
+    # write meta data
+    createFastlaneFile( applicationName, "title.txt", data['name'] )
+    createFastlaneFile( applicationName, "short_description.txt", data['summary'] )
+    createFastlaneFile( applicationName, "full_description.txt", data['description'] )
+    createYml(applicationName, data)
+    downloadScreenshots(applicationName, data)
+    createMetadataArchive(applicationName)
+
+# Generate metadata for the given appstream and desktop files
+def processAppstreamFile(appstreamFileName, desktopFileName):
+    appstreamFile = open(appstreamFileName, "rb");
+    desktopData = None
+    if desktopFileName and os.path.exists(desktopFileName):
+        desktopFile = open(desktopFileName, "rb");
+        desktopData = desktopFile.read();
+    applicationName = os.path.basename(appstreamFileName)[:-12]
+    processAppstreamData(applicationName, appstreamFile.read(), desktopData)
+
+# scan source directory for manifests/metadata we can work with
+def scanSourceDir():
+    files = glob.iglob(arguments.source + "/**/AndroidManifest.xml*", recursive=True)
+    for file in files:
+        # third-party libraries might contain AndroidManifests which we are not interested in
+        if "3rdparty" in file:
+            continue
+
+        # find application id from manifest files
+        root = ET.parse(file)
+        appname = root.getroot().attrib['package']
+        is_app = False
+        prefix = '{http://schemas.android.com/apk/res/android}'
+        for md in root.findall("application/activity/meta-data"):
+            if md.attrib[prefix + 'name'] == 'android.app.lib_name':
+                is_app = True
+
+        if not appname or not is_app:
+            continue
+
+        # now that we have the app id, look for matching appdata/desktop files
+        appdataFiles = glob.iglob(arguments.source + "/**/" + appname + ".appdata.xml", recursive=True)
+        appdataFile = None
+        for f in appdataFiles:
+            appdataFile = f;
+            break;
+        if not appdataFile:
+            continue
+
+        desktopFiles = glob.iglob(arguments.source + "/**/" + appname + ".desktop", recursive=True)
+        desktopFile = None
+        for f in desktopFiles:
+            desktopFile = f;
+            break;
+
+        processAppstreamFile(appdataFile, desktopFile)
+
+
+### Script Commences
+
+# Parse the command line arguments we've been given
+parser = argparse.ArgumentParser(description='Generate fastlane metadata for Android apps from appstream metadata')
+parser.add_argument('--apk', type=str, required=False, help='APK file to extract metadata from')
+parser.add_argument('--appstream', type=str, required=False, help='Appstream file to extract metadata from')
+parser.add_argument('--desktop', type=str, required=False, help='Desktop file to extract additional metadata from')
+parser.add_argument('--source', type=str, required=False, help='Source directory to find metadata in')
+parser.add_argument('--output', type=str, required=True, help='Path to which the metadata output should be written to')
+arguments = parser.parse_args()
+
+# ensure the output path exists
+os.makedirs(arguments.output, exist_ok=True)
+
+# if we have an appstream file explicitly specified, let's use that one
+if arguments.appstream and os.path.exists(arguments.appstream):
+    processAppstreamFile(arguments.appstream, arguments.desktop)
+    sys.exit(0)
+
+# else, if we have an APK, try to find the appstream file in there
+# this ensures compatibility with the old metadata generation
+if arguments.apk and os.path.exists(arguments.apk):
+    processApkFile(arguments.apk)
+    sys.exit(0)
+
+# else, look in the source dir for appstream/desktop files
+# this follows roughly what get-apk-args from binary factory does
+if arguments.source and os.path.exists(arguments.source):
+    scanSourceDir()
+    sys.exit(0)
+
+# else: missing arguments
+print("Either one of --appstream, --apk or --source have to be provided!")
+sys.exit(1)