diff options
| -rw-r--r-- | toolchain/ECMAndroidDeployQt.cmake | 8 | ||||
| -rwxr-xr-x | toolchain/generate-fastlane-metadata.py | 329 |
2 files changed, 337 insertions, 0 deletions
diff --git a/toolchain/ECMAndroidDeployQt.cmake b/toolchain/ECMAndroidDeployQt.cmake index 74f4c550..544f5a8d 100644 --- a/toolchain/ECMAndroidDeployQt.cmake +++ b/toolchain/ECMAndroidDeployQt.cmake @@ -1,5 +1,6 @@ cmake_minimum_required (VERSION 3.7 FATAL_ERROR) find_package(Qt5Core REQUIRED) +find_package(Python3 COMPONENTS Interpreter REQUIRED) function(ecm_androiddeployqt QTANDROID_EXPORTED_TARGET ECM_ADDITIONAL_FIND_ROOT_PATH) set(EXPORT_DIR "${CMAKE_BINARY_DIR}/${QTANDROID_EXPORTED_TARGET}_build_apk/") @@ -64,6 +65,12 @@ function(ecm_androiddeployqt QTANDROID_EXPORTED_TARGET ECM_ADDITIONAL_FIND_ROOT_ if (NOT TARGET create-apk) add_custom_target(create-apk) + if (NOT DEFINED ANDROID_FASTLANE_METADATA_OUTPUT_DIR) + set(ANDROID_FASTLANE_METADATA_OUTPUT_DIR ${CMAKE_BINARY_DIR}/fastlane) + endif() + add_custom_target(create-fastlane + COMMAND Python3::Interpreter ${CMAKE_CURRENT_LIST_DIR}/generate-fastlane-metadata.py --output ${ANDROID_FASTLANE_METADATA_OUTPUT_DIR} --source ${CMAKE_SOURCE_DIR} + ) endif() if (NOT DEFINED ANDROID_APK_OUTPUT_DIR) @@ -88,4 +95,5 @@ function(ecm_androiddeployqt QTANDROID_EXPORTED_TARGET ECM_ADDITIONAL_FIND_ROOT_ COMMAND adb install -r "${ANDROID_APK_OUTPUT_DIR}/${QTANDROID_EXPORTED_TARGET}-${CMAKE_ANDROID_ARCH_ABI}.apk" ) add_dependencies(create-apk ${CREATEAPK_TARGET_NAME}) + add_dependencies(${CREATEAPK_TARGET_NAME} create-fastlane) endfunction() diff --git a/toolchain/generate-fastlane-metadata.py b/toolchain/generate-fastlane-metadata.py new file mode 100755 index 00000000..d1f6da67 --- /dev/null +++ b/toolchain/generate-fastlane-metadata.py @@ -0,0 +1,329 @@ +#!/usr/bin/env python3 +# +# SPDX-FileCopyrightText: 2018-2020 Aleix Pol Gonzalez <aleixpol@kde.org> +# SPDX-FileCopyrightText: 2019-2020 Ben Cooksley <bcooksley@kde.org> +# SPDX-FileCopyrightText: 2020 Volker Krause <vkrause@kde.org> +# +# SPDX-License-Identifier: GPL-2.0-or-later +# +# Generates fastlane metadata for Android apps from appstream files. +# + +import argparse +import glob +import io +import os +import re +import requests +import subprocess +import sys +import tempfile +import xdg.DesktopEntry +import xml.etree.ElementTree as ET +import yaml +import zipfile + +# Constants used in this script +languageMap = { + None: "en-US", + "ca": "ca-ES" +} + +# Android appdata.xml textual item parser +# This function handles reading standard text entries within an Android appdata.xml file +# In particular, it handles splitting out the various translations, and converts some HTML to something which F-Droid can make use of +def readText(elem, found): + # Determine the language this entry is in + lang = elem.get('{http://www.w3.org/XML/1998/namespace}lang') + if lang == 'x-test': + return + + # Do we have any text for this language yet? + # If not, get everything setup + if not lang in found: + found[lang] = "" + + # Do we have a HTML List Item? + if elem.tag == 'li': + found[lang] += "ยท " + + # If there is text available, we'll want to extract it + # Additionally, if this element has any children, make sure we read those as well + # It isn't clear if it is possible for an item to not have text, but to have children which do have text + # The code will currently skip these if they're encountered + if elem.text: + found[lang] += elem.text + for child in elem: + readText(child, found) + + # Finally, if this element is a HTML Paragraph (p) or HTML List Item (li) make sure we add a new line for presentation purposes + if elem.tag == 'li' or elem.tag == 'p': + found[lang] += "\n" + + +# Create the various Fastlane format files per the information we've previously extracted +# These files are laid out following the Fastlane specification (links below) +# https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots +# https://docs.fastlane.tools/actions/supply/ +def createFastlaneFile( applicationName, filenameToPopulate, fileContent ): + # Go through each language and content pair we've been given + for lang, text in fileContent.items(): + # First, do we need to amend the language id, to turn the Android language ID into something more F-Droid/Fastlane friendly? + languageCode = languageMap.get(lang, lang) + + # Next we need to determine the path to the directory we're going to be writing the data into + repositoryBasePath = arguments.output + path = os.path.join( repositoryBasePath, 'metadata', applicationName, languageCode ) + + # Make sure the directory exists + os.makedirs(path, exist_ok=True) + + # Now write out file contents! + with open(path + '/' + filenameToPopulate, 'w') as f: + f.write(text) + +# Create the summary appname.yml file used by F-Droid to summarise this particular entry in the repository +# see https://f-droid.org/en/docs/Build_Metadata_Reference/ +def createYml(appname, data): + # Prepare to retrieve the existing information + info = {} + + # Determine the path to the appname.yml file + repositoryBasePath = arguments.output + path = os.path.join( repositoryBasePath, 'metadata', appname + '.yml' ) + + # Update the categories first + # Now is also a good time to add 'KDE' to the list of categories as well + info['Categories'] = data['categories'][None] + ['KDE'] + + # Update the general sumamry as well + info['Summary'] = data['summary'][None] + + # Check to see if we have a Homepage... + if 'url-homepage' in data: + info['WebSite'] = data['url-homepage'][None] + + # What about a bug tracker? + if 'url-bugtracker' in data: + info['IssueTracker'] = data['url-bugtracker'][None] + + if 'project_license' in data: + info["License"] = data['project_license'][None] + + if 'source-repo' in data: + info['SourceCode'] = data['source-repo'] + + # static data + info['Donate'] = 'https://kde.org/community/donations/' + info['Translation'] = 'https://l10n.kde.org/' + + # Finally, with our updates completed, we can save the updated appname.yml file back to disk + with open(path, 'w') as output: + yaml.dump(info, output, default_flow_style=False) + +# Download screenshots referenced in the appstream data +# see https://f-droid.org/en/docs/All_About_Descriptions_Graphics_and_Screenshots/ +def downloadScreenshots(applicationName, data): + if not 'screenshots' in data: + return + + basePath = arguments.output + path = os.path.join(basePath, 'metadata', applicationName, 'en-US', 'images', 'phoneScreenshots') + os.makedirs(path, exist_ok=True) + + for screenshot in data['screenshots']: + fileName = screenshot[screenshot.rindex('/') + 1:] + r = requests.get(screenshot) + with open(os.path.join(path, fileName), 'wb') as f: + f.write(r.content) + +# Put all metadata for the given application name into an archive +# We need this to easily transfer the entire metadata to the signing machine for integration +# into the F-Droid nightly repository +def createMetadataArchive(applicationName): + srcPath = os.path.join(arguments.output, 'metadata') + archive = zipfile.ZipFile(os.path.join(srcPath, 'fastlane-' + applicationName + '.zip'), 'w') + archive.write(os.path.join(srcPath, applicationName + '.yml'), applicationName + '.yml') + + oldcwd = os.getcwd() + os.chdir(srcPath) + for file in glob.iglob(applicationName + '/**', recursive=True): + archive.write(file, file) + os.chdir(oldcwd) + +# Main function for extracting metadata from APK files +def processApkFile(apkFilepath): + # First, determine the name of the application we have here + # This is needed in order to locate the metadata files within the APK that have the information we need + + # Prepare the aapt (Android SDK command) to inspect the provided APK + commandToRun = "aapt dump badging %s" % (apkFilepath) + manifest = subprocess.check_output( commandToRun, shell=True ).decode('utf-8') + # Search through the aapt output for the name of the application + result = re.search(' name=\'([^\']*)\'', manifest) + applicationName = result.group(1) + + # Attempt to look within the APK provided for the metadata information we will need + with zipfile.ZipFile(apkFilepath, 'r') as contents: + appdataFile = contents.open("assets/share/metainfo/%s.appdata.xml" % applicationName) + desktopFileContent = None + try: + desktopFileContent = contents.read("assets/share/applications/%s.desktop" % applicationName) + except: + None + processAppstreamData(applicationName, appdataFile.read(), desktopFileContent) + +# Extract meta data from appstream/desktop file contents +def processAppstreamData(applicationName, appstreamData, desktopData): + data = {} + # Within this file we look at every entry, and where possible try to export it's content so we can use it later + root = ET.fromstring(appstreamData) + for child in root: + # Make sure we start with a blank slate for this entry + output = {} + + # Grab the name of this particular attribute we're looking at + # Within the Fastlane specification, it is possible to have several items with the same name but as different types + # We therefore include this within our extracted name for the attribute to differentiate them + tag = child.tag + if 'type' in child.attrib: + tag += '-' + child.attrib['type'] + + # Have we found some information already for this particular attribute? + if tag in data: + output = data[tag] + + # Are we dealing with category information here? + # If so, then we need to look into this items children to find out all the categories this APK belongs in + if tag == 'categories': + cats = [] + for x in child: + cats.append(x.text) + output = { None: cats } + + # screenshot links + elif tag == 'screenshots': + output = [] + for screenshot in child: + if screenshot.tag == 'screenshot': + for image in screenshot: + if image.tag == 'image': + output.append(image.text) + + # Otherwise this is just textual information we need to extract + else: + readText(child, output) + + # Save the information we've gathered! + data[tag] = output + + # Did we find any categories? + # Sometimes we don't find any within the Fastlane information, but without categories the F-Droid store isn't of much use + # In the event this happens, fallback to the *.desktop file for the application to see if it can provide any insight. + if not 'categories' in data and desktopData: + # The Python XDG extension/wrapper requires that it be able to read the file itself + # To ensure it is able to do this, we transfer the content of the file from the APK out to a temporary file to keep it happy + (fd, path) = tempfile.mkstemp(suffix=name + ".desktop") + handle = open(fd, "wb") + handle.write(desktopFileContents.read()) + handle.close() + # Parse the XDG format *.desktop file, and extract the categories within it + desktopFile = xdg.DesktopEntry.DesktopEntry(path) + data['categories'] = { None: desktopFile.getCategories() } + + # Try to figure out the source repository + if arguments.source and os.path.exists(os.path.join(arguments.source, '.git')): + output = subprocess.check_output('git remote show -n origin', shell=True, cwd = arguments.source).decode('utf-8') + result = re.search(' Fetch URL: (.*)\n', output) + data['source-repo'] = result.group(1) + + # write meta data + createFastlaneFile( applicationName, "title.txt", data['name'] ) + createFastlaneFile( applicationName, "short_description.txt", data['summary'] ) + createFastlaneFile( applicationName, "full_description.txt", data['description'] ) + createYml(applicationName, data) + downloadScreenshots(applicationName, data) + createMetadataArchive(applicationName) + +# Generate metadata for the given appstream and desktop files +def processAppstreamFile(appstreamFileName, desktopFileName): + appstreamFile = open(appstreamFileName, "rb"); + desktopData = None + if desktopFileName and os.path.exists(desktopFileName): + desktopFile = open(desktopFileName, "rb"); + desktopData = desktopFile.read(); + applicationName = os.path.basename(appstreamFileName)[:-12] + processAppstreamData(applicationName, appstreamFile.read(), desktopData) + +# scan source directory for manifests/metadata we can work with +def scanSourceDir(): + files = glob.iglob(arguments.source + "/**/AndroidManifest.xml*", recursive=True) + for file in files: + # third-party libraries might contain AndroidManifests which we are not interested in + if "3rdparty" in file: + continue + + # find application id from manifest files + root = ET.parse(file) + appname = root.getroot().attrib['package'] + is_app = False + prefix = '{http://schemas.android.com/apk/res/android}' + for md in root.findall("application/activity/meta-data"): + if md.attrib[prefix + 'name'] == 'android.app.lib_name': + is_app = True + + if not appname or not is_app: + continue + + # now that we have the app id, look for matching appdata/desktop files + appdataFiles = glob.iglob(arguments.source + "/**/" + appname + ".appdata.xml", recursive=True) + appdataFile = None + for f in appdataFiles: + appdataFile = f; + break; + if not appdataFile: + continue + + desktopFiles = glob.iglob(arguments.source + "/**/" + appname + ".desktop", recursive=True) + desktopFile = None + for f in desktopFiles: + desktopFile = f; + break; + + processAppstreamFile(appdataFile, desktopFile) + + +### Script Commences + +# Parse the command line arguments we've been given +parser = argparse.ArgumentParser(description='Generate fastlane metadata for Android apps from appstream metadata') +parser.add_argument('--apk', type=str, required=False, help='APK file to extract metadata from') +parser.add_argument('--appstream', type=str, required=False, help='Appstream file to extract metadata from') +parser.add_argument('--desktop', type=str, required=False, help='Desktop file to extract additional metadata from') +parser.add_argument('--source', type=str, required=False, help='Source directory to find metadata in') +parser.add_argument('--output', type=str, required=True, help='Path to which the metadata output should be written to') +arguments = parser.parse_args() + +# ensure the output path exists +os.makedirs(arguments.output, exist_ok=True) + +# if we have an appstream file explicitly specified, let's use that one +if arguments.appstream and os.path.exists(arguments.appstream): + processAppstreamFile(arguments.appstream, arguments.desktop) + sys.exit(0) + +# else, if we have an APK, try to find the appstream file in there +# this ensures compatibility with the old metadata generation +if arguments.apk and os.path.exists(arguments.apk): + processApkFile(arguments.apk) + sys.exit(0) + +# else, look in the source dir for appstream/desktop files +# this follows roughly what get-apk-args from binary factory does +if arguments.source and os.path.exists(arguments.source): + scanSourceDir() + sys.exit(0) + +# else: missing arguments +print("Either one of --appstream, --apk or --source have to be provided!") +sys.exit(1) |