From c15d2e9735ef2fbe2a25f9859ae88e8ee397c64e Mon Sep 17 00:00:00 2001 From: Kornelius Rohrschneider Date: Thu, 25 Jul 2024 02:10:23 +0200 Subject: [PATCH 1/2] Added automatic excludelist updating at runtime An automatic excludelist updating mechanism has been added at runtime. To do this, each time the linuxdeploy command is invoked, it now tries to download the current version of the excludelist, parses it and saves it as a vector. If any of these steps fail, a warning is emitted. For each library that should be tested against the excludelist, the newly downloaded excludelist is now used if it has been saved correctly, and the pre-packaged excludelist otherwise. This feature has been added as previously, it had required a new linuxdeploy version to be shipped and downloaded / used by everyone in order for excludelist changes to take effect. This also meant that updates had to "trickle down" (e.g. when someone uses a framework like tauri that itself uses linuxdeploy) and it created a single point of failure (a possible missing new linuxdeploy version). --- src/core/appdir.cpp | 90 ++++++++++++++++++++++++++++++++++++++++++++- src/main.cpp | 7 ++++ 2 files changed, 96 insertions(+), 1 deletion(-) diff --git a/src/core/appdir.cpp b/src/core/appdir.cpp index 4a696a5..0504a8d 100644 --- a/src/core/appdir.cpp +++ b/src/core/appdir.cpp @@ -5,10 +5,12 @@ #include #include #include +#include // library headers #include #include +#include // local headers @@ -122,6 +124,10 @@ namespace linuxdeploy { // decides whether copyright files deployment is performed bool disableCopyrightFilesDeployment = false; + // updated excludelist, downloaded at runtime + bool updatedExcludelistExists = false; + std::vector updatedExcludelist; + public: PrivateData() : copyOperationsStorage(), stripOperations(), setElfRPathOperations(), visitedFiles(), appDirPath(), excludeLibraryPatterns() { copyrightFilesManager = copyright::ICopyrightFilesManager::getInstance(); @@ -429,7 +435,15 @@ namespace linuxdeploy { return false; }; - if (!forceDeploy && (isInExcludelist(path.filename(), generatedExcludelist) || isInExcludelist(path.filename(), excludeLibraryPatterns))) { + // get most recent existing excludelist + std::vector excludelist; + if (updatedExcludelistExists) { + excludelist = updatedExcludelist; + } else { + excludelist = generatedExcludelist; + } + + if (!forceDeploy && (isInExcludelist(path.filename(), excludelist) || isInExcludelist(path.filename(), excludeLibraryPatterns))) { ldLog() << "Skipping deployment of blacklisted library" << path << std::endl; // mark file as visited @@ -692,6 +706,80 @@ namespace linuxdeploy { return true; } + static size_t writeCallback(void *content, size_t size, size_t nmemb, void *userdata) { + ((std::string*) userdata)->append((char*) content, size * nmemb); + return size * nmemb; + } + + static std::string downloadExcludelist() { + const std::string excludelistLink = "https://raw.githubusercontent.com/probonopd/AppImages/master/excludelist"; + CURL* curl; + std::string readBuffer; + + curl = curl_easy_init(); + if (curl) { + curl_easy_setopt(curl, CURLOPT_URL, excludelistLink.c_str()); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeCallback); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &readBuffer); + + CURLcode res = curl_easy_perform(curl); + curl_easy_cleanup(curl); + if (res != CURLE_OK) { + throw 1; + } + + // check for valid HTTP response + long response_code = 0; + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code); + if (response_code != 200) { + throw 1; + } + } else { + throw 1; + } + return readBuffer; + } + + static std::vector processExcludelist(const std::string& excludelist) { + std::set excludedItems; + std::istringstream stream(excludelist); + std::string line; + + while (std::getline(stream, line)) { + // Remove comments + auto commentPos = line.find('#'); + if (commentPos != std::string::npos) { + line = line.substr(0, commentPos); + } + + // Trim whitespace + line.erase(0, line.find_first_not_of(" \t")); + line.erase(line.find_last_not_of(" \t") + 1); + + if (!line.empty()) { + excludedItems.insert(line); + } + } + + return std::vector(excludedItems.begin(), excludedItems.end()); + } + + bool AppDir::updateExcludelist() { + try { + std::string excludelistString = downloadExcludelist(); + std::vector updatedExcludelist = processExcludelist(excludelistString); + if (updatedExcludelist.size() == 0) { + return false; + } + d->updatedExcludelistExists = true; + d->updatedExcludelist = updatedExcludelist; + } catch (int _) { + return false; + } + return true; + } + bool AppDir::deployLibrary(const fs::path& path, const fs::path& destination) { return d->deployLibrary(path, false, true, destination); } diff --git a/src/main.cpp b/src/main.cpp index e5a654c..bbad656 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -111,6 +111,13 @@ int main(int argc, char** argv) { appdir::AppDir appDir(appDirPath.Get()); appDir.setExcludeLibraryPatterns(excludeLibraryPatterns.Get()); + // update excludelist + ldLog() << std::endl << "-- Updating excludelist --" << std::endl; + if (!appDir.updateExcludelist()) { + ldLog() << LD_WARNING << "Failed to update excludelist, using packaged copy" << std::endl; + ldLog() << LD_WARNING << "Please run linuxdeploy with internet access or update it" << std::endl; + } + // allow disabling copyright files deployment via environment variable if (getenv("DISABLE_COPYRIGHT_FILES_DEPLOYMENT") != nullptr) { ldLog() << std::endl << LD_WARNING << "Copyright files deployment disabled" << std::endl; From ec55d65d20cfb698779526c38fc1d3008b88b83d Mon Sep 17 00:00:00 2001 From: Kornelius Rohrschneider Date: Sat, 24 Aug 2024 20:15:39 +0200 Subject: [PATCH 2/2] Added CLI option to ignore the excludelist A new CLI option (--ignore-excludelist) has been added to linuxdeploy. When using it, the excludelist of the core libraries is ignored and the core libraries that are depended upon are included in the AppDir. The excludelist updating will only take place if this option is not set (as it's not necessary otherwise). Additionally, the code quality has been improved. --- src/core/appdir.cpp | 42 +++++++++++++++++++++++++++--------------- src/main.cpp | 19 +++++++++++++------ 2 files changed, 40 insertions(+), 21 deletions(-) diff --git a/src/core/appdir.cpp b/src/core/appdir.cpp index 0504a8d..153d618 100644 --- a/src/core/appdir.cpp +++ b/src/core/appdir.cpp @@ -124,7 +124,10 @@ namespace linuxdeploy { // decides whether copyright files deployment is performed bool disableCopyrightFilesDeployment = false; - // updated excludelist, downloaded at runtime + // whether the excludelist should be ignored + bool ignoreExcludelist = false; + + // updated excludelist, downloaded at runtime if not ignored bool updatedExcludelistExists = false; std::vector updatedExcludelist; @@ -425,30 +428,35 @@ namespace linuxdeploy { case 0: return true; case FNM_NOMATCH: - break; + return false; default: ldLog() << LD_ERROR << "fnmatch() reported error:" << fnmatchResult << std::endl; return false; } } - - return false; }; - // get most recent existing excludelist - std::vector excludelist; - if (updatedExcludelistExists) { - excludelist = updatedExcludelist; - } else { - excludelist = generatedExcludelist; + if (!ignoreExcludelist) { + // get most recent existing excludelist + std::vector excludelist; + if (updatedExcludelistExists) { + excludelist = updatedExcludelist; + } else { + excludelist = generatedExcludelist; + } + + if (!forceDeploy && isInExcludelist(path.filename(), excludelist)) { + // mark file as visited and return + ldLog() << "Skipping deployment of excluded core library" << path << std::endl; + visitedFiles.insert(path); + return true; + } } - if (!forceDeploy && (isInExcludelist(path.filename(), excludelist) || isInExcludelist(path.filename(), excludeLibraryPatterns))) { - ldLog() << "Skipping deployment of blacklisted library" << path << std::endl; - - // mark file as visited + if (!forceDeploy && isInExcludelist(path.filename(), excludeLibraryPatterns)) { + // mark file as visited and return + ldLog() << "Skipping deployment of manually blacklisted library" << path << std::endl; visitedFiles.insert(path); - return true; } @@ -765,6 +773,10 @@ namespace linuxdeploy { return std::vector(excludedItems.begin(), excludedItems.end()); } + AppDir::ignoreExcludelist() { + d->ignoreExcludelist = true; + } + bool AppDir::updateExcludelist() { try { std::string excludelistString = downloadExcludelist(); diff --git a/src/main.cpp b/src/main.cpp index bbad656..6533266 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -49,6 +49,8 @@ int main(int argc, char** argv) { args::ValueFlagList inputPlugins(parser, "name", "Input plugins to run (check whether they are available with --list-plugins)", {'p', "plugin"}); args::ValueFlagList outputPlugins(parser, "name", "Output plugins to run (check whether they are available with --list-plugins)", {'o', "output"}); + args::ValueFlag ignoreExcludelist(parser, "ignore excludelist", "Ignore the excludelist and also include core libraries into the AppDir", {"ignore-excludelist"}); + try { parser.ParseCLI(argc, argv); } catch (args::Help&) { @@ -111,12 +113,17 @@ int main(int argc, char** argv) { appdir::AppDir appDir(appDirPath.Get()); appDir.setExcludeLibraryPatterns(excludeLibraryPatterns.Get()); - // update excludelist - ldLog() << std::endl << "-- Updating excludelist --" << std::endl; - if (!appDir.updateExcludelist()) { - ldLog() << LD_WARNING << "Failed to update excludelist, using packaged copy" << std::endl; - ldLog() << LD_WARNING << "Please run linuxdeploy with internet access or update it" << std::endl; - } + if (ignoreExcludelist) { + ldLog() << std::endl << "-- Ignoring excludelist --" << std::endl; + appDir.ignoreExcludelist(); + } else { + // update excludelist + ldLog() << std::endl << "-- Updating excludelist --" << std::endl; + if (!appDir.updateExcludelist()) { + ldLog() << LD_WARNING << "Failed to update excludelist, using packaged copy" << std::endl; + ldLog() << LD_WARNING << "Please run linuxdeploy with internet access or update it" << std::endl; + } + } // allow disabling copyright files deployment via environment variable if (getenv("DISABLE_COPYRIGHT_FILES_DEPLOYMENT") != nullptr) {