Skip to content

Add automatic excludelist updating at runtime and ignore-excludelist CLI option #287

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Add automatic excludelist updating at runtime and ignore-excludelist CLI option #287

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c15d2e9
Added automatic excludelist updating at runtime
Korne127 Jul 25, 2024
ec55d65
Added CLI option to ignore the excludelist
Korne127 Aug 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
114 changes: 107 additions & 7 deletions src/core/appdir.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,12 @@
#include <set>
#include <string>
#include <vector>
#include <sstream>

// library headers
#include <CImg.h>
#include <fnmatch.h>
#include <curl/curl.h>


// local headers
Expand Down Expand Up @@ -122,6 +124,13 @@ namespace linuxdeploy {
// decides whether copyright files deployment is performed
bool disableCopyrightFilesDeployment = false;

// whether the excludelist should be ignored
bool ignoreExcludelist = false;

// updated excludelist, downloaded at runtime if not ignored
bool updatedExcludelistExists = false;
std::vector<std::string> updatedExcludelist;

public:
PrivateData() : copyOperationsStorage(), stripOperations(), setElfRPathOperations(), visitedFiles(), appDirPath(), excludeLibraryPatterns() {
copyrightFilesManager = copyright::ICopyrightFilesManager::getInstance();
Expand Down Expand Up @@ -419,22 +428,35 @@ namespace linuxdeploy {
case 0:
return true;
case FNM_NOMATCH:
break;
return false;
default:
ldLog() << LD_ERROR << "fnmatch() reported error:" << fnmatchResult << std::endl;
return false;
}
}

return false;
};

if (!forceDeploy && (isInExcludelist(path.filename(), generatedExcludelist) || isInExcludelist(path.filename(), excludeLibraryPatterns))) {
ldLog() << "Skipping deployment of blacklisted library" << path << std::endl;
if (!ignoreExcludelist) {
// get most recent existing excludelist
std::vector<std::string> excludelist;
if (updatedExcludelistExists) {
excludelist = updatedExcludelist;
} else {
excludelist = generatedExcludelist;
}

if (!forceDeploy && isInExcludelist(path.filename(), excludelist)) {
// mark file as visited and return
ldLog() << "Skipping deployment of excluded core library" << path << std::endl;
visitedFiles.insert(path);
return true;
}
}

// mark file as visited
if (!forceDeploy && isInExcludelist(path.filename(), excludeLibraryPatterns)) {
// mark file as visited and return
ldLog() << "Skipping deployment of manually blacklisted library" << path << std::endl;
visitedFiles.insert(path);

return true;
}

Expand Down Expand Up @@ -692,6 +714,84 @@ namespace linuxdeploy {
return true;
}

static size_t writeCallback(void *content, size_t size, size_t nmemb, void *userdata) {
((std::string*) userdata)->append((char*) content, size * nmemb);
return size * nmemb;
}

static std::string downloadExcludelist() {
const std::string excludelistLink = "https://raw.githubusercontent.com/probonopd/AppImages/master/excludelist";
CURL* curl;
std::string readBuffer;

curl = curl_easy_init();
if (curl) {
curl_easy_setopt(curl, CURLOPT_URL, excludelistLink.c_str());
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeCallback);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &readBuffer);

CURLcode res = curl_easy_perform(curl);
curl_easy_cleanup(curl);
if (res != CURLE_OK) {
throw 1;
}

// check for valid HTTP response
long response_code = 0;
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code);
if (response_code != 200) {
throw 1;
}
} else {
throw 1;
}
return readBuffer;
}

static std::vector<std::string> processExcludelist(const std::string& excludelist) {
std::set<std::string> excludedItems;
std::istringstream stream(excludelist);
std::string line;

while (std::getline(stream, line)) {
// Remove comments
auto commentPos = line.find('#');
if (commentPos != std::string::npos) {
line = line.substr(0, commentPos);
}

// Trim whitespace
line.erase(0, line.find_first_not_of(" \t"));
line.erase(line.find_last_not_of(" \t") + 1);

if (!line.empty()) {
excludedItems.insert(line);
}
}

return std::vector<std::string>(excludedItems.begin(), excludedItems.end());
}

AppDir::ignoreExcludelist() {
d->ignoreExcludelist = true;
}

bool AppDir::updateExcludelist() {
try {
std::string excludelistString = downloadExcludelist();
std::vector<std::string> updatedExcludelist = processExcludelist(excludelistString);
if (updatedExcludelist.size() == 0) {
return false;
}
d->updatedExcludelistExists = true;
d->updatedExcludelist = updatedExcludelist;
} catch (int _) {
return false;
}
return true;
}

bool AppDir::deployLibrary(const fs::path& path, const fs::path& destination) {
return d->deployLibrary(path, false, true, destination);
}
Expand Down
14 changes: 14 additions & 0 deletions src/main.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,8 @@ int main(int argc, char** argv) {
args::ValueFlagList<std::string> inputPlugins(parser, "name", "Input plugins to run (check whether they are available with --list-plugins)", {'p', "plugin"});
args::ValueFlagList<std::string> outputPlugins(parser, "name", "Output plugins to run (check whether they are available with --list-plugins)", {'o', "output"});

args::ValueFlag<std::string> ignoreExcludelist(parser, "ignore excludelist", "Ignore the excludelist and also include core libraries into the AppDir", {"ignore-excludelist"});

try {
parser.ParseCLI(argc, argv);
} catch (args::Help&) {
Expand Down Expand Up @@ -111,6 +113,18 @@ int main(int argc, char** argv) {
appdir::AppDir appDir(appDirPath.Get());
appDir.setExcludeLibraryPatterns(excludeLibraryPatterns.Get());

if (ignoreExcludelist) {
ldLog() << std::endl << "-- Ignoring excludelist --" << std::endl;
appDir.ignoreExcludelist();
} else {
// update excludelist
ldLog() << std::endl << "-- Updating excludelist --" << std::endl;
if (!appDir.updateExcludelist()) {
ldLog() << LD_WARNING << "Failed to update excludelist, using packaged copy" << std::endl;
ldLog() << LD_WARNING << "Please run linuxdeploy with internet access or update it" << std::endl;
}
}

// allow disabling copyright files deployment via environment variable
if (getenv("DISABLE_COPYRIGHT_FILES_DEPLOYMENT") != nullptr) {
ldLog() << std::endl << LD_WARNING << "Copyright files deployment disabled" << std::endl;
Expand Down