From d2c4d7c5a0e8632cc6cdb1cc16a9d10b3058dd3b Mon Sep 17 00:00:00 2001 From: Viktor Szakats Date: Thu, 9 Nov 2023 16:07:46 +0000 Subject: [PATCH] ci: add Windows builds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add automated Windows builds and tests. Use curl-for-win with llvm + mingw64, and a minimal libcurl build with no external dependencies to build x64, ARM64 and x86 `trurl.exe`. Boost build performance by not building the curl tool [EXPERIMENTAL]. Use a customized curl-for-win build with disabled TLS to further reduce footprint and build time. Non-UNITY libcurl builds can make turl binaries about 120KB smaller, but they require 2x build times (4m vs. 2m), so opted not to use those here. Also enable tests and fix issues along the way: - libcurl with IDN support cannot be used because trurl itself lacks UNICODE support and thus fails to accept non-ASCII strings via the command-line. ``` expected: 'https://xn--rksmrgs-5wao1o.se/\n' got: '' 104: failed 'https://räksmörgås.se' -g '{puny:host}' ``` Ref: https://github.com/curl/trurl/actions/runs/6863796328/job/18664263891#step:3:4406 - add `test.py` support for a runner like `wine`. Via `--runner=` option. This disables `valgrind` tests. - add `test.py` to override the default `trurl` binary to test. Via `--trurl=` option. - skip `stderr` tests when using a runner. (`wine` does trash `stderr` output) - fix to enable `punycode2idn` only when libcurl has IDN support. - delete line-ending spaces from `test.json`. - add `--keep-port` to 6 tests to avoid relying on libcurl builds with specific protocols enabled, such as HTTPS or FTP. - add a new test with default-port using http/80. - update 4 tests to use http/imap instead of https/imaps to make them work with no-TLS libcurl. - build libcurl with IMAP to make 'options' URL field extraction work in tests. Fixes #109 Closes #249 --- .github/workflows/curl-for-win.yml | 56 +++++++++++++++++++++ test.py | 79 +++++++++++++++++++----------- tests.json | 54 +++++++++++++------- trurl.c | 19 ++++--- 4 files changed, 155 insertions(+), 53 deletions(-) create mode 100644 .github/workflows/curl-for-win.yml diff --git a/.github/workflows/curl-for-win.yml b/.github/workflows/curl-for-win.yml new file mode 100644 index 00000000..0e703de8 --- /dev/null +++ b/.github/workflows/curl-for-win.yml @@ -0,0 +1,56 @@ +# Copyright (C) Viktor Szakats. See LICENSE.md +# SPDX-License-Identifier: curl +--- +name: curl-for-win + +on: + push: + branches: ["master"] + pull_request: + branches: ["master"] + +permissions: {} + +env: + CW_GET: 'curl' + CW_MAP: '0' + CW_JOBS: '3' + CW_PKG_NODELETE: '1' + CW_PKG_FLATTEN: '1' + DOCKER_CONTENT_TRUST: '1' + +jobs: + win-llvm: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + path: 'trurl' + fetch-depth: 8 + - name: 'build' + env: + CW_LLVM_MINGW_DL: '1' + CW_LLVM_MINGW_ONLY: '0' + CW_TURL_TEST: '1' + run: | + git clone --depth 1 https://github.com/curl/curl-for-win + mv curl-for-win/* . + export CW_CONFIG='-dev-zero-imap-osnotls-osnoidn-nocurltool-win' + export CW_REVISION='${{ github.sha }}' + . ./_versions.sh + docker trust inspect --pretty "${DOCKER_IMAGE}" + time docker pull "${DOCKER_IMAGE}" + docker images --digests + time docker run --volume "$(pwd):$(pwd)" --workdir "$(pwd)" \ + --env-file <(env | grep -a -E \ + '^(CW_|GITHUB_)') \ + "${DOCKER_IMAGE}" \ + sh -c ./_ci-linux-debian.sh + + - name: 'list dependencies' + run: cat urls.txt + - uses: actions/upload-artifact@v3 + with: + name: 'trurl-windows' + retention-days: 5 + path: curl-*-*-*/trurl* diff --git a/test.py b/test.py index 60fc81d1..b05b7772 100644 --- a/test.py +++ b/test.py @@ -59,8 +59,9 @@ def testComponent(value, exp): class TestCase: - def __init__(self, testIndex, baseCmd, **testCase): + def __init__(self, testIndex, runnerCmd, baseCmd, **testCase): self.testIndex = testIndex + self.runnerCmd = runnerCmd self.baseCmd = baseCmd self.arguments = testCase["input"]["arguments"] self.expected = testCase["expected"] @@ -74,7 +75,10 @@ def runCommand(self, cmdfilter: Optional[str], runWithValgrind: bool): cmd = [self.baseCmd] args = self.arguments - if runWithValgrind: + if self.runnerCmd != "": + cmd = [self.runnerCmd] + args = [self.baseCmd] + self.arguments + elif runWithValgrind: cmd = [VALGRINDTEST] args = VALGRINDARGS + [self.baseCmd] + self.arguments @@ -96,6 +100,11 @@ def runCommand(self, cmdfilter: Optional[str], runWithValgrind: bool): # assume stderr is always going to be string stderr = output.stderr + # runners (e.g. wine) spill their own output into stderr, + # ignore stderr tests when using a runner. + if self.runnerCmd != "" and "stderr" in self.expected: + stderr = self.expected["stderr"] + self.commandOutput = CommandOutput(stdout, output.returncode, stderr) return True @@ -161,41 +170,53 @@ def main(argc, argv): # the .exe on the end is necessary when using absolute paths if sys.platform == "win32" or sys.platform == "cygwin": baseCmd += ".exe" + + with open(path.join(baseDir, TESTFILE), "r") as file: + allTests = json.load(file) + testIndexesToRun = [] + + # if argv[1] exists and starts with int + cmdfilter = "" + testIndexesToRun = list(range(len(allTests))) + runWithValgrind = False + verboseDetail = False + runnerCmd = "" + + if argc > 1: + for arg in argv[1:]: + if arg[0].isnumeric(): + # run only test cases separated by "," + testIndexesToRun = [] + + for caseIndex in arg.split(","): + testIndexesToRun.append(int(caseIndex)) + elif arg == "--with-valgrind": + runWithValgrind = True + elif arg == "--verbose": + verboseDetail = True + elif arg.startswith("--trurl="): + baseCmd = arg[len("--trurl="):] + elif arg.startswith("--runner="): + runnerCmd = arg[len("--runner="):] + else: + cmdfilter = argv[1] + # check if the trurl executable exists if path.isfile(baseCmd): # get the version info for the feature list + args = ["--version"] + if runnerCmd != "": + cmd = [runnerCmd] + args = [baseCmd] + args + else: + cmd = [baseCmd] output = run( - [baseCmd, "--version"], + cmd + args, stdout=PIPE, stderr=PIPE, encoding="utf-8" ) features = output.stdout.split('\n')[1].split()[1:] - with open(path.join(baseDir, TESTFILE), "r") as file: - allTests = json.load(file) - testIndexesToRun = [] - - # if argv[1] exists and starts with int - cmdfilter = "" - testIndexesToRun = list(range(len(allTests))) - runWithValgrind = False - verboseDetail = False - - if argc > 1: - for arg in argv[1:]: - if arg[0].isnumeric(): - # run only test cases separated by "," - testIndexesToRun = [] - - for caseIndex in arg.split(","): - testIndexesToRun.append(int(caseIndex)) - elif arg == "--with-valgrind": - runWithValgrind = True - elif arg == "--verbose": - verboseDetail = True - else: - cmdfilter = argv[1] - numTestsFailed = 0 numTestsPassed = 0 numTestsSkipped = 0 @@ -208,7 +229,7 @@ def main(argc, argv): numTestsSkipped += 1 continue - test = TestCase(testIndex + 1, baseCmd, **allTests[testIndex]) + test = TestCase(testIndex + 1, runnerCmd, baseCmd, **allTests[testIndex]) if test.runCommand(cmdfilter, runWithValgrind): if test.test(): # passed diff --git a/tests.json b/tests.json index f6226249..78c084ba 100644 --- a/tests.json +++ b/tests.json @@ -260,13 +260,28 @@ { "input": { "arguments": [ + "http://curl.se:22/", + "-s", + "port=80" + ] + }, + "expected": { + "stdout": "http://curl.se/\n", + "stderr": "", + "returncode": 0 + } + }, + { + "input": { + "arguments": [ + "--keep-port", "https://curl.se:22/", "-s", "port=443" ] }, "expected": { - "stdout": "https://curl.se/\n", + "stdout": "https://curl.se:443/\n", "stderr": "", "returncode": 0 } @@ -274,6 +289,7 @@ { "input": { "arguments": [ + "--keep-port", "https://curl.se:22/", "-s", "port=443", @@ -282,7 +298,7 @@ ] }, "expected": { - "stdout": "https://curl.se/\n", + "stdout": "https://curl.se:443/\n", "stderr": "", "returncode": 0 } @@ -307,13 +323,13 @@ "arguments": [ "--default-port", "--url", - "https://curl.se/we/are.html", + "http://curl.se/we/are.html", "--get", "{port}" ] }, "expected": { - "stdout": "443\n", + "stdout": "80\n", "stderr": "", "returncode": 0 } @@ -489,13 +505,13 @@ "input": { "arguments": [ "--url", - "https://curl.se/we/are.html", + "http://curl.se/we/are.html", "-g", "{default:port}" ] }, "expected": { - "stdout": "443\n", + "stdout": "80\n", "stderr": "", "returncode": 0 } @@ -846,11 +862,12 @@ { "input": { "arguments": [ + "--keep-port", "https://hello:443/foo" ] }, "expected": { - "stdout": "https://hello/foo\n", + "stdout": "https://hello:443/foo\n", "stderr": "", "returncode": 0 } @@ -858,11 +875,12 @@ { "input": { "arguments": [ + "--keep-port", "ftp://hello:21/foo" ] }, "expected": { - "stdout": "ftp://hello/foo\n", + "stdout": "ftp://hello:21/foo\n", "stderr": "", "returncode": 0 } @@ -884,13 +902,14 @@ { "input": { "arguments": [ + "--keep-port", "ftp://hello:443/foo", "-s", "scheme=https" ] }, "expected": { - "stdout": "https://hello/foo\n", + "stdout": "https://hello:443/foo\n", "stderr": "", "returncode": 0 } @@ -1191,6 +1210,7 @@ { "input": { "arguments": [ + "--keep-port", "https://curl.se", "--iterate", "port=80 81 443" @@ -1199,7 +1219,7 @@ "expected": { "stderr": "", "returncode": 0, - "stdout": "https://curl.se:80/\nhttps://curl.se:81/\nhttps://curl.se/\n" + "stdout": "https://curl.se:80/\nhttps://curl.se:81/\nhttps://curl.se:443/\n" } }, { @@ -1656,7 +1676,7 @@ { "input": { "arguments": [ - "imaps://user:password;crazy@[ff00::1234%hello]:1234/path?a=b&c=d#fragment", + "imap://user:password;crazy@[ff00::1234%hello]:1234/path?a=b&c=d#fragment", "--json" ] }, @@ -1664,9 +1684,9 @@ "returncode": 0, "stdout": [ { - "url": "imaps://user:password;crazy@[ff00::1234%25hello]:1234/path?a=b&c=d#fragment", + "url": "imap://user:password;crazy@[ff00::1234%25hello]:1234/path?a=b&c=d#fragment", "parts": { - "scheme": "imaps", + "scheme": "imap", "user": "user", "password": "password", "options": "crazy", @@ -2014,7 +2034,7 @@ "returncode": 0, "stderr": "" } - }, + }, { "input" : { "arguments": [ @@ -2040,7 +2060,7 @@ } ] } - ] + ] } }, { @@ -2068,7 +2088,7 @@ } ] } - ] + ] } }, { @@ -2096,7 +2116,7 @@ } ] } - ] + ] } }, { diff --git a/trurl.c b/trurl.c index 0d9d87fb..9a9b56d0 100644 --- a/trurl.c +++ b/trurl.c @@ -202,8 +202,8 @@ static void show_version(void) PROGNAME, TRURL_VERSION_TXT, data->version, LIBCURL_VERSION); /* puny code isn't guaranteed based on the version, so it must be polled * from libcurl */ +#if defined(SUPPORTS_PUNYCODE) || defined(SUPPORTS_PUNY2IDN) bool supports_puny = false; -#ifdef SUPPORTS_PUNYCODE const char *const *feature_name = data->feature_names; while(*feature_name && !supports_puny) { supports_puny = !strncmp(*feature_name, "IDN", 3); @@ -211,21 +211,26 @@ static void show_version(void) } #endif - fprintf(stdout, "features: %s", supports_puny?"punycode ":""); + fprintf(stdout, "features:"); +#ifdef SUPPORTS_PUNY2IDN + if(supports_puny) + fprintf(stdout, " punycode"); +#endif #ifdef SUPPORTS_ALLOW_SPACE - fprintf(stdout, "white-space "); + fprintf(stdout, " white-space"); #endif #ifdef SUPPORTS_ZONEID - fprintf(stdout, "zone-id "); + fprintf(stdout, " zone-id"); #endif #ifdef SUPPORTS_URL_STRERROR - fprintf(stdout, "url-strerror "); + fprintf(stdout, " url-strerror"); #endif #ifdef SUPPORTS_NORM_IPV4 - fprintf(stdout, "normalize-ipv4 "); + fprintf(stdout, " normalize-ipv4"); #endif #ifdef SUPPORTS_PUNY2IDN - fprintf(stdout, "punycode2idn"); + if(supports_puny) + fprintf(stdout, " punycode2idn"); #endif fprintf(stdout, "\n");