From eb8dbac9e18dcca28fda079e945f39a56103304c Mon Sep 17 00:00:00 2001 From: Ralf Hubert Date: Mon, 30 Dec 2024 07:48:00 +0100 Subject: [PATCH 1/6] test: add extractor test --- test/black-box/extractors/input/test.7z | Bin 0 -> 127 bytes test/black-box/extractors/input/test.dat | 1 + test/black-box/extractors/input/test.dat.gz | Bin 0 -> 30 bytes test/black-box/extractors/input/test.dat.xz | Bin 0 -> 68 bytes test/black-box/extractors/input/test.tgz | Bin 0 -> 120 bytes test/black-box/extractors/input/test.zip | Bin 0 -> 167 bytes .../extractors/recipes/extract_test.yaml | 47 ++++++++++++++++++ test/black-box/extractors/recipes/nodir.yaml | 15 ++++++ test/black-box/extractors/run.sh | 17 +++++++ 9 files changed, 80 insertions(+) create mode 100644 test/black-box/extractors/input/test.7z create mode 100644 test/black-box/extractors/input/test.dat create mode 100644 test/black-box/extractors/input/test.dat.gz create mode 100644 test/black-box/extractors/input/test.dat.xz create mode 100644 test/black-box/extractors/input/test.tgz create mode 100644 test/black-box/extractors/input/test.zip create mode 100644 test/black-box/extractors/recipes/extract_test.yaml create mode 100644 test/black-box/extractors/recipes/nodir.yaml create mode 100755 test/black-box/extractors/run.sh diff --git a/test/black-box/extractors/input/test.7z b/test/black-box/extractors/input/test.7z new file mode 100644 index 0000000000000000000000000000000000000000..bf5f65592197de1b770b2dbf3c4ab8f062bf495a GIT binary patch literal 127 zcmXr7+Ou9=hJnR0Ekl@<0Rp0+^n={fmW&Jxh761>Yz&N?tPJejj0}v5ii`|Aj0_xH zjN9MeX=Y$xWt8NBY5-G$!VD!0sSL$HQjZ~pArVM3Fi0|pZ~=|F@tdhG>NcY&8zX~4 J!;(e@1^{~-6Al0X literal 0 HcmV?d00001 diff --git a/test/black-box/extractors/input/test.dat b/test/black-box/extractors/input/test.dat new file mode 100644 index 00000000..56a6051c --- /dev/null +++ b/test/black-box/extractors/input/test.dat @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/test/black-box/extractors/input/test.dat.gz b/test/black-box/extractors/input/test.dat.gz new file mode 100644 index 0000000000000000000000000000000000000000..8bedc9937baa07b08c873b1b40433328644f8557 GIT binary patch literal 30 lcmb2|=HN&%E=p%$E=ese(Mw4zVK8Q4*#7=bGa~~70|0|22$BE* literal 0 HcmV?d00001 diff --git a/test/black-box/extractors/input/test.dat.xz b/test/black-box/extractors/input/test.dat.xz new file mode 100644 index 0000000000000000000000000000000000000000..1e9bf409e512cc8b5e6c24f47918b312a0d0eca2 GIT binary patch literal 68 zcmexsUKJ6=z`*kC+7>sK1FVdSjA9Hhz|xf2%gDfB2;#pzHFZiWpS~6YP(|DXx!dyF MKGy=}fI1_i08ASYo&W#< literal 0 HcmV?d00001 diff --git a/test/black-box/extractors/input/test.tgz b/test/black-box/extractors/input/test.tgz new file mode 100644 index 0000000000000000000000000000000000000000..fa0d4cd1fab8166aaabf6a68e405c4f04c26cda4 GIT binary patch literal 120 zcmb2|=3oE==C|hzxehsquw1b9@>BSI)3<0I>()(PX5Ygd4|+LrDDAoKV;JmXzxHoI zj_q5G`#xLm89y`J|IB&GisF@9U1lDiqMW3ad7^q=)vasyrX_lK@1Ar2t=-ppt-4x> T`9QK^eGFr!H`8qf4F(1PbRRH+ literal 0 HcmV?d00001 diff --git a/test/black-box/extractors/input/test.zip b/test/black-box/extractors/input/test.zip new file mode 100644 index 0000000000000000000000000000000000000000..b5cabc1387dd3a345388894769b680bd33257369 GIT binary patch literal 167 zcmWIWW@h1H0D)De^CGvuzthYJWP>mVgA7ASYH^8PN@7W92qyz`fpJkf2$xoHGcdBe yU}j(d6NUlaj7)OOxD1g1nGH5@Nh64bFp?EwB${yn-mGj89gGZqK!!7j!vFxcf*;xd literal 0 HcmV?d00001 diff --git a/test/black-box/extractors/recipes/extract_test.yaml b/test/black-box/extractors/recipes/extract_test.yaml new file mode 100644 index 00000000..243c1255 --- /dev/null +++ b/test/black-box/extractors/recipes/extract_test.yaml @@ -0,0 +1,47 @@ +root: True + +checkoutSCM: + - scm: url + url: ${INPUT_FILES}/test.tgz + dir: tar + digestSHA256: "2fba5ef610331d7d7ac5ee7614d0c3daa78f330dc13399ddc39b2d698c63bca3" + - scm: url + url: ${INPUT_FILES}/test.dat.gz + dir: gzip + digestSHA256: "35ea2d0c6c815aed2835f41c495d9d3600e156ec561666b5c7f61112a81d6291" + if: ${IS_POSIX} + - scm: url + url: ${INPUT_FILES}/test.dat.xz + dir: xz + digestSHA256: "d81d8062ec99672a61a56a4be271d55886a63fec319dc0e00f3318a6f01b87c6" + if: ${IS_POSIX} + - scm: url + url: ${INPUT_FILES}/test.7z + dir: 7z + digestSHA256: "8bc55c46bd4d8974e99223a9f566c7f563bae2f9f8afa787e1d7df69acdf4d04" + if: ${IS_POSIX} + - scm: url + url: ${INPUT_FILES}/test.zip + dir: zip + digestSHA256: "8e2612b6ab1bacb5c0b342a81925ba6a0349ca8f66db4fc3dd22ddd53ff4f430" + - scm: url + url: ${INPUT_FILES}/test.dat + dir: plain + digestSHA256: "6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b" + +depends: + - nodir + +buildScript: | + cat > test.sha256 << EOF + 6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b test.dat + EOF + SHA256_FILE=$(pwd)/test.sha256 + for d in $(find $1/ -mindepth 1 -type d); do + pushd $d + sha256sum -c ${SHA256_FILE} + popd + done + +packageScript: | + echo "success" diff --git a/test/black-box/extractors/recipes/nodir.yaml b/test/black-box/extractors/recipes/nodir.yaml new file mode 100644 index 00000000..216e98d6 --- /dev/null +++ b/test/black-box/extractors/recipes/nodir.yaml @@ -0,0 +1,15 @@ +checkoutSCM: + - scm: url + url: ${INPUT_FILES}/test.tgz + digestSHA256: "2fba5ef610331d7d7ac5ee7614d0c3daa78f330dc13399ddc39b2d698c63bca3" + +buildScript: | + cat > test.sha256 << EOF + 6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b test.dat + EOF + SHA256_FILE=$(pwd)/test.sha256 + pushd $1 + sha256sum -c ${SHA256_FILE} + popd + +packageScript: /bin/true diff --git a/test/black-box/extractors/run.sh b/test/black-box/extractors/run.sh new file mode 100755 index 00000000..3b03c339 --- /dev/null +++ b/test/black-box/extractors/run.sh @@ -0,0 +1,17 @@ +#!/bin/bash -e + +. ../../test-lib.sh 2>/dev/null || { echo "Must run in script directory!" ; exit 1 ; } +cleanup + +INPUT=$PWD/input +test "${INPUT:0:1}" = "/" +INPUT="$(mangle_path "$INPUT")" + +IS_POSIX="false" +if is_posix ; then + IS_POSIX="true" +fi + +# Build and fetch result path +run_bob dev -DINPUT_FILES="${INPUT}" -DIS_POSIX="$IS_POSIX" extract_test + From 68850e0a93d2c53633fc51f62f9781002d09bd72 Mon Sep 17 00:00:00 2001 From: Ralf Hubert Date: Mon, 30 Dec 2024 07:49:44 +0100 Subject: [PATCH 2/6] urlscm: refactor extractors --- pym/bob/scm/url.py | 165 +++++++++++++++++++++++++++++---------------- 1 file changed, 108 insertions(+), 57 deletions(-) diff --git a/pym/bob/scm/url.py b/pym/bob/scm/url.py index 4cb4fb52..3920305b 100644 --- a/pym/bob/scm/url.py +++ b/pym/bob/scm/url.py @@ -10,6 +10,7 @@ replacePath from .scm import Scm, ScmAudit from http.client import HTTPException +from abc import abstractmethod import asyncio import concurrent.futures.process import contextlib @@ -155,6 +156,99 @@ def dumpMode(mode): isWin32 = sys.platform == "win32" + +class Extractor(): + def __init__(self, dir, file, strip): + self.dir = dir + self.file = file + self.strip = strip + + async def _extract(self, cmds, invoker): + destination = invoker.joinPath(self.dir, self.file) + canary = invoker.joinPath(self.dir, "." + self.file + ".extracted") + if isYounger(destination, canary): + for cmd in cmds: + if shutil.which(cmd[0]) is None: continue + await invoker.checkCommand(cmd, cwd=self.dir) + invoker.trace("", canary) + with open(canary, "wb") as f: + pass + os.utime(canary) + break + else: + invoker.fail("No suitable extractor found!") + + @abstractmethod + async def extract(self, invoker, destination, cwd): + return False + +# Use the Python tar/zip extraction only on Windows. They are slower and in +# case of tarfile broken in certain ways (e.g. tarfile will result in +# different file modes!). But it shouldn't make a difference on Windows. +class TarExtractor(Extractor): + def __init__(self, dir, file, strip): + super().__init__(dir, file, strip) + + async def extract(self, invoker): + cmds = [] + if isWin32 and self.strip == 0: + cmds.append(["python", "-m", "tarfile", "-e", self.file]) + + cmd = ["tar", "-x", "--no-same-owner", "--no-same-permissions", + "-f", self.file] + if self.strip > 0: + cmd.append("--strip-components={}".format(self.strip)) + cmds.append(cmd) + + await self._extract(cmds, invoker) + + +class ZipExtractor(Extractor): + def __init__(self, dir, file, strip): + super().__init__(dir, file, strip) + if strip != 0: + raise BuildError("Extractor does not support 'stripComponents'!") + + async def extract(self, invoker): + cmds = [] + if isWin32: + cmds.append(["python", "-m", "zipfile", "-e", self.file, "."]) + + cmds.append(["unzip", "-o", self.file]) + await self._extract(cmds, invoker) + + +class GZipExtractor(Extractor): + def __init__(self, dir, file, strip): + super().__init__(dir, file, strip) + if strip != 0: + raise BuildError("Extractor does not support 'stripComponents'!") + + async def extract(self, invoker): + cmds = [["gunzip", "-kf", self.file]] + await self._extract(cmds, invoker) + +class XZExtractor(Extractor): + def __init__(self, dir, file, strip): + super().__init__(dir, file, strip) + if strip != 0: + raise BuildError("Extractor does not support 'stripComponents'!") + + async def extract(self, invoker): + cmds = [["unxz", "-kf", self.file]] + await self._extract(cmds, invoker) + +class SevenZipExtractor(Extractor): + def __init__(self, dir, file, strip): + super().__init__(dir, file, strip) + if strip != 0: + raise BuildError("Extractor does not support 'stripComponents'!") + + async def extract(self, invoker): + cmds = [["7z", "x", "-y", self.file]] + await self._extract(cmds, invoker) + + class UrlScm(Scm): __DEFAULTS = { @@ -212,27 +306,12 @@ class UrlScm(Scm): (".zip", "zip"), ] - # Use the Python tar/zip extraction only on Windows. They are slower and in - # case of tarfile broken in certain ways (e.g. tarfile will result in - # different file modes!). But it shouldn't make a difference on Windows. EXTRACTORS = { - "tar" : [ - (isWin32, "python", ["-m", "tarfile", "-e", "{}"], None), - (True, "tar", ["-x", "--no-same-owner", "--no-same-permissions", "-f", "{}"], "--strip-components={}"), - ], - "gzip" : [ - (True, "gunzip", ["-kf", "{}"], None), - ], - "xz" : [ - (True, "unxz", ["-kf", "{}"], None), - ], - "7z" : [ - (True, "7z", ["x", "-y", "{}"], None), - ], - "zip" : [ - (isWin32, "python", ["-m", "zipfile", "-e", "{}", "."], None), - (True, "unzip", ["-o", "{}"], None), - ], + "tar" : TarExtractor, + "gzip" : GZipExtractor, + "xz" : XZExtractor, + "7z" : SevenZipExtractor, + "zip" : ZipExtractor, } def __init__(self, spec, overrides=[], stripUser=None, @@ -600,19 +679,9 @@ async def invoke(self, invoker): await self._put(invoker, workspaceFile, destination, url) # Run optional extractors - extractors = self.__getExtractors() - canary = invoker.joinPath(self.__dir, "." + self.__fn + ".extracted") - if extractors and isYounger(destination, canary): - for cmd in extractors: - if shutil.which(cmd[0]) is None: continue - await invoker.checkCommand(cmd, cwd=self.__dir) - invoker.trace("", canary) - with open(canary, "wb") as f: - pass - os.utime(canary) - break - else: - invoker.fail("No suitable extractor found!") + extractor = self.__getExtractor() + if extractor is not None: + await extractor.extract(invoker) def asDigestScript(self): """Return forward compatible stable string describing this url. @@ -659,38 +728,20 @@ def calcLiveBuildId(self, workspacePath): else: return None - def __getExtractors(self): - extractors = None + def __getExtractor(self): + extractor = None if self.__extract in ["yes", "auto", True]: for (ext, tool) in UrlScm.EXTENSIONS: if self.__fn.endswith(ext): - extractors = UrlScm.EXTRACTORS[tool] + extractor = UrlScm.EXTRACTORS[tool](self.__dir, self.__fn, self.__strip) break - if not extractors and self.__extract != "auto": + if extractor is None and self.__extract != "auto": raise ParseError("Don't know how to extract '"+self.__fn+"' automatically.") elif self.__extract in UrlScm.EXTRACTORS: - extractors = UrlScm.EXTRACTORS[self.__extract] + extractor = UrlScm.EXTRACTORS[self.__extract](self.__dir, self.__fn, self.__strip) elif self.__extract not in ["no", False]: raise ParseError("Invalid extract mode: " + self.__extract) - - if extractors is None: - return [] - - ret = [] - for extractor in extractors: - if not extractor[0]: continue - if self.__strip > 0: - if extractor[3] is None: - continue - strip = [extractor[3].format(self.__strip)] - else: - strip = [] - ret.append([extractor[1]] + [a.format(self.__fn) for a in extractor[2]] + strip) - - if not ret: - raise BuildError("Extractor does not support 'stripComponents'!") - - return ret + return extractor class UrlAudit(ScmAudit): From 380cb71cf68aa57fa60dba131849b13a28f631a9 Mon Sep 17 00:00:00 2001 From: Ralf Hubert Date: Tue, 14 Jan 2025 07:15:14 +0100 Subject: [PATCH 3/6] input: add urlScmSeparateDownload policy --- pym/bob/input.py | 6 ++++++ pym/bob/intermediate.py | 1 + 2 files changed, 7 insertions(+) diff --git a/pym/bob/input.py b/pym/bob/input.py index c627d3a5..aeb65162 100644 --- a/pym/bob/input.py +++ b/pym/bob/input.py @@ -3029,6 +3029,7 @@ class RecipeSet: schema.Optional('defaultFileMode') : bool, schema.Optional('substituteMetaEnv') : bool, schema.Optional('managedLayers') : bool, + schema.Optional('urlScmSeparateDownload') : bool, }, error="Invalid policy specified! Are you using an appropriate version of Bob?" ), @@ -3083,6 +3084,11 @@ class RecipeSet: InfoOnce("managedLayers policy is not set. Only unmanaged layers are supported.", help="See http://bob-build-tool.readthedocs.io/en/latest/manual/policies.html#managedlayers for more information.") ), + "urlScmSeparateDownload": ( + "0.25.1.dev27", + InfoOnce("urlScmSeparateDownload policy is not set. Extracted archives of the 'url' SCM are retained in the workspace.", + help="See http://bob-build-tool.readthedocs.io/en/latest/manual/policies.html#urlscmseparatedownload for more information.") + ) } _ignoreCmdConfig = False diff --git a/pym/bob/intermediate.py b/pym/bob/intermediate.py index 44efb259..7b8e5106 100644 --- a/pym/bob/intermediate.py +++ b/pym/bob/intermediate.py @@ -553,6 +553,7 @@ def fromRecipeSet(cls, recipeSet): 'gitCommitOnBranch' : recipeSet.getPolicy('gitCommitOnBranch'), 'fixImportScmVariant' : recipeSet.getPolicy('fixImportScmVariant'), 'defaultFileMode' : recipeSet.getPolicy('defaultFileMode'), + 'urlScmSeparateDownload' : recipeSet.getPolicy('urlScmSeparateDownload'), } self.__data['archiveSpec'] = recipeSet.archiveSpec() self.__data['envWhiteList'] = sorted(recipeSet.envWhiteList()) From 11e4a464bbc19ee234a0e1740ab0377ee03eb003 Mon Sep 17 00:00:00 2001 From: Ralf Hubert Date: Tue, 14 Jan 2025 07:15:27 +0100 Subject: [PATCH 4/6] doc: document urlScmSeparateDownload policy --- doc/manual/policies.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/doc/manual/policies.rst b/doc/manual/policies.rst index 9e07dcb5..c65705c6 100644 --- a/doc/manual/policies.rst +++ b/doc/manual/policies.rst @@ -344,6 +344,22 @@ New behavior Unmanaged layers are expected in the same directory. +urlScmSeparateDownload +~~~~~~~~~~~~~~~~~~~~~~ + +Introduced in: 1.0 + +This policy controls where bob places downloaded files of UrlScms if extraction is +used. + +Old behavior + The downloaded file could be found in the workspace next to the extracted files. + +New behavior + The downloaded file is stored next to the workspace in a separate download folder. + Only the extracted content is in the workspace. + + .. _policies-obsolete: Obsolete policies From 2dffbccccc82577e58dc18cbaf65d742df7801ef Mon Sep 17 00:00:00 2001 From: Ralf Hubert Date: Fri, 10 Jan 2025 05:39:58 +0100 Subject: [PATCH 5/6] url-scm: store downloaded files in separate directory Separate the downloaded files from the extracted files by downloading them into a `download` directory next to the workspace. Also the canary is generated there. The Gzip and XZ-Extractor always extract the files into the directory of the compressed file. Therefor the compressed files is copied into the workspace-directory first. By removing `-k`the compressed files are no longer kept. To trigger a attic move of old workspaces a version information is added to the url-scm spec. --- pym/bob/scm/__init__.py | 3 +- pym/bob/scm/url.py | 99 ++++++++++++++++------- test/black-box/extractors/config.yaml | 2 + test/black-box/extractors/run.sh | 15 ++++ test/unit/test_input_urlscm.py | 110 +++++++++++++++----------- 5 files changed, 154 insertions(+), 75 deletions(-) create mode 100644 test/black-box/extractors/config.yaml diff --git a/pym/bob/scm/__init__.py b/pym/bob/scm/__init__.py index c6bc8842..fbff4abf 100644 --- a/pym/bob/scm/__init__.py +++ b/pym/bob/scm/__init__.py @@ -77,6 +77,7 @@ def getScm(spec, overrides=[], recipeSet=None): recipeSet and recipeSet.getPolicy('scmIgnoreUser'), recipeSet.getPreMirrors() if recipeSet else [], recipeSet.getFallbackMirrors() if recipeSet else [], - recipeSet and recipeSet.getPolicy('defaultFileMode')) + recipeSet and recipeSet.getPolicy('defaultFileMode'), + recipeSet and recipeSet.getPolicy('urlScmSeparateDownload')) else: raise ParseError("Unknown SCM '{}'".format(scm)) diff --git a/pym/bob/scm/url.py b/pym/bob/scm/url.py index 3920305b..7324ac06 100644 --- a/pym/bob/scm/url.py +++ b/pym/bob/scm/url.py @@ -158,14 +158,15 @@ def dumpMode(mode): class Extractor(): - def __init__(self, dir, file, strip): + def __init__(self, dir, file, strip, separateDownload): self.dir = dir self.file = file self.strip = strip + self.separateDownload = separateDownload async def _extract(self, cmds, invoker): - destination = invoker.joinPath(self.dir, self.file) - canary = invoker.joinPath(self.dir, "." + self.file + ".extracted") + destination = self.getCompressedFilePath(invoker) + canary = destination+".extracted" if isYounger(destination, canary): for cmd in cmds: if shutil.which(cmd[0]) is None: continue @@ -178,6 +179,10 @@ async def _extract(self, cmds, invoker): else: invoker.fail("No suitable extractor found!") + def getCompressedFilePath(self, invoker): + downloadFolder = os.path.join(os.pardir, "download") if self.separateDownload else "" + return os.path.abspath(invoker.joinPath(downloadFolder, self.dir, self.file)) \ + @abstractmethod async def extract(self, invoker, destination, cwd): return False @@ -186,16 +191,17 @@ async def extract(self, invoker, destination, cwd): # case of tarfile broken in certain ways (e.g. tarfile will result in # different file modes!). But it shouldn't make a difference on Windows. class TarExtractor(Extractor): - def __init__(self, dir, file, strip): - super().__init__(dir, file, strip) + def __init__(self, dir, file, strip, separateDownload): + super().__init__(dir, file, strip, separateDownload) async def extract(self, invoker): cmds = [] + compressedFilePath = self.getCompressedFilePath(invoker) if isWin32 and self.strip == 0: - cmds.append(["python", "-m", "tarfile", "-e", self.file]) + cmds.append(["python", "-m", "tarfile", "-e", compressedFilePath]) cmd = ["tar", "-x", "--no-same-owner", "--no-same-permissions", - "-f", self.file] + "-f", compressedFilePath] if self.strip > 0: cmd.append("--strip-components={}".format(self.strip)) cmds.append(cmd) @@ -204,48 +210,66 @@ async def extract(self, invoker): class ZipExtractor(Extractor): - def __init__(self, dir, file, strip): - super().__init__(dir, file, strip) + def __init__(self, dir, file, strip, separateDownload): + super().__init__(dir, file, strip, separateDownload) if strip != 0: raise BuildError("Extractor does not support 'stripComponents'!") async def extract(self, invoker): cmds = [] + compressedFilePath = self.getCompressedFilePath(invoker) if isWin32: - cmds.append(["python", "-m", "zipfile", "-e", self.file, "."]) + cmds.append(["python", "-m", "zipfile", + "-e", compressedFilePath, "."]) - cmds.append(["unzip", "-o", self.file]) + cmds.append(["unzip", "-o", compressedFilePath]) await self._extract(cmds, invoker) class GZipExtractor(Extractor): - def __init__(self, dir, file, strip): - super().__init__(dir, file, strip) + def __init__(self, dir, file, strip, separateDownload): + super().__init__(dir, file, strip, separateDownload) if strip != 0: raise BuildError("Extractor does not support 'stripComponents'!") async def extract(self, invoker): - cmds = [["gunzip", "-kf", self.file]] - await self._extract(cmds, invoker) + # gunzip extracts the file at the location of the input file. Copy the + # downloaded file to the workspace directory prio to uncompressing it + cmd = ["gunzip"] + if self.separateDownload: + shutil.copyfile(self.getCompressedFilePath(invoker), + invoker.joinPath(self.dir, self.file)) + else: + cmd.append("-k") + cmd.extend(["-f", self.file]) + await self._extract([cmd], invoker) + class XZExtractor(Extractor): - def __init__(self, dir, file, strip): - super().__init__(dir, file, strip) + def __init__(self, dir, file, strip, separateDownload): + super().__init__(dir, file, strip, separateDownload) if strip != 0: raise BuildError("Extractor does not support 'stripComponents'!") async def extract(self, invoker): - cmds = [["unxz", "-kf", self.file]] - await self._extract(cmds, invoker) + cmd = ["unxz"] + if self.separateDownload: + shutil.copyfile(self.getCompressedFilePath(invoker), + invoker.joinPath(self.dir, self.file)) + else: + cmd.append("-k") + cmd.extend(["-f", self.file]) + await self._extract([cmd], invoker) + class SevenZipExtractor(Extractor): - def __init__(self, dir, file, strip): - super().__init__(dir, file, strip) + def __init__(self, dir, file, strip, separateDownload): + super().__init__(dir, file, strip, separateDownload) if strip != 0: raise BuildError("Extractor does not support 'stripComponents'!") async def extract(self, invoker): - cmds = [["7z", "x", "-y", self.file]] + cmds = [["7z", "x", "-y", self.getCompressedFilePath(invoker)]] await self._extract(cmds, invoker) @@ -315,7 +339,8 @@ class UrlScm(Scm): } def __init__(self, spec, overrides=[], stripUser=None, - preMirrors=[], fallbackMirrors=[], defaultFileMode=None): + preMirrors=[], fallbackMirrors=[], defaultFileMode=None, + separateDownload=False): super().__init__(spec, overrides) self.__url = spec["url"] self.__digestSha1 = spec.get("digestSHA1") @@ -354,6 +379,7 @@ def __init__(self, spec, overrides=[], stripUser=None, self.__fallbackMirrorsUrls = spec.get("fallbackMirrors") self.__fallbackMirrorsUpload = spec.get("__fallbackMirrorsUpload") self.__fileMode = spec.get("fileMode", 0o600 if defaultFileMode else None) + self.__separateDownload = spec.get("__separateDownload", separateDownload) def getProperties(self, isJenkins, pretty=False): ret = super().getProperties(isJenkins) @@ -374,6 +400,7 @@ def getProperties(self, isJenkins, pretty=False): 'fallbackMirrors' : self.__getFallbackMirrorsUrls(), '__fallbackMirrorsUpload' : self.__getFallbackMirrorsUpload(), 'fileMode' : dumpMode(self.__fileMode) if pretty else self.__fileMode, + '__separateDownload': self.__separateDownload, }) return ret @@ -596,6 +623,9 @@ async def _put(self, invoker, workspaceFile, source, url): invoker.fail("Upload not supported for URL scheme: " + url.scheme) def canSwitch(self, oldScm): + if self.__separateDownload != oldScm.__separateDownload: + return False + diff = self._diffSpec(oldScm) if "scm" in diff: return False @@ -630,7 +660,16 @@ async def switch(self, invoker, oldScm): async def invoke(self, invoker): os.makedirs(invoker.joinPath(self.__dir), exist_ok=True) workspaceFile = os.path.join(self.__dir, self.__fn) + extractor = self.__getExtractor() + destination = invoker.joinPath(self.__dir, self.__fn) + if extractor is not None and self.__separateDownload: + downloadDestination = invoker.joinPath(os.pardir, "download", self.__dir) + # os.makedirs doc: + # Note: makedirs() will become confused if the path elements to create include pardir (eg. “..” on UNIX systems). + # -> use normpath to collaps up-level reference + os.makedirs(os.path.normpath(downloadDestination), exist_ok=True) + destination = invoker.joinPath(os.pardir, "download", self.__dir, self.__fn) # Download only if necessary if not self.isDeterministic() or not os.path.isfile(destination): @@ -679,7 +718,6 @@ async def invoke(self, invoker): await self._put(invoker, workspaceFile, destination, url) # Run optional extractors - extractor = self.__getExtractor() if extractor is not None: await extractor.extract(invoker) @@ -688,7 +726,9 @@ def asDigestScript(self): The format is "digest dir extract" if a SHA checksum was specified. Otherwise it is "url dir extract". A "s#" is appended if leading paths - are stripped where # is the number of stripped elements. + are stripped where # is the number of stripped elements. Also appended + is "m" if fileMode is set. + "sep" is appendend if the archive is not stored in the workspace. """ if self.__stripUser: filt = removeUserFromUrl @@ -698,7 +738,8 @@ def asDigestScript(self): self.__digestSha1 or filt(self.__url) ) + " " + posixpath.join(self.__dir, self.__fn) + " " + str(self.__extract) + \ ( " s{}".format(self.__strip) if self.__strip > 0 else "" ) + \ - ( " m{}".format(self.__fileMode) if self.__fileMode is not None else "") + ( " m{}".format(self.__fileMode) if self.__fileMode is not None else "") + \ + ( " sep" if self.__separateDownload else "" ) def getDirectory(self): return self.__dir @@ -733,12 +774,14 @@ def __getExtractor(self): if self.__extract in ["yes", "auto", True]: for (ext, tool) in UrlScm.EXTENSIONS: if self.__fn.endswith(ext): - extractor = UrlScm.EXTRACTORS[tool](self.__dir, self.__fn, self.__strip) + extractor = UrlScm.EXTRACTORS[tool](self.__dir, self.__fn, + self.__strip, self.__separateDownload) break if extractor is None and self.__extract != "auto": raise ParseError("Don't know how to extract '"+self.__fn+"' automatically.") elif self.__extract in UrlScm.EXTRACTORS: - extractor = UrlScm.EXTRACTORS[self.__extract](self.__dir, self.__fn, self.__strip) + extractor = UrlScm.EXTRACTORS[self.__extract](self.__dir, self.__fn, + self.__strip, self.__separateDownload) elif self.__extract not in ["no", False]: raise ParseError("Invalid extract mode: " + self.__extract) return extractor diff --git a/test/black-box/extractors/config.yaml b/test/black-box/extractors/config.yaml new file mode 100644 index 00000000..70ad234a --- /dev/null +++ b/test/black-box/extractors/config.yaml @@ -0,0 +1,2 @@ +policies: + urlScmSeparateDownload: True diff --git a/test/black-box/extractors/run.sh b/test/black-box/extractors/run.sh index 3b03c339..98253b73 100755 --- a/test/black-box/extractors/run.sh +++ b/test/black-box/extractors/run.sh @@ -15,3 +15,18 @@ fi # Build and fetch result path run_bob dev -DINPUT_FILES="${INPUT}" -DIS_POSIX="$IS_POSIX" extract_test +check_files() { + expect_not_exist dev/src/extract_test/1/workspace/$1/test.${2:-$1}.extracted + expect_not_exist dev/src/extract_test/1/workspace/$1/test.${2:-$1} + expect_exist dev/src/extract_test/1/download/$1/test.${2:-$1}.extracted + expect_exist dev/src/extract_test/1/download/$1/test.${2:-$1} +} + +check_files "tar" "tgz" +check_files "zip" + +if is_posix; then + check_files "gzip" "dat.gz" + check_files "xz" "dat.xz" + check_files "7z" +fi diff --git a/test/unit/test_input_urlscm.py b/test/unit/test_input_urlscm.py index 96d1867c..9b91c606 100644 --- a/test/unit/test_input_urlscm.py +++ b/test/unit/test_input_urlscm.py @@ -43,6 +43,15 @@ class DummyStep: def getPackage(self): return DummyPackage() +class TemporaryWorkspace: + def __enter__(self): + self.tmpDir = tempfile.TemporaryDirectory() + ws = os.path.join(self.tmpDir.name, "workspace") + os.makedirs(ws) + return ws + def __exit__(self, exception_type, exception_value, exception_traceback): + self.tmpDir.cleanup() + class UrlScmExecutor: def invokeScm(self, workspace, scm, switch=False, oldScm=None): @@ -59,16 +68,16 @@ def invokeScm(self, workspace, scm, switch=False, oldScm=None): executor.shutdown() def createUrlScm(self, spec = {}, preMirrors=[], fallbackMirrors=[], - defaultFileMode=None): + defaultFileMode=None, separateDownload=True): s = { 'scm' : 'url', 'url' : self.url, 'recipe' : "foo.yaml#0", - '__source' : "Recipe foo", + '__source' : "Recipe foo" } s.update(spec) return UrlScm(s, preMirrors=preMirrors, fallbackMirrors=fallbackMirrors, - defaultFileMode=defaultFileMode) + defaultFileMode=defaultFileMode, separateDownload=separateDownload) class UrlScmTest(UrlScmExecutor): @@ -115,7 +124,7 @@ def assertMode(self, fn, mode=(0o666 if isWindows() else 0o600)): class TestLiveBuildId(UrlScmTest, TestCase): def callCalcLiveBuildId(self, scm): - with tempfile.TemporaryDirectory() as workspace: + with TemporaryWorkspace() as workspace: self.invokeScm(workspace, scm) return scm.calcLiveBuildId(workspace) @@ -224,34 +233,34 @@ class TestDigestMatch(UrlScmTest, TestCase): def testSHA1Match(self): scm = self.createUrlScm({ "digestSHA1" : self.urlSha1 }) - with tempfile.TemporaryDirectory() as workspace: + with TemporaryWorkspace() as workspace: self.invokeScm(workspace, scm) def testSHA1Mismatch(self): scm = self.createUrlScm({ "digestSHA1" : "0"*40 }) - with tempfile.TemporaryDirectory() as workspace: + with TemporaryWorkspace() as workspace: with self.assertRaises(InvocationError): self.invokeScm(workspace, scm) def testSHA256Match(self): scm = self.createUrlScm({ "digestSHA256" : self.urlSha256 }) - with tempfile.TemporaryDirectory() as workspace: + with TemporaryWorkspace() as workspace: self.invokeScm(workspace, scm) def testSHA256Mismatch(self): scm = self.createUrlScm({ "digestSHA256" : "0"*64 }) - with tempfile.TemporaryDirectory() as workspace: + with TemporaryWorkspace() as workspace: with self.assertRaises(InvocationError): self.invokeScm(workspace, scm) def testSHA512Match(self): scm = self.createUrlScm({ "digestSHA512" : self.urlSha512 }) - with tempfile.TemporaryDirectory() as workspace: + with TemporaryWorkspace() as workspace: self.invokeScm(workspace, scm) def testSHA512Mismatch(self): scm = self.createUrlScm({ "digestSHA512" : "0"*128 }) - with tempfile.TemporaryDirectory() as workspace: + with TemporaryWorkspace() as workspace: with self.assertRaises(InvocationError): self.invokeScm(workspace, scm) @@ -263,7 +272,7 @@ def testDownload(self): scm = self.createUrlScm({ "url" : "http://localhost:{}/test.txt".format(srv.port), }) - with tempfile.TemporaryDirectory() as workspace: + with TemporaryWorkspace() as workspace: self.invokeScm(workspace, scm) fn = os.path.join(workspace, "test.txt") self.assertContent(fn) @@ -275,7 +284,7 @@ def testDownloadAgain(self): scm = self.createUrlScm({ "url" : "http://localhost:{}/test.txt".format(srv.port), }) - with tempfile.TemporaryDirectory() as workspace: + with TemporaryWorkspace() as workspace: fn = os.path.join(workspace, "test.txt") self.invokeScm(workspace, scm) self.assertContent(fn) @@ -296,7 +305,7 @@ def testDownloadRetry(self): "url" : "http://localhost:{}/test.txt".format(srv.port), "retries" : 2 }) - with tempfile.TemporaryDirectory() as workspace: + with TemporaryWorkspace() as workspace: self.invokeScm(workspace, scm) self.assertContent(os.path.join(workspace, "test.txt")) @@ -307,7 +316,7 @@ def testDownloadRetryFailing(self): "url" : "http://localhost:{}/test.txt".format(srv.port), "retries" : 1 }) - with tempfile.TemporaryDirectory() as workspace: + with TemporaryWorkspace() as workspace: with self.assertRaises(InvocationError): self.invokeScm(workspace, scm) @@ -318,7 +327,7 @@ def testDownloadNotExisting(self): scm = self.createUrlScm({ "url" : "http://localhost:{}/invalid.txt".format(srv.port), }) - with tempfile.TemporaryDirectory() as workspace: + with TemporaryWorkspace() as workspace: with self.assertRaises(InvocationError): self.invokeScm(workspace, scm) @@ -328,7 +337,7 @@ def testNoResponse(self): scm = self.createUrlScm({ "url" : "http://localhost:{}/test.txt".format(srv.port), }) - with tempfile.TemporaryDirectory() as workspace: + with TemporaryWorkspace() as workspace: with self.assertRaises(InvocationError): self.invokeScm(workspace, scm) @@ -391,7 +400,7 @@ def testTarGz(self): "url" : self.tarGzFile, "digestSHA1" : self.tarGzDigestSha1, }) - with tempfile.TemporaryDirectory() as workspace: + with TemporaryWorkspace() as workspace: self.invokeScm(workspace, scm) self.assertExists(os.path.join(workspace, "src", "test.txt")) @@ -401,7 +410,7 @@ def testTarGzStripComponents(self): "digestSHA1" : self.tarGzDigestSha1, "stripComponents" : 1, }) - with tempfile.TemporaryDirectory() as workspace: + with TemporaryWorkspace() as workspace: self.invokeScm(workspace, scm) self.assertNotExists(os.path.join(workspace, "src")) self.assertExists(os.path.join(workspace, "test.txt")) @@ -412,7 +421,7 @@ def testTarGzNoExtract(self): "digestSHA1" : self.tarGzDigestSha1, "extract" : False, }) - with tempfile.TemporaryDirectory() as workspace: + with TemporaryWorkspace() as workspace: self.invokeScm(workspace, scm) self.assertExists(os.path.join(workspace, "test.tar.gz")) self.assertNotExists(os.path.join(workspace, "src")) @@ -423,7 +432,7 @@ def testGz(self): "url" : self.gzFile, "digestSHA256" : self.gzDigestSha256, }) - with tempfile.TemporaryDirectory() as workspace: + with TemporaryWorkspace() as workspace: self.invokeScm(workspace, scm) self.assertExists(os.path.join(workspace, "test.txt.gz")) self.assertExists(os.path.join(workspace, "test.txt")) @@ -434,7 +443,7 @@ def testGzStripComponentsNotSupported(self): "digestSHA256" : self.gzDigestSha256, "stripComponents" : 1, }) - with tempfile.TemporaryDirectory() as workspace: + with TemporaryWorkspace() as workspace: with self.assertRaises(InvocationError): self.invokeScm(workspace, scm) @@ -457,7 +466,7 @@ def testFileMirrorFailed(self): } ]) - with tempfile.TemporaryDirectory() as workspace: + with TemporaryWorkspace() as workspace: self.invokeScm(workspace, scm) def testHttpMirrorFailed(self): @@ -472,7 +481,7 @@ def testHttpMirrorFailed(self): "mirror" : r"http://localhost:{}/\1".format(srv.port), }] ) - with tempfile.TemporaryDirectory() as workspace: + with TemporaryWorkspace() as workspace: self.invokeScm(workspace, scm) self.assertEqual(0, srv.headRequests) @@ -491,7 +500,7 @@ def testNoMirrorsIfIndeterministic(self): "upload" : True, }] ) - with tempfile.TemporaryDirectory() as workspace: + with TemporaryWorkspace() as workspace: self.invokeScm(workspace, scm) self.assertEqual(0, srv.getRequests) @@ -517,7 +526,7 @@ def testPreMirrorFileUsed(self): } ]) - with tempfile.TemporaryDirectory() as workspace: + with TemporaryWorkspace() as workspace: self.invokeScm(workspace, scm) self.assertExists(os.path.join(workspace, "evil.txt")) @@ -535,7 +544,7 @@ def testFallbackMirrorFileUsed(self): } ]) - with tempfile.TemporaryDirectory() as workspace: + with TemporaryWorkspace() as workspace: self.invokeScm(workspace, scm) self.assertExists(os.path.join(workspace, self.fn)) @@ -553,7 +562,7 @@ def testHttpMirrorUsed(self): "mirror" : r"http://localhost:{}/\1".format(srv.port), }] ) - with tempfile.TemporaryDirectory() as workspace: + with TemporaryWorkspace() as workspace: self.invokeScm(workspace, scm) def testGracefulMirrorFallback(self): @@ -585,7 +594,7 @@ def testGracefulMirrorFallback(self): ], ) - with tempfile.TemporaryDirectory() as workspace: + with TemporaryWorkspace() as workspace: self.invokeScm(workspace, scm) self.assertEqual(1, m1.getRequests) @@ -612,7 +621,7 @@ def testHttpMirrorUpload(self): "upload" : True, }] ) - with tempfile.TemporaryDirectory() as workspace: + with TemporaryWorkspace() as workspace: self.invokeScm(workspace, scm) self.assertExists(mirrorPath) @@ -637,7 +646,7 @@ def testHttpMirrorUploadRetry(self): "upload" : True, }] ) - with tempfile.TemporaryDirectory() as workspace: + with TemporaryWorkspace() as workspace: self.invokeScm(workspace, scm) self.assertEqual(2, srv.putRequests) @@ -660,7 +669,7 @@ def testHttpMirrorNoReplaceExisting(self): "upload" : True, }] ) - with tempfile.TemporaryDirectory() as workspace: + with TemporaryWorkspace() as workspace: self.invokeScm(workspace, scm) self.assertEqual(1, srv.headRequests) @@ -699,7 +708,7 @@ def testUploadIfDownloadedFromMirror(self): }], ) - with tempfile.TemporaryDirectory() as workspace: + with TemporaryWorkspace() as workspace: self.invokeScm(workspace, scm) self.assertExists(secondMirrorPath) @@ -718,7 +727,7 @@ def testRogueMirrorFails(self): } ]) - with tempfile.TemporaryDirectory() as workspace: + with TemporaryWorkspace() as workspace: with self.assertRaises(InvocationError): self.invokeScm(workspace, scm) @@ -735,7 +744,7 @@ def testNoUploadBroken(self): } ]) - with tempfile.TemporaryDirectory() as workspace: + with TemporaryWorkspace() as workspace: with self.assertRaises(InvocationError): self.invokeScm(workspace, scm) @@ -748,21 +757,21 @@ class TestFileMode(UrlScmTest, TestCase): def testOldDefaultFileMode(self): """Test old behaviour of defaultFileMode policy""" os.chmod(self.path, 0o764) - with tempfile.TemporaryDirectory() as workspace: + with TemporaryWorkspace() as workspace: self.invokeScm(workspace, self.createUrlScm()) self.assertMode(os.path.join(workspace, self.fn), 0o764) def testNewDefaultFileMode(self): """Test new behaviour of defaultFileMode policy""" os.chmod(self.path, 0o764) - with tempfile.TemporaryDirectory() as workspace: + with TemporaryWorkspace() as workspace: self.invokeScm(workspace, self.createUrlScm(defaultFileMode=True)) self.assertMode(os.path.join(workspace, self.fn), 0o600) def testFileModeOverride(self): """Test that fileMode attribute takes precedence""" os.chmod(self.path, 0o777) - with tempfile.TemporaryDirectory() as workspace: + with TemporaryWorkspace() as workspace: scm = self.createUrlScm({ "fileMode" : 0o640 }, defaultFileMode=True) self.invokeScm(workspace, scm) @@ -770,7 +779,7 @@ def testFileModeOverride(self): def testSwitch(self): os.chmod(self.path, 0o777) - with tempfile.TemporaryDirectory() as workspace: + with TemporaryWorkspace() as workspace: oldScm = self.createUrlScm({ "fileMode" : 0o640 }, defaultFileMode=True) self.invokeScm(workspace, oldScm) @@ -835,21 +844,21 @@ def testArchive(self): for ext in ("tar", "tar.bz2", "tar.gz", "tar.xz", "zip"): with self.subTest(extension=ext): self.url = makeFileUrl(os.path.abspath("data/url-scm/foo-1.2.3." + ext)) - with tempfile.TemporaryDirectory() as workspace: + with TemporaryWorkspace() as workspace: scm = self.createUrlScm() self.invokeScm(workspace, scm) self.assertTree(workspace) def testStripComponents(self): self.url = makeFileUrl(os.path.abspath("data/url-scm/foo-1.2.3.tar")) - with tempfile.TemporaryDirectory() as workspace: + with TemporaryWorkspace() as workspace: scm = self.createUrlScm({ "stripComponents" : 1 }) self.invokeScm(workspace, scm) self.assertTree(workspace, "") def testStripComponentsUnsupported(self): self.url = makeFileUrl(os.path.abspath("data/url-scm/foo-1.2.3.zip")) - with tempfile.TemporaryDirectory() as workspace: + with TemporaryWorkspace() as workspace: scm = self.createUrlScm({ "stripComponents" : 1 }) with self.assertRaises(BuildError): self.invokeScm(workspace, scm) @@ -859,18 +868,27 @@ def testSingleFile(self): for ext in ("gz", "xz"): with self.subTest(extension=ext): self.url = makeFileUrl(os.path.abspath("data/url-scm/test.txt." + ext)) - with tempfile.TemporaryDirectory() as workspace: + with TemporaryWorkspace() as workspace: scm = self.createUrlScm() self.invokeScm(workspace, scm) - self.assertTrue(os.path.exists(os.path.join(workspace, "test.txt." + ext))) + self.assertTrue(os.path.exists(os.path.join(workspace, "..", + "download", "test.txt." + ext))) self.assertFileMd5(os.path.join(workspace, "test.txt"), "d3b07384d113edec49eaa6238ad5ff00") + def testNoSeparate(self): + self.url = makeFileUrl(os.path.abspath("data/url-scm/foo-1.2.3.zip")) + with TemporaryWorkspace() as workspace: + scm = self.createUrlScm(separateDownload=False) + self.invokeScm(workspace, scm) + self.assertTrue(os.path.exists(os.path.join(workspace, "foo-1.2.3.zip"))) + self.assertTree(workspace) + def testNoExtrace(self): self.url = makeFileUrl(os.path.abspath("data/url-scm/foo-1.2.3.tar")) for extract in ("no", False): with self.subTest(mode=extract): - with tempfile.TemporaryDirectory() as workspace: + with TemporaryWorkspace() as workspace: scm = self.createUrlScm({ "extract" : extract }) self.invokeScm(workspace, scm) self.assertTrue(os.path.exists(os.path.join(workspace, "foo-1.2.3.tar"))) @@ -878,14 +896,14 @@ def testNoExtrace(self): def testSpecificExtract(self): self.url = makeFileUrl(os.path.abspath("data/url-scm/foo-1.2.3.tar")) - with tempfile.TemporaryDirectory() as workspace: + with TemporaryWorkspace() as workspace: scm = self.createUrlScm({ "extract" : "tar" }) self.invokeScm(workspace, scm) self.assertTree(workspace) def testWrongSpecificExtract(self): self.url = makeFileUrl(os.path.abspath("data/url-scm/foo-1.2.3.tar")) - with tempfile.TemporaryDirectory() as workspace: + with TemporaryWorkspace() as workspace: scm = self.createUrlScm({ "extract" : "zip" }) with self.assertRaises(InvocationError): self.invokeScm(workspace, scm) From c619d27068db22370e9875e69f5d689ce0083df8 Mon Sep 17 00:00:00 2001 From: Ralf Hubert Date: Tue, 14 Jan 2025 12:33:13 +0100 Subject: [PATCH 6/6] urlscm: remove downloaded files on ATTIC move Delete the download directory if the scm-workspace is moved to attic. --- pym/bob/builder.py | 2 ++ pym/bob/scm/scm.py | 3 +++ pym/bob/scm/url.py | 6 ++++++ test/black-box/url-scm-switch/config.yaml | 2 +- test/black-box/url-scm-switch/run.sh | 14 ++++++++++++++ 5 files changed, 26 insertions(+), 1 deletion(-) diff --git a/pym/bob/builder.py b/pym/bob/builder.py index 8346da11..02eb9f68 100644 --- a/pym/bob/builder.py +++ b/pym/bob/builder.py @@ -1209,6 +1209,8 @@ async def _cookCheckoutStep(self, checkoutStep, depth): os.makedirs(atticPath) atticPath = os.path.join(atticPath, atticName) os.rename(scmPath, atticPath) + if scmDir in scmMap: + scmMap[scmDir].postAttic(prettySrcPath) BobState().setAtticDirectoryState(atticPath, scmSpec) atticPaths.add(scmPath, atticPath) del oldCheckoutState[scmDir] diff --git a/pym/bob/scm/scm.py b/pym/bob/scm/scm.py index bef6adbd..cf86efe5 100644 --- a/pym/bob/scm/scm.py +++ b/pym/bob/scm/scm.py @@ -361,6 +361,9 @@ def calcLiveBuildId(self, workspacePath): """Calculate live build-id from workspace.""" return None + def postAttic(self, workspace): + pass + class ScmAudit(metaclass=ABCMeta): @classmethod async def fromDir(cls, workspace, dir, extra): diff --git a/pym/bob/scm/url.py b/pym/bob/scm/url.py index 7324ac06..5836533f 100644 --- a/pym/bob/scm/url.py +++ b/pym/bob/scm/url.py @@ -786,6 +786,12 @@ def __getExtractor(self): raise ParseError("Invalid extract mode: " + self.__extract) return extractor + def postAttic(self, workspace): + if self.__separateDownload: + # os.path.exists returns False if os.pardir is in the path -> normalize it + downloadDestination = os.path.normpath(os.path.join(workspace, os.pardir, "download", self.__dir)) + if os.path.exists(downloadDestination): + shutil.rmtree(downloadDestination) class UrlAudit(ScmAudit): diff --git a/test/black-box/url-scm-switch/config.yaml b/test/black-box/url-scm-switch/config.yaml index d11aee1e..62e9a765 100644 --- a/test/black-box/url-scm-switch/config.yaml +++ b/test/black-box/url-scm-switch/config.yaml @@ -1 +1 @@ -bobMinimumVersion: "0.17.3.dev82" +bobMinimumVersion: "0.25.1.dev32" diff --git a/test/black-box/url-scm-switch/run.sh b/test/black-box/url-scm-switch/run.sh index 01acc86b..c45a740b 100755 --- a/test/black-box/url-scm-switch/run.sh +++ b/test/black-box/url-scm-switch/run.sh @@ -6,9 +6,15 @@ . ../../test-lib.sh 2>/dev/null || { echo "Must run in script directory!" ; exit 1 ; } cleanup +tempdir=$(mktemp -d) +trap 'rm -rf "${tempdir}"' EXIT + url="$(mangle_path "$(realpath file.txt)")" url2="$(mangle_path "$(realpath file2.txt)")" +tar -cvzf $tempdir/file.tgz file.txt +tar -cvzf $tempdir/file2.tgz file2.txt + # Build and fetch result path run_bob dev root -DURL="$url" path=$(run_bob query-path -DURL="$url" -f {src} root) @@ -36,3 +42,11 @@ run_bob dev root -DURL="$url2" expect_not_exist "$path/file.txt" expect_not_exist "$path/canary.txt" diff -q "$path/file2.txt" file2.txt + +cleanup +run_bob dev root -DURL="$tempdir/file.tgz" +expect_exist $path/../download/file.tgz + +run_bob dev root -DURL="$tempdir/file2.tgz" +expect_exist $path/../download/file2.tgz +expect_not_exist $path/../download/file.tgz