Skip to content

Commit

Permalink
Check proc file in pget wrapper
Browse files Browse the repository at this point in the history
  • Loading branch information
nevillelyh committed Oct 2, 2024
1 parent 042f05a commit 6848e10
Show file tree
Hide file tree
Showing 2 changed files with 6 additions and 5 deletions.
4 changes: 2 additions & 2 deletions src/cuda.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ def install_cuda(args: argparse.Namespace, version: str) -> str:
logger.info(f'Downloading CUDA {version}...')
cuda = cudas[version]
file = os.path.join(args.cache, cuda.filename)
cmd = ['pget', '--force', cuda.url, file]
cmd = ['/usr/local/bin/pget', '--force', cuda.url, file]
subprocess.run(cmd, check=True)

logger.info(f'Installing CUDA {version}...')
Expand Down Expand Up @@ -97,7 +97,7 @@ def install_cudnn(args: argparse.Namespace, version: str, cuda_major: str) -> st
logger.info(f'Downloading CuDNN {key}...')
cudnn = cudnns[key]
file = os.path.join(args.cache, cudnn.filename)
cmd = ['pget', '--force', cudnn.url, file]
cmd = ['/usr/local/bin/pget', '--force', cudnn.url, file]
subprocess.run(cmd, check=True)

logger.info(f'Installing CuDNN {key}...')
Expand Down
7 changes: 4 additions & 3 deletions src/pget
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
set -euo pipefail

PGET_BIN="/usr/local/bin/pget-bin"
PROC_FILE="/srv/r8/fuse-rpc/proc/pget"

url=''
dst=''
Expand All @@ -20,8 +21,8 @@ for arg in "$@"; do
fi
done

if [ -z "$url" ] || [ -z "$dst" ]; then
# Fail to parse arguments, use vanilla PGET
if [ ! -f "$PROC_FILE" ] || [ -z "$url" ] || [ -z "$dst" ]; then
# No proc file or fail to parse arguments, use vanilla PGET
exec "$PGET_BIN" "$@"
else
# Finger print object name with URL, size, ETag, and last modified
Expand All @@ -41,7 +42,7 @@ else
size="$(echo "$fingerprint" | grep -oP '(?<=^content-length: )[0-9]+')"

# Request file from FUSE
echo "{\"name\":\"$name\",\"size\":$size,\"url\":\"$url\"}" > /srv/r8/fuse-rpc/proc/pget
echo "{\"name\":\"$name\",\"size\":$size,\"url\":\"$url\"}" > "$PROC_FILE"

# Symlink file to destination
mkdir -p "$(dirname "$dst")"
Expand Down

0 comments on commit 6848e10

Please sign in to comment.