omic/pocketminer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use wget/python instead of curl for MinIO download (curl not in image)

Browse Source
This commit is contained in:
Olamide Isreal
2026-03-23 15:56:09 +01:00
parent e74a9441a4
commit 7abd6eb267
1 changed files with 17 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
main.nf
View File

@@ -56,32 +56,40 @@ process POCKETMINER {
fi
fi
# Method 3: Download from MinIO via curl (no auth needed for public)
# Method 3: Download from MinIO via wget
if [ -z "\$PDB_FILE" ]; then
S3_PATH="\$(echo '${pdb_path}' | sed 's|^s3://||')"
MINIO_URL="http://datalake-hl.datalake.svc.cluster.local:9000/\$S3_PATH"
echo "Downloading from MinIO: \$MINIO_URL" >> run.log
curl -sf "\$MINIO_URL" -o input.pdb 2>> run.log
wget -q "\$MINIO_URL" -O input.pdb 2>> run.log || true
if [ -f input.pdb ] && [ -s input.pdb ]; then
echo "Downloaded from MinIO (\$(wc -c < input.pdb) bytes)" >> run.log
PDB_FILE="input.pdb"
else
echo "MinIO download failed or empty" >> run.log
echo "wget download failed, trying python..." >> run.log
rm -f input.pdb
fi
fi
# Method 4: Download from MinIO using AWS env vars
if [ -z "\$PDB_FILE" ] && [ -n "\${AWS_ACCESS_KEY_ID:-}" ]; then
# Method 4: Download from MinIO using python urllib
if [ -z "\$PDB_FILE" ]; then
S3_PATH="\$(echo '${pdb_path}' | sed 's|^s3://||')"
ENDPOINT="\${AWS_ENDPOINT_URL:-http://datalake-hl.datalake.svc.cluster.local:9000}"
echo "Downloading with AWS creds from: \$ENDPOINT/\$S3_PATH" >> run.log
curl -sf -u "\${AWS_ACCESS_KEY_ID}:\${AWS_SECRET_ACCESS_KEY}" "\$ENDPOINT/\$S3_PATH" -o input.pdb 2>> run.log
echo "Downloading with python from: \$ENDPOINT/\$S3_PATH" >> run.log
python -c "
import urllib.request
url = '\$ENDPOINT/\$S3_PATH'
try:
urllib.request.urlretrieve(url, 'input.pdb')
print(f'Downloaded from {url}')
except Exception as e:
print(f'Failed: {e}')
" >> run.log 2>&1
if [ -f input.pdb ] && [ -s input.pdb ]; then
echo "Downloaded with AWS creds (\$(wc -c < input.pdb) bytes)" >> run.log
echo "Downloaded via python (\$(wc -c < input.pdb) bytes)" >> run.log
PDB_FILE="input.pdb"
else
echo "AWS cred download failed" >> run.log
echo "Python download failed" >> run.log
rm -f input.pdb
fi
fi