From 7abd6eb267adfc4935ccaa162ce6e6f179a849ad Mon Sep 17 00:00:00 2001 From: Olamide Isreal Date: Mon, 23 Mar 2026 15:56:09 +0100 Subject: [PATCH] Use wget/python instead of curl for MinIO download (curl not in image) --- main.nf | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/main.nf b/main.nf index b8094f0..53b88a8 100644 --- a/main.nf +++ b/main.nf @@ -56,32 +56,40 @@ process POCKETMINER { fi fi - # Method 3: Download from MinIO via curl (no auth needed for public) + # Method 3: Download from MinIO via wget if [ -z "\$PDB_FILE" ]; then S3_PATH="\$(echo '${pdb_path}' | sed 's|^s3://||')" MINIO_URL="http://datalake-hl.datalake.svc.cluster.local:9000/\$S3_PATH" echo "Downloading from MinIO: \$MINIO_URL" >> run.log - curl -sf "\$MINIO_URL" -o input.pdb 2>> run.log + wget -q "\$MINIO_URL" -O input.pdb 2>> run.log || true if [ -f input.pdb ] && [ -s input.pdb ]; then echo "Downloaded from MinIO (\$(wc -c < input.pdb) bytes)" >> run.log PDB_FILE="input.pdb" else - echo "MinIO download failed or empty" >> run.log + echo "wget download failed, trying python..." >> run.log rm -f input.pdb fi fi - # Method 4: Download from MinIO using AWS env vars - if [ -z "\$PDB_FILE" ] && [ -n "\${AWS_ACCESS_KEY_ID:-}" ]; then + # Method 4: Download from MinIO using python urllib + if [ -z "\$PDB_FILE" ]; then S3_PATH="\$(echo '${pdb_path}' | sed 's|^s3://||')" ENDPOINT="\${AWS_ENDPOINT_URL:-http://datalake-hl.datalake.svc.cluster.local:9000}" - echo "Downloading with AWS creds from: \$ENDPOINT/\$S3_PATH" >> run.log - curl -sf -u "\${AWS_ACCESS_KEY_ID}:\${AWS_SECRET_ACCESS_KEY}" "\$ENDPOINT/\$S3_PATH" -o input.pdb 2>> run.log + echo "Downloading with python from: \$ENDPOINT/\$S3_PATH" >> run.log + python -c " +import urllib.request +url = '\$ENDPOINT/\$S3_PATH' +try: + urllib.request.urlretrieve(url, 'input.pdb') + print(f'Downloaded from {url}') +except Exception as e: + print(f'Failed: {e}') +" >> run.log 2>&1 if [ -f input.pdb ] && [ -s input.pdb ]; then - echo "Downloaded with AWS creds (\$(wc -c < input.pdb) bytes)" >> run.log + echo "Downloaded via python (\$(wc -c < input.pdb) bytes)" >> run.log PDB_FILE="input.pdb" else - echo "AWS cred download failed" >> run.log + echo "Python download failed" >> run.log rm -f input.pdb fi fi