omic/digital-patients
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Debug stage_data.nf: test PVC access, sshpass install, and network connectivity

Browse Source
This commit is contained in:
Olamide Isreal
2026-04-01 10:46:28 +01:00
parent 8695fa01e9
commit 4db4b2ac06
1 changed files with 22 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

77
stage_data.nf
View File

@@ -4,76 +4,37 @@ process STAGE_DATA {
container 'harbor.cluster.omic.ai/omic/digital-patients/synthea:cudf'
memory '4 GB'
cpus 2
errorStrategy 'terminate'
script:
"""
set +e
BASE="/omic/eureka/digital-patients"
echo "=== Cleaning up old _parts directories ==="
find \$BASE -name "*_parts" -type d -exec rm -rf {} + 2>/dev/null
echo "Cleanup done"
echo "=== Step 1: Check PVC ==="
ls /omic/eureka/ 2>&1 | head -10
ls -la \$BASE/ 2>&1 || echo "Base dir does not exist"
echo "=== Creating directory structure ==="
echo "=== Step 2: Create dirs ==="
mkdir -p \$BASE/imputed \$BASE/healthy \$BASE/supporting-data/vcf \$BASE/supporting-data/ucsc-liftover \$BASE/supporting-data/genome \$BASE/output
echo "Dirs created"
ls -la \$BASE/
echo "=== Current PVC state ==="
ls -la \$BASE/ 2>/dev/null || echo "Base dir issue"
echo "=== Step 3: Install sshpass ==="
conda install -y -c conda-forge sshpass 2>&1 | tail -5
which sshpass 2>&1
sshpass -V 2>&1
echo "=== Installing sshpass ==="
conda install -y -c conda-forge sshpass 2>&1 | tail -3
which sshpass && echo "sshpass OK" || echo "sshpass NOT FOUND"
echo "=== Step 4: Test connectivity ==="
ping -c 2 nucleus.omic.ai 2>&1 || echo "ping failed"
nc -zv nucleus.omic.ai 9100 -w 5 2>&1 || echo "nc failed"
download() {
remote="\$1"
local_path="\$2"
if [ -f "\$local_path" ]; then
echo "SKIP (exists): \$local_path"
return 0
fi
echo "Downloading: \$remote -> \$local_path"
sshpass -p 'bl3rg3r5' scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -P 9100 "omic@nucleus.omic.ai:\$remote" "\$local_path" 2>&1
if [ -f "\$local_path" ]; then
ls -lh "\$local_path"
return 0
else
echo "FAILED: \$local_path"
return 1
fi
}
echo "=== Step 5: Test scp ==="
sshpass -p 'bl3rg3r5' ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p 9100 omic@nucleus.omic.ai "echo SSH_OK && ls /mnt/Avatar/digital_patient/ | head -3" 2>&1
echo "=== Downloading imputed files ==="
download "/mnt/Avatar/imputed/ukbb/imputed/F5_SCHIZO.gwas.imputed_v3.both_sexes.tsv.bgz" "\$BASE/imputed/F5_SCHIZO.gwas.imputed_v3.both_sexes.tsv.bgz"
download "/mnt/Avatar/imputed/ukbb/imputed/F5_SCHIZO.gwas.imputed_v3.female.tsv.bgz" "\$BASE/imputed/F5_SCHIZO.gwas.imputed_v3.female.tsv.bgz"
download "/mnt/Avatar/imputed/ukbb/imputed/F5_SCHIZO.gwas.imputed_v3.male.tsv.bgz" "\$BASE/imputed/F5_SCHIZO.gwas.imputed_v3.male.tsv.bgz"
echo "=== Step 6: Download one small test file ==="
sshpass -p 'bl3rg3r5' scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -P 9100 omic@nucleus.omic.ai:/mnt/Avatar/dd/synthea/supporting-data/vcf/vcf_template.vcf \$BASE/supporting-data/vcf/vcf_template.vcf 2>&1
ls -la \$BASE/supporting-data/vcf/vcf_template.vcf 2>&1
echo "=== Downloading gnomad files ==="
download "/mnt/Avatar/digital_patient/gnomad.genomes.v4.1.sites.female.txt" "\$BASE/healthy/gnomad.genomes.v4.1.sites.female.txt"
download "/mnt/Avatar/digital_patient/gnomad.genomes.v4.1.sites.male.txt" "\$BASE/healthy/gnomad.genomes.v4.1.sites.male.txt"
echo "=== Downloading supporting-data files ==="
download "/mnt/Avatar/dd/synthea/supporting-data/genome/hg38.fa" "\$BASE/supporting-data/genome/hg38.fa"
download "/mnt/Avatar/dd/synthea/supporting-data/genome/hg38.dict" "\$BASE/supporting-data/genome/hg38.dict"
download "/mnt/Avatar/dd/synthea/supporting-data/ucsc-liftover/hg19ToHg38.over.chain.gz" "\$BASE/supporting-data/ucsc-liftover/hg19ToHg38.over.chain.gz"
download "/mnt/Avatar/dd/synthea/supporting-data/vcf/vcf_template.vcf" "\$BASE/supporting-data/vcf/vcf_template.vcf"
echo "=== Downloading phenotype file ==="
download "/mnt/Avatar/imputed/ukbb/metadata/ukbb_phenotypes.csv" "\$BASE/ukbb_phenotypes_filtered.csv"
echo "=== Final verification ==="
echo "Imputed:"
ls -lh \$BASE/imputed/*.bgz 2>/dev/null || echo " NONE"
echo "Healthy:"
ls -lh \$BASE/healthy/*.txt 2>/dev/null || echo " NONE"
echo "Genome:"
ls -lh \$BASE/supporting-data/genome/* 2>/dev/null || echo " NONE"
echo "Support:"
ls -lh \$BASE/supporting-data/vcf/* \$BASE/supporting-data/ucsc-liftover/* 2>/dev/null || echo " NONE"
echo "Small:"
ls -lh \$BASE/MANE* \$BASE/regulon* \$BASE/LM22* \$BASE/ukbb* 2>/dev/null || echo " NONE"
echo "=== STAGING COMPLETE ==="
echo "=== DEBUG COMPLETE ==="
"""
}