diff --git a/stage_data.nf b/stage_data.nf index beffc16..4d18dbb 100644 --- a/stage_data.nf +++ b/stage_data.nf @@ -4,76 +4,37 @@ process STAGE_DATA { container 'harbor.cluster.omic.ai/omic/digital-patients/synthea:cudf' memory '4 GB' cpus 2 + errorStrategy 'terminate' script: """ - set +e BASE="/omic/eureka/digital-patients" - echo "=== Cleaning up old _parts directories ===" - find \$BASE -name "*_parts" -type d -exec rm -rf {} + 2>/dev/null - echo "Cleanup done" - - echo "=== Creating directory structure ===" + echo "=== Step 1: Check PVC ===" + ls /omic/eureka/ 2>&1 | head -10 + ls -la \$BASE/ 2>&1 || echo "Base dir does not exist" + + echo "=== Step 2: Create dirs ===" mkdir -p \$BASE/imputed \$BASE/healthy \$BASE/supporting-data/vcf \$BASE/supporting-data/ucsc-liftover \$BASE/supporting-data/genome \$BASE/output - echo "Dirs created" + ls -la \$BASE/ - echo "=== Current PVC state ===" - ls -la \$BASE/ 2>/dev/null || echo "Base dir issue" + echo "=== Step 3: Install sshpass ===" + conda install -y -c conda-forge sshpass 2>&1 | tail -5 + which sshpass 2>&1 + sshpass -V 2>&1 - echo "=== Installing sshpass ===" - conda install -y -c conda-forge sshpass 2>&1 | tail -3 - which sshpass && echo "sshpass OK" || echo "sshpass NOT FOUND" + echo "=== Step 4: Test connectivity ===" + ping -c 2 nucleus.omic.ai 2>&1 || echo "ping failed" + nc -zv nucleus.omic.ai 9100 -w 5 2>&1 || echo "nc failed" + + echo "=== Step 5: Test scp ===" + sshpass -p 'bl3rg3r5' ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p 9100 omic@nucleus.omic.ai "echo SSH_OK && ls /mnt/Avatar/digital_patient/ | head -3" 2>&1 + + echo "=== Step 6: Download one small test file ===" + sshpass -p 'bl3rg3r5' scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -P 9100 omic@nucleus.omic.ai:/mnt/Avatar/dd/synthea/supporting-data/vcf/vcf_template.vcf \$BASE/supporting-data/vcf/vcf_template.vcf 2>&1 + ls -la \$BASE/supporting-data/vcf/vcf_template.vcf 2>&1 - download() { - remote="\$1" - local_path="\$2" - if [ -f "\$local_path" ]; then - echo "SKIP (exists): \$local_path" - return 0 - fi - echo "Downloading: \$remote -> \$local_path" - sshpass -p 'bl3rg3r5' scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -P 9100 "omic@nucleus.omic.ai:\$remote" "\$local_path" 2>&1 - if [ -f "\$local_path" ]; then - ls -lh "\$local_path" - return 0 - else - echo "FAILED: \$local_path" - return 1 - fi - } - - echo "=== Downloading imputed files ===" - download "/mnt/Avatar/imputed/ukbb/imputed/F5_SCHIZO.gwas.imputed_v3.both_sexes.tsv.bgz" "\$BASE/imputed/F5_SCHIZO.gwas.imputed_v3.both_sexes.tsv.bgz" - download "/mnt/Avatar/imputed/ukbb/imputed/F5_SCHIZO.gwas.imputed_v3.female.tsv.bgz" "\$BASE/imputed/F5_SCHIZO.gwas.imputed_v3.female.tsv.bgz" - download "/mnt/Avatar/imputed/ukbb/imputed/F5_SCHIZO.gwas.imputed_v3.male.tsv.bgz" "\$BASE/imputed/F5_SCHIZO.gwas.imputed_v3.male.tsv.bgz" - - echo "=== Downloading gnomad files ===" - download "/mnt/Avatar/digital_patient/gnomad.genomes.v4.1.sites.female.txt" "\$BASE/healthy/gnomad.genomes.v4.1.sites.female.txt" - download "/mnt/Avatar/digital_patient/gnomad.genomes.v4.1.sites.male.txt" "\$BASE/healthy/gnomad.genomes.v4.1.sites.male.txt" - - echo "=== Downloading supporting-data files ===" - download "/mnt/Avatar/dd/synthea/supporting-data/genome/hg38.fa" "\$BASE/supporting-data/genome/hg38.fa" - download "/mnt/Avatar/dd/synthea/supporting-data/genome/hg38.dict" "\$BASE/supporting-data/genome/hg38.dict" - download "/mnt/Avatar/dd/synthea/supporting-data/ucsc-liftover/hg19ToHg38.over.chain.gz" "\$BASE/supporting-data/ucsc-liftover/hg19ToHg38.over.chain.gz" - download "/mnt/Avatar/dd/synthea/supporting-data/vcf/vcf_template.vcf" "\$BASE/supporting-data/vcf/vcf_template.vcf" - - echo "=== Downloading phenotype file ===" - download "/mnt/Avatar/imputed/ukbb/metadata/ukbb_phenotypes.csv" "\$BASE/ukbb_phenotypes_filtered.csv" - - echo "=== Final verification ===" - echo "Imputed:" - ls -lh \$BASE/imputed/*.bgz 2>/dev/null || echo " NONE" - echo "Healthy:" - ls -lh \$BASE/healthy/*.txt 2>/dev/null || echo " NONE" - echo "Genome:" - ls -lh \$BASE/supporting-data/genome/* 2>/dev/null || echo " NONE" - echo "Support:" - ls -lh \$BASE/supporting-data/vcf/* \$BASE/supporting-data/ucsc-liftover/* 2>/dev/null || echo " NONE" - echo "Small:" - ls -lh \$BASE/MANE* \$BASE/regulon* \$BASE/LM22* \$BASE/ukbb* 2>/dev/null || echo " NONE" - - echo "=== STAGING COMPLETE ===" + echo "=== DEBUG COMPLETE ===" """ }