Digital Trials pipeline configured for WES
Source-only snapshot of the cluster branch for WES execution. Large reference files (HPA/MANE/ensemble FASTA, model weights, ~597 MB) are omitted: they are baked into the container images at build time and mounted from the dreamdock-data PVC at runtime, and exceed the Gitea request size limit. Pipeline entry point is main.nf, which orchestrates the biotransformer, conplex and tissue modules as a single workflow. Ligand inputs are read from the eureka workspace; protein_zarr and chembl_db come from the dreamdock-data PVC.
This commit is contained in:
Olamide Isreal
15
k8s/README.md
Normal file
15
k8s/README.md
Normal file
@@ -0,0 +1,15 @@
|
||||
# access the current workspace
|
||||
kubectl exec -it -n bioinformatics $(kubectl get pod -l app=digital-trials-nextflow -n bioinformatics | grep Run | awk '{ print $1 }') -- bash
|
||||
nextflow run main.nf -profile k8s
|
||||
|
||||
# DEPRECATE: cleanup error pods in bioinformatics
|
||||
kubectl get pod -n bioinformatics | grep -E "Pending|Error" | awk '{print $1}' | xargs -P 10 -I {} kubectl delete pod -n bioinformatics {}
|
||||
|
||||
# sync data (/data/bugra/similarity-search/utility/vec_db/nf_fingerptint) from node to workspace
|
||||
kubectl delete -f k8s/job-copy-node-to-pvc.yaml; kubectl apply -f k8s/job-copy-node-to-pvc.yaml
|
||||
|
||||
# run the actual nextflow jobs
|
||||
kubectl apply -f k8s/job-nextflow-digital-trials.yaml
|
||||
|
||||
# [WARNING] destroy the curent nextflow job
|
||||
kubectl delete -f k8s/job-nextflow-digital-trials.yaml
|
||||
73
k8s/deployment-worksplace.yaml
Normal file
73
k8s/deployment-worksplace.yaml
Normal file
@@ -0,0 +1,73 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: digital-trials-nextflow
|
||||
namespace: bioinformatics
|
||||
spec:
|
||||
progressDeadlineSeconds: 600
|
||||
replicas: 1
|
||||
revisionHistoryLimit: 10
|
||||
selector:
|
||||
matchLabels:
|
||||
app: digital-trials-nextflow
|
||||
strategy:
|
||||
rollingUpdate:
|
||||
maxSurge: 25%
|
||||
maxUnavailable: 25%
|
||||
type: RollingUpdate
|
||||
template:
|
||||
metadata:
|
||||
creationTimestamp: null
|
||||
labels:
|
||||
app: digital-trials-nextflow
|
||||
spec:
|
||||
containers:
|
||||
- command:
|
||||
- sleep
|
||||
- infinity
|
||||
image: nextflow/nextflow:25.04.6
|
||||
imagePullPolicy: IfNotPresent
|
||||
name: nextflow
|
||||
workingDir: /mnt/dreamdock-data/digital_trials
|
||||
resources:
|
||||
limits:
|
||||
cpu: "2"
|
||||
memory: 4Gi
|
||||
requests:
|
||||
cpu: "1"
|
||||
memory: 2Gi
|
||||
terminationMessagePath: /dev/termination-log
|
||||
terminationMessagePolicy: File
|
||||
volumeMounts:
|
||||
- mountPath: /mnt/ZINC-22
|
||||
name: zinc-22-volume
|
||||
- mountPath: /mnt/dreamdock-data
|
||||
name: dreamdock-volume
|
||||
dnsPolicy: ClusterFirst
|
||||
restartPolicy: Always
|
||||
schedulerName: default-scheduler
|
||||
securityContext: {}
|
||||
serviceAccount: nextflow-sa
|
||||
serviceAccountName: nextflow-sa
|
||||
terminationGracePeriodSeconds: 30
|
||||
volumes:
|
||||
- name: zinc-22-volume
|
||||
hostPath:
|
||||
path: /mnt/ZINC-22
|
||||
type: Directory
|
||||
- name: dreamdock-volume
|
||||
persistentVolumeClaim:
|
||||
claimName: dreamdock-data
|
||||
# ---
|
||||
# apiVersion: v1
|
||||
# kind: PersistentVolumeClaim
|
||||
# metadata:
|
||||
# name: digital-trials-data
|
||||
# namespace: bioinformatics
|
||||
# spec:
|
||||
# accessModes:
|
||||
# - ReadWriteMany
|
||||
# resources:
|
||||
# requests:
|
||||
# storage: 6000Gi
|
||||
# storageClassName: truenas-nfs
|
||||
54
k8s/job-copy-node-to-pvc.yaml
Normal file
54
k8s/job-copy-node-to-pvc.yaml
Normal file
@@ -0,0 +1,54 @@
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: job-data-copy-digital-trials-node-to-pvc
|
||||
namespace: bioinformatics
|
||||
spec:
|
||||
backoffLimit: 6
|
||||
completionMode: NonIndexed
|
||||
completions: 1
|
||||
manualSelector: false
|
||||
parallelism: 1
|
||||
suspend: false
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- args:
|
||||
- |
|
||||
apk add --no-cache rsync
|
||||
mkdir -p /target
|
||||
rsync -av --exclude .git/ --exclude .git-old/ --exclude work/ /source/ /target/digital_trials/
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
image: alpine
|
||||
imagePullPolicy: Always
|
||||
name: data-copy
|
||||
resources: {}
|
||||
terminationMessagePath: /dev/termination-log
|
||||
terminationMessagePolicy: File
|
||||
volumeMounts:
|
||||
- mountPath: /source
|
||||
name: k8s-node
|
||||
- mountPath: /target
|
||||
name: pvc-volume
|
||||
dnsPolicy: ClusterFirst
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: k8s-node23
|
||||
restartPolicy: Never
|
||||
schedulerName: default-scheduler
|
||||
securityContext: {}
|
||||
terminationGracePeriodSeconds: 30
|
||||
tolerations:
|
||||
- effect: NoExecute
|
||||
key: omic-app
|
||||
operator: Equal
|
||||
value: similarity-search
|
||||
volumes:
|
||||
- name: pvc-volume
|
||||
persistentVolumeClaim:
|
||||
claimName: dreamdock-data
|
||||
- hostPath:
|
||||
path: /data/bugra/digital_trials ## CHANGE ME
|
||||
type: ""
|
||||
name: k8s-node
|
||||
52
k8s/job-nextflow-digital-trials.yaml
Normal file
52
k8s/job-nextflow-digital-trials.yaml
Normal file
@@ -0,0 +1,52 @@
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: job-nextflow-digital-trials
|
||||
namespace: bioinformatics
|
||||
spec:
|
||||
backoffLimit: 1
|
||||
completionMode: NonIndexed
|
||||
completions: 1
|
||||
manualSelector: false
|
||||
parallelism: 1
|
||||
podReplacementPolicy: TerminatingOrFailed
|
||||
suspend: false
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- command:
|
||||
- /bin/bash
|
||||
- -c
|
||||
- cd /mnt/dreamdock-data/digital_trials && nextflow run main.nf -profile k8s ## CHANGE ME
|
||||
image: nextflow/nextflow:25.04.6
|
||||
imagePullPolicy: IfNotPresent
|
||||
name: nextflow
|
||||
resources:
|
||||
limits:
|
||||
cpu: "4"
|
||||
memory: 8Gi
|
||||
requests:
|
||||
cpu: "2"
|
||||
memory: 4Gi
|
||||
terminationMessagePath: /dev/termination-log
|
||||
terminationMessagePolicy: File
|
||||
volumeMounts: ## CHANGE ME
|
||||
- mountPath: /mnt/ZINC-22
|
||||
name: zinc-22-volume
|
||||
- mountPath: /mnt/dreamdock-data
|
||||
name: dreamdock-volume
|
||||
dnsPolicy: ClusterFirst
|
||||
restartPolicy: Never
|
||||
schedulerName: default-scheduler
|
||||
securityContext: {}
|
||||
serviceAccount: nextflow-sa
|
||||
serviceAccountName: nextflow-sa
|
||||
terminationGracePeriodSeconds: 30
|
||||
volumes: ## CHANGE ME
|
||||
- name: zinc-22-volume
|
||||
hostPath:
|
||||
path: /mnt/ZINC-22
|
||||
type: Directory
|
||||
- name: dreamdock-volume
|
||||
persistentVolumeClaim:
|
||||
claimName: dreamdock-data
|
||||
Reference in New Issue
Block a user