Add files using upload-large-folder tool

.gitattributes

@@ -33,3 +33,13 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+fold_0/logs.models.fold_0.ENCSR594NOE/logfile.modelling.fold_0.ENCSR594NOE.stdout_v1.txt filter=lfs diff=lfs merge=lfs -text
+fold_0/logs.models.fold_0.ENCSR594NOE/logfile.modelling.fold_0.ENCSR594NOE.stdout.txt filter=lfs diff=lfs merge=lfs -text
+fold_1/logs.models.fold_1.ENCSR594NOE/logfile.modelling.fold_1.ENCSR594NOE.stdout.txt filter=lfs diff=lfs merge=lfs -text
+fold_1/logs.models.fold_1.ENCSR594NOE/logfile.modelling.fold_1.ENCSR594NOE.stdout_v1.txt filter=lfs diff=lfs merge=lfs -text
+fold_4/logs.models.fold_4.ENCSR594NOE/logfile.modelling.fold_4.ENCSR594NOE.stdout.txt filter=lfs diff=lfs merge=lfs -text
+fold_4/logs.models.fold_4.ENCSR594NOE/logfile.modelling.fold_4.ENCSR594NOE.stdout_v1.txt filter=lfs diff=lfs merge=lfs -text
+fold_2/logs.models.fold_2.ENCSR594NOE/logfile.modelling.fold_2.ENCSR594NOE.stdout.txt filter=lfs diff=lfs merge=lfs -text
+fold_2/logs.models.fold_2.ENCSR594NOE/logfile.modelling.fold_2.ENCSR594NOE.stdout_v1.txt filter=lfs diff=lfs merge=lfs -text
+fold_3/logs.models.fold_3.ENCSR594NOE/logfile.modelling.fold_3.ENCSR594NOE.stdout_v1.txt filter=lfs diff=lfs merge=lfs -text
+fold_3/logs.models.fold_3.ENCSR594NOE/logfile.modelling.fold_3.ENCSR594NOE.stdout.txt filter=lfs diff=lfs merge=lfs -text

fold_0/logs.models.fold_0.ENCSR594NOE/logfile.modelling.fold_0.ENCSR594NOE.chrombpnet.params.json

@@ -0,0 +1,11 @@
+{
+    "counts_loss_weight": "56.8",
+    "filters": "512",
+    "n_dil_layers": "8",
+    "bias_model_path": "/oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR594NOE//chrombpnet_model_feb15_fold_0/bias_model_scaled.h5",
+    "inputlen": "2114",
+    "outputlen": "1000",
+    "max_jitter": "500",
+    "chr_fold_path": "/oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/splits/fold_0.json",
+    "negative_sampling_ratio": "0.1"
+}

fold_0/logs.models.fold_0.ENCSR594NOE/logfile.modelling.fold_0.ENCSR594NOE.chrombpnet_data_params.tsv

@@ -0,0 +1,3 @@
+counts_sum_min_thresh	31.0
+counts_sum_max_thresh	15641.54
+trainings_pts_post_thresh	166238

fold_0/logs.models.fold_0.ENCSR594NOE/logfile.modelling.fold_0.ENCSR594NOE.chrombpnet_no_bias_formatting.stdout.txt

@@ -0,0 +1 @@
+singularity exec --nv /home/groups/akundaje/anusri/simg/tf-atlas_gcp-modeling.sif python get_new_tf_model_format.py -i /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR594NOE//chrombpnet_model_feb15_fold_0/chrombpnet_wo_bias.h5 -o /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR594NOE//chrombpnet_model_feb15_fold_0/new_model_formats/chrombpnet_wo_bias

fold_0/logs.models.fold_0.ENCSR594NOE/logfile.modelling.fold_0.ENCSR594NOE.stdout.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd6461dadd89ad982338996480e6165d952e48fd6181e86d762455b4256052de
+size 13983848

fold_0/logs.models.fold_0.ENCSR594NOE/logfile.modelling.fold_0.ENCSR594NOE.stdout_v1.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6403f1e54e4c8426c172e9aa3ea4f8346271f4d8de3a1cdfd90ee39865cc848d
+size 13982163

fold_0/model.bias_scaled.fold_0.ENCSR594NOE.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ced64b8c2424c74189635814c17dbecb57254acd88ad0c3b85eb42976cd5f73
+size 2688440

fold_0/model.bias_scaled.fold_0.ENCSR594NOE.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b1f5eb8f926094e4d2d3e2d2f3cd8f47f9df123adef726bad4b6f3f3da7a9148
+size 1198080

fold_0/model.chrombpnet.fold_0.ENCSR594NOE.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e6594a878bc4fc394f88f15c6f555afb06766e452ffdf6b7c7408d8795a99ec3
+size 26447928

fold_0/model.chrombpnet.fold_0.ENCSR594NOE.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b28b1e20d30dd911f9c24eaad662c9e16a03371177304b2d78918ac75e27996
+size 27607040

fold_0/model.chrombpnet_nobias.fold_0.ENCSR594NOE.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:24a30cec28cd10ace5947d1b46115055fef5612f51126e06833bb9565deed13a
+size 25583536

fold_0/model.chrombpnet_nobias.fold_0.ENCSR594NOE.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e09a3858006ac6ef796983fb53242cf601403a6e6640e12f6f32baadfc6fdde
+size 26081280

fold_1/logs.models.fold_1.ENCSR594NOE/logfile.modelling.fold_1.ENCSR594NOE.stdout.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ff30b490651881f2d699884a5319b6f895f590e5e5d5e8ed56b1873628b0e3f
+size 13488322

fold_1/logs.models.fold_1.ENCSR594NOE/logfile.modelling.fold_1.ENCSR594NOE.stdout_v1.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b5e838150739364be20dbb504eab6b7f55c2195ca705bb7fdf37e49dc255f07
+size 13486649

fold_1/model.bias_scaled.fold_1.ENCSR594NOE.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:779dbb71d712d33260532670b03217eeafa4d44e740f999b0d9dc73f20c7dfc7
+size 2688440

fold_1/model.bias_scaled.fold_1.ENCSR594NOE.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3590648546877c6cc16303cde44f35181abfb4eb498b15b9e64f9c4ca196431a
+size 1198080

fold_1/model.chrombpnet.fold_1.ENCSR594NOE.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:782d873ad428a1dfd632ef3b695a2be1381f6a438bca9fa1bfa969e18d0bca76
+size 26447928

fold_1/model.chrombpnet.fold_1.ENCSR594NOE.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:319ecccc9cbc4c25e10714e90a0d861c6075a29c0a3597ef104fe19819864c5e
+size 27607040

fold_1/model.chrombpnet_nobias.fold_1.ENCSR594NOE.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a292461fb9f037dcd777f4448965c5211333095595d84545a5f65c3ce092cb42
+size 25583536

fold_1/model.chrombpnet_nobias.fold_1.ENCSR594NOE.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b1ebbad9ea8bfcfc1b1d7598c9911034919f024791138412408df055474cdfe0
+size 26081280

fold_2/logs.models.fold_2.ENCSR594NOE/logfile.modelling.fold_2.ENCSR594NOE.args.json

@@ -0,0 +1,23 @@
+{
+    "genome": "/scratch/groups/akundaje/anusri/chromatin_atlas/reference/hg38.genome.fa",
+    "bigwig": "/oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR594NOE//preprocessing/bigWigs/ENCSR594NOE.bigWig",
+    "peaks": "/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR594NOE//chrombpnet_model_feb15_fold_2//filtered.peaks.bed",
+    "nonpeaks": "/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR594NOE//chrombpnet_model_feb15_fold_2//filtered.nonpeaks.bed",
+    "output_prefix": "/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR594NOE//chrombpnet_model_feb15_fold_2//chrombpnet",
+    "chr_fold_path": "/scratch/groups/akundaje/anusri/chromatin_atlas/splits/fold_2.json",
+    "trackables": [
+        "logcount_predictions_loss",
+        "loss",
+        "logits_profile_predictions_loss",
+        "val_logcount_predictions_loss",
+        "val_loss",
+        "val_logits_profile_predictions_loss"
+    ],
+    "epochs": 50,
+    "early_stop": 5,
+    "batch_size": 64,
+    "learning_rate": 0.001,
+    "params": "/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR594NOE//chrombpnet_model_feb15_fold_2//chrombpnet_model_params.tsv",
+    "seed": 1234,
+    "architecture_from_file": "/home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/models/chrombpnet_with_bias_model.py"
+}

fold_2/logs.models.fold_2.ENCSR594NOE/logfile.modelling.fold_2.ENCSR594NOE.bias_formatting.stderr.txt

@@ -0,0 +1,38 @@
+INFO:    underlay of /etc/localtime required more than 50 (88) bind mounts
+INFO:    underlay of /usr/bin/nvidia-smi required more than 50 (355) bind mounts
+2023-08-18 16:27:28.566987: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-08-18 16:27:30.873493: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2023-08-18 16:27:30.876874: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1
+2023-08-18 16:27:31.531092: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: 
+pciBusID: 0000:c0:00.0 name: NVIDIA A100-SXM4-80GB computeCapability: 8.0
+coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 79.15GiB deviceMemoryBandwidth: 1.85TiB/s
+2023-08-18 16:27:31.531148: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-08-18 16:27:31.552632: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2023-08-18 16:27:31.552690: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2023-08-18 16:27:31.562524: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2023-08-18 16:27:31.567222: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2023-08-18 16:27:31.583877: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2023-08-18 16:27:31.588220: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2023-08-18 16:27:31.589162: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2023-08-18 16:27:31.605830: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2023-08-18 16:27:31.606228: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+2023-08-18 16:27:31.607436: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2023-08-18 16:27:31.628021: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: 
+pciBusID: 0000:c0:00.0 name: NVIDIA A100-SXM4-80GB computeCapability: 8.0
+coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 79.15GiB deviceMemoryBandwidth: 1.85TiB/s
+2023-08-18 16:27:31.628066: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-08-18 16:27:31.628097: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2023-08-18 16:27:31.628119: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2023-08-18 16:27:31.628140: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2023-08-18 16:27:31.628160: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2023-08-18 16:27:31.628179: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2023-08-18 16:27:31.628198: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2023-08-18 16:27:31.628218: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2023-08-18 16:27:31.669445: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2023-08-18 16:27:31.671883: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-08-18 16:27:34.621723: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix:
+2023-08-18 16:27:34.621790: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267]      0 
+2023-08-18 16:27:34.621803: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0:   N 
+2023-08-18 16:27:34.749707: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 75650 MB memory) -> physical GPU (device: 0, name: NVIDIA A100-SXM4-80GB, pci bus id: 0000:c0:00.0, compute capability: 8.0)
+2023-08-18 16:27:35.960113: W tensorflow/python/util/util.cc:348] Sets are not currently considered sequences, but this may change in the future, so consider avoiding using them.

fold_2/logs.models.fold_2.ENCSR594NOE/logfile.modelling.fold_2.ENCSR594NOE.bias_formatting.stdout.txt

@@ -0,0 +1 @@
+singularity exec --nv /home/groups/akundaje/anusri/simg/tf-atlas_gcp-modeling.sif python get_new_tf_model_format.py -i /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR594NOE//chrombpnet_model_feb15_fold_2/bias_model_scaled.h5 -o /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR594NOE//chrombpnet_model_feb15_fold_2/new_model_formats/bias_model_scaled

fold_2/logs.models.fold_2.ENCSR594NOE/logfile.modelling.fold_2.ENCSR594NOE.chrombpnet.params.json

@@ -0,0 +1,11 @@
+{
+    "counts_loss_weight": "56.8",
+    "filters": "512",
+    "n_dil_layers": "8",
+    "bias_model_path": "/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR594NOE//chrombpnet_model_feb15_fold_2/bias_model_scaled.h5",
+    "inputlen": "2114",
+    "outputlen": "1000",
+    "max_jitter": "500",
+    "chr_fold_path": "/scratch/groups/akundaje/anusri/chromatin_atlas/splits/fold_2.json",
+    "negative_sampling_ratio": "0.1"
+}

fold_2/logs.models.fold_2.ENCSR594NOE/logfile.modelling.fold_2.ENCSR594NOE.chrombpnet_data_params.tsv

@@ -0,0 +1,3 @@
+counts_sum_min_thresh	36.0
+counts_sum_max_thresh	16130.3
+trainings_pts_post_thresh	179461

fold_2/logs.models.fold_2.ENCSR594NOE/logfile.modelling.fold_2.ENCSR594NOE.chrombpnet_formatting.stderr.txt

@@ -0,0 +1,40 @@
+INFO:    underlay of /etc/localtime required more than 50 (88) bind mounts
+INFO:    underlay of /usr/bin/nvidia-smi required more than 50 (355) bind mounts
+2023-08-18 15:16:31.776877: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-08-18 15:16:34.254147: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2023-08-18 15:16:34.257874: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1
+2023-08-18 15:16:35.343038: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: 
+pciBusID: 0000:07:00.0 name: NVIDIA A100-SXM4-80GB computeCapability: 8.0
+coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 79.15GiB deviceMemoryBandwidth: 1.85TiB/s
+2023-08-18 15:16:35.343197: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-08-18 15:16:35.362172: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2023-08-18 15:16:35.362229: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2023-08-18 15:16:35.371669: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2023-08-18 15:16:35.376036: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2023-08-18 15:16:35.391734: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2023-08-18 15:16:35.395890: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2023-08-18 15:16:35.396797: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2023-08-18 15:16:35.507864: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2023-08-18 15:16:35.508171: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+2023-08-18 15:16:35.509018: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2023-08-18 15:16:35.598824: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: 
+pciBusID: 0000:07:00.0 name: NVIDIA A100-SXM4-80GB computeCapability: 8.0
+coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 79.15GiB deviceMemoryBandwidth: 1.85TiB/s
+2023-08-18 15:16:35.598861: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-08-18 15:16:35.598884: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2023-08-18 15:16:35.598906: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2023-08-18 15:16:35.598921: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2023-08-18 15:16:35.598934: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2023-08-18 15:16:35.598947: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2023-08-18 15:16:35.598961: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2023-08-18 15:16:35.598974: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2023-08-18 15:16:35.681571: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2023-08-18 15:16:35.682880: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-08-18 15:16:37.549528: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix:
+2023-08-18 15:16:37.549704: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267]      0 
+2023-08-18 15:16:37.549723: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0:   N 
+2023-08-18 15:16:37.556825: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 75650 MB memory) -> physical GPU (device: 0, name: NVIDIA A100-SXM4-80GB, pci bus id: 0000:07:00.0, compute capability: 8.0)
+2023-08-18 15:16:39.852731: W tensorflow/python/util/util.cc:348] Sets are not currently considered sequences, but this may change in the future, so consider avoiding using them.
+/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/layers/core.py:1059: UserWarning:  is not loaded, but a Lambda layer uses it. It may cause errors.
+  , UserWarning)

fold_2/logs.models.fold_2.ENCSR594NOE/logfile.modelling.fold_2.ENCSR594NOE.chrombpnet_formatting.stdout.txt

@@ -0,0 +1 @@
+singularity exec --nv /home/groups/akundaje/anusri/simg/tf-atlas_gcp-modeling.sif python get_new_tf_model_format.py -i /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR594NOE//chrombpnet_model_feb15_fold_2/chrombpnet.h5 -o /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR594NOE//chrombpnet_model_feb15_fold_2/new_model_formats/chrombpnet

fold_2/logs.models.fold_2.ENCSR594NOE/logfile.modelling.fold_2.ENCSR594NOE.chrombpnet_model_params.tsv

@@ -0,0 +1,9 @@
+counts_loss_weight	56.8
+filters	512
+n_dil_layers	8
+bias_model_path	/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR594NOE//chrombpnet_model_feb15_fold_2/bias_model_scaled.h5
+inputlen	2114
+outputlen	1000
+max_jitter	500
+chr_fold_path	/scratch/groups/akundaje/anusri/chromatin_atlas/splits/fold_2.json
+negative_sampling_ratio	0.1

fold_2/logs.models.fold_2.ENCSR594NOE/logfile.modelling.fold_2.ENCSR594NOE.chrombpnet_no_bias_formatting.stderr.txt

@@ -0,0 +1 @@
+singularity exec --nv /home/groups/akundaje/anusri/simg/tf-atlas_gcp-modeling.sif python get_new_tf_model_format.py -i /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR594NOE//chrombpnet_model_feb15_fold_2/chrombpnet_wo_bias.h5 -o /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR594NOE//chrombpnet_model_feb15_fold_2/new_model_formats/chrombpnet_wo_bias

fold_2/logs.models.fold_2.ENCSR594NOE/logfile.modelling.fold_2.ENCSR594NOE.epoch_loss.csv

@@ -0,0 +1,18 @@
+epoch,logcount_predictions_loss,logits_profile_predictions_loss,loss,val_logcount_predictions_loss,val_logits_profile_predictions_loss,val_loss
+0,1.8506684303283691,1735.555419921875,1840.67333984375,1.2115092277526855,1645.9083251953125,1714.721435546875
+1,1.0299354791641235,1544.0611572265625,1602.560546875,0.9900615811347961,1574.33837890625,1630.57470703125
+2,0.9585480690002441,1496.80712890625,1551.2520751953125,0.9128531813621521,1586.30029296875,1638.1510009765625
+3,0.9152749180793762,1468.258056640625,1520.2454833984375,0.9299142956733704,1543.91455078125,1596.7340087890625
+4,0.8744787573814392,1448.6949462890625,1498.365966796875,0.8831651210784912,1542.1495361328125,1592.3133544921875
+5,0.8403225541114807,1433.939453125,1481.668701171875,0.856060802936554,1565.5972900390625,1614.222412109375
+6,0.8122480511665344,1421.1552734375,1467.291748046875,0.8466861248016357,1547.9727783203125,1596.06396484375
+7,0.7884587049484253,1411.57470703125,1456.360595703125,0.8262052536010742,1531.436767578125,1578.364501953125
+8,0.7632220387458801,1404.3984375,1447.7493896484375,0.8552877306938171,1542.248291015625,1590.8292236328125
+9,0.7461246252059937,1396.8731689453125,1439.2515869140625,0.8513373136520386,1531.9486083984375,1580.3040771484375
+10,0.7288497686386108,1393.811767578125,1435.21044921875,0.8329720497131348,1554.103515625,1601.4163818359375
+11,0.6439115405082703,1361.7130126953125,1398.286376953125,0.8079436421394348,1520.5650634765625,1566.4554443359375
+12,0.6009595394134521,1346.3138427734375,1380.448486328125,0.8046413064002991,1520.9813232421875,1566.68505859375
+13,0.5687857270240784,1337.2587890625,1369.5660400390625,0.8444505929946899,1522.812255859375,1570.7779541015625
+14,0.5363478660583496,1330.8759765625,1361.3406982421875,0.8745900392532349,1528.48583984375,1578.1619873046875
+15,0.4832797944545746,1313.5443115234375,1340.993408203125,0.8381330370903015,1530.6826171875,1578.28857421875
+16,0.45772507786750793,1306.9765625,1332.976806640625,0.8475481271743774,1524.472900390625,1572.614013671875

fold_2/logs.models.fold_2.ENCSR594NOE/logfile.modelling.fold_2.ENCSR594NOE.stderr.txt

@@ -0,0 +1,332 @@
+INFO:    underlay of /etc/localtime required more than 50 (88) bind mounts
+INFO:    underlay of /usr/bin/nvidia-smi required more than 50 (355) bind mounts
+INFO:    underlay of /etc/localtime required more than 50 (88) bind mounts
+INFO:    underlay of /usr/bin/nvidia-smi required more than 50 (355) bind mounts
+2023-01-18 01:05:28.901275: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-01-18 01:13:57.132908: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2023-01-18 01:13:57.137156: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1
+2023-01-18 01:13:57.267098: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: 
+pciBusID: 0000:4d:00.0 name: NVIDIA A100-SXM4-40GB computeCapability: 8.0
+coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 39.42GiB deviceMemoryBandwidth: 1.41TiB/s
+2023-01-18 01:13:57.267187: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-01-18 01:13:57.346894: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2023-01-18 01:13:57.347083: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2023-01-18 01:13:57.363166: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2023-01-18 01:13:57.449769: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2023-01-18 01:13:57.518570: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2023-01-18 01:13:57.540541: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2023-01-18 01:13:57.544021: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2023-01-18 01:13:57.548864: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2023-01-18 01:13:57.549242: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+2023-01-18 01:13:57.550149: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2023-01-18 01:13:57.551957: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: 
+pciBusID: 0000:4d:00.0 name: NVIDIA A100-SXM4-40GB computeCapability: 8.0
+coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 39.42GiB deviceMemoryBandwidth: 1.41TiB/s
+2023-01-18 01:13:57.551987: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-01-18 01:13:57.552005: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2023-01-18 01:13:57.552020: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2023-01-18 01:13:57.552035: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2023-01-18 01:13:57.552050: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2023-01-18 01:13:57.552065: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2023-01-18 01:13:57.552080: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2023-01-18 01:13:57.552094: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2023-01-18 01:13:57.555573: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2023-01-18 01:13:57.556865: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-01-18 01:14:00.262081: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix:
+2023-01-18 01:14:00.262178: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267]      0 
+2023-01-18 01:14:00.262191: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0:   N 
+2023-01-18 01:14:00.269085: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 37414 MB memory) -> physical GPU (device: 0, name: NVIDIA A100-SXM4-40GB, pci bus id: 0000:4d:00.0, compute capability: 8.0)
+2023-01-18 01:14:01.644830: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2)
+2023-01-18 01:14:01.659148: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 1999930000 Hz
+2023-01-18 01:14:01.865636: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2023-01-18 01:14:04.942987: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2023-01-18 01:14:04.951904: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2023-01-18 01:14:41.511974: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-01-18 01:14:43.386584: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2023-01-18 01:14:43.855945: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1
+2023-01-18 01:14:43.953880: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: 
+pciBusID: 0000:4d:00.0 name: NVIDIA A100-SXM4-40GB computeCapability: 8.0
+coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 39.42GiB deviceMemoryBandwidth: 1.41TiB/s
+2023-01-18 01:14:43.954010: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-01-18 01:14:43.956896: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2023-01-18 01:14:43.957003: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2023-01-18 01:14:43.958221: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2023-01-18 01:14:43.958534: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2023-01-18 01:14:43.961010: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2023-01-18 01:14:43.961662: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2023-01-18 01:14:43.961956: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2023-01-18 01:14:43.963873: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2023-01-18 01:14:43.964199: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+2023-01-18 01:14:43.964280: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2023-01-18 01:14:43.965240: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: 
+pciBusID: 0000:4d:00.0 name: NVIDIA A100-SXM4-40GB computeCapability: 8.0
+coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 39.42GiB deviceMemoryBandwidth: 1.41TiB/s
+2023-01-18 01:14:43.965265: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-01-18 01:14:43.965282: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2023-01-18 01:14:43.965297: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2023-01-18 01:14:43.965311: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2023-01-18 01:14:43.965325: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2023-01-18 01:14:43.965339: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2023-01-18 01:14:43.965352: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2023-01-18 01:14:43.965379: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2023-01-18 01:14:43.967135: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2023-01-18 01:14:43.967170: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-01-18 01:14:44.463110: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix:
+2023-01-18 01:14:44.463243: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267]      0 
+2023-01-18 01:14:44.463256: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0:   N 
+2023-01-18 01:14:44.466766: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 37414 MB memory) -> physical GPU (device: 0, name: NVIDIA A100-SXM4-40GB, pci bus id: 0000:4d:00.0, compute capability: 8.0)
+2023-01-18 01:20:43.643858: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2)
+2023-01-18 01:20:43.644335: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 1999930000 Hz
+2023-01-18 01:20:45.191631: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2023-01-18 01:20:45.796067: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2023-01-18 01:20:45.814379: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2023-01-18 01:20:49.065753: I tensorflow/stream_executor/cuda/cuda_blas.cc:1838] TensorFloat-32 will be used for the matrix multiplication. This will only be logged once.
+2023-01-18 03:19:54.021872: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-01-18 03:19:57.048215: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2023-01-18 03:19:57.049377: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1
+2023-01-18 03:19:57.197801: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: 
+pciBusID: 0000:4d:00.0 name: NVIDIA A100-SXM4-40GB computeCapability: 8.0
+coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 39.42GiB deviceMemoryBandwidth: 1.41TiB/s
+2023-01-18 03:19:57.197930: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-01-18 03:19:57.200684: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2023-01-18 03:19:57.200760: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2023-01-18 03:19:57.201970: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2023-01-18 03:19:57.202272: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2023-01-18 03:19:57.204945: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2023-01-18 03:19:57.205634: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2023-01-18 03:19:57.205999: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2023-01-18 03:19:57.209653: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2023-01-18 03:19:57.210002: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+2023-01-18 03:19:57.210085: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2023-01-18 03:19:57.211881: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: 
+pciBusID: 0000:4d:00.0 name: NVIDIA A100-SXM4-40GB computeCapability: 8.0
+coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 39.42GiB deviceMemoryBandwidth: 1.41TiB/s
+2023-01-18 03:19:57.211906: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-01-18 03:19:57.211926: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2023-01-18 03:19:57.211942: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2023-01-18 03:19:57.211958: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2023-01-18 03:19:57.211974: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2023-01-18 03:19:57.211989: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2023-01-18 03:19:57.212004: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2023-01-18 03:19:57.212019: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2023-01-18 03:19:57.215538: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2023-01-18 03:19:57.215574: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-01-18 03:19:57.755943: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix:
+2023-01-18 03:19:57.756088: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267]      0 
+2023-01-18 03:19:57.756100: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0:   N 
+2023-01-18 03:19:57.759908: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 37414 MB memory) -> physical GPU (device: 0, name: NVIDIA A100-SXM4-40GB, pci bus id: 0000:4d:00.0, compute capability: 8.0)
+2023-01-18 03:22:05.385462: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2)
+2023-01-18 03:22:05.389300: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 1999930000 Hz
+2023-01-18 03:22:05.493409: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2023-01-18 03:22:06.053503: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2023-01-18 03:22:06.055926: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/layers/core.py:1059: UserWarning:  is not loaded, but a Lambda layer uses it. It may cause errors.
+  , UserWarning)
+/home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:69: RuntimeWarning: invalid value encountered in true_divide
+  cur_jsd=jensenshannon(true_counts[idx,:]/np.nansum(true_counts[idx,:]),pred_probs[idx,:])
+/home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/utils/metrics_utils.py:196: RuntimeWarning: invalid value encountered in true_divide
+  profile_prob = profile / np.sum(profile)
+/home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:78: RuntimeWarning: invalid value encountered in true_divide
+  shuffled_labels_prob=shuffled_labels/np.nansum(shuffled_labels)
+/home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:88: RuntimeWarning: invalid value encountered in true_divide
+  curr_jsd_rnd=jensenshannon(true_counts[idx,:]/np.nansum(true_counts[idx,:]),shuffled_labels_prob)
+No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.
+findfont: Font family ['normal'] not found. Falling back to DejaVu Sans.
+findfont: Font family ['normal'] not found. Falling back to DejaVu Sans.
+findfont: Font family ['normal'] not found. Falling back to DejaVu Sans.
+No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.
+No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.
+2023-01-18 03:24:27.805469: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-01-18 03:24:30.449593: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2023-01-18 03:24:30.450789: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1
+2023-01-18 03:24:30.593012: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: 
+pciBusID: 0000:4d:00.0 name: NVIDIA A100-SXM4-40GB computeCapability: 8.0
+coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 39.42GiB deviceMemoryBandwidth: 1.41TiB/s
+2023-01-18 03:24:30.593139: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-01-18 03:24:30.595906: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2023-01-18 03:24:30.595962: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2023-01-18 03:24:30.597166: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2023-01-18 03:24:30.597485: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2023-01-18 03:24:30.600089: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2023-01-18 03:24:30.600750: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2023-01-18 03:24:30.601100: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2023-01-18 03:24:30.604755: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2023-01-18 03:24:30.605090: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+2023-01-18 03:24:30.605178: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2023-01-18 03:24:30.606966: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: 
+pciBusID: 0000:4d:00.0 name: NVIDIA A100-SXM4-40GB computeCapability: 8.0
+coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 39.42GiB deviceMemoryBandwidth: 1.41TiB/s
+2023-01-18 03:24:30.606990: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-01-18 03:24:30.607009: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2023-01-18 03:24:30.607024: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2023-01-18 03:24:30.607039: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2023-01-18 03:24:30.607053: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2023-01-18 03:24:30.607068: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2023-01-18 03:24:30.607082: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2023-01-18 03:24:30.607097: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2023-01-18 03:24:30.610578: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2023-01-18 03:24:30.610610: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-01-18 03:24:31.149451: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix:
+2023-01-18 03:24:31.149589: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267]      0 
+2023-01-18 03:24:31.149602: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0:   N 
+2023-01-18 03:24:31.153331: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 37414 MB memory) -> physical GPU (device: 0, name: NVIDIA A100-SXM4-40GB, pci bus id: 0000:4d:00.0, compute capability: 8.0)
+WARNING:tensorflow:No training configuration found in the save file, so the model was *not* compiled. Compile it manually.
+2023-01-18 03:25:42.775787: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2)
+2023-01-18 03:25:42.778459: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 1999930000 Hz
+2023-01-18 03:25:42.840949: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2023-01-18 03:25:43.380467: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2023-01-18 03:25:43.382275: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+/home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:69: RuntimeWarning: invalid value encountered in true_divide
+  cur_jsd=jensenshannon(true_counts[idx,:]/np.nansum(true_counts[idx,:]),pred_probs[idx,:])
+/home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/utils/metrics_utils.py:196: RuntimeWarning: invalid value encountered in true_divide
+  profile_prob = profile / np.sum(profile)
+/home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:78: RuntimeWarning: invalid value encountered in true_divide
+  shuffled_labels_prob=shuffled_labels/np.nansum(shuffled_labels)
+/home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:88: RuntimeWarning: invalid value encountered in true_divide
+  curr_jsd_rnd=jensenshannon(true_counts[idx,:]/np.nansum(true_counts[idx,:]),shuffled_labels_prob)
+No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.
+findfont: Font family ['normal'] not found. Falling back to DejaVu Sans.
+findfont: Font family ['normal'] not found. Falling back to DejaVu Sans.
+findfont: Font family ['normal'] not found. Falling back to DejaVu Sans.
+No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.
+No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.
+2023-01-18 03:28:28.342131: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-01-18 03:28:31.144096: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2023-01-18 03:28:31.145290: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1
+2023-01-18 03:28:31.292206: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: 
+pciBusID: 0000:4d:00.0 name: NVIDIA A100-SXM4-40GB computeCapability: 8.0
+coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 39.42GiB deviceMemoryBandwidth: 1.41TiB/s
+2023-01-18 03:28:31.292344: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-01-18 03:28:31.295100: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2023-01-18 03:28:31.295157: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2023-01-18 03:28:31.296381: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2023-01-18 03:28:31.296686: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2023-01-18 03:28:31.299251: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2023-01-18 03:28:31.299918: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2023-01-18 03:28:31.300270: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2023-01-18 03:28:31.303952: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2023-01-18 03:28:31.304296: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+2023-01-18 03:28:31.304389: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2023-01-18 03:28:31.306183: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: 
+pciBusID: 0000:4d:00.0 name: NVIDIA A100-SXM4-40GB computeCapability: 8.0
+coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 39.42GiB deviceMemoryBandwidth: 1.41TiB/s
+2023-01-18 03:28:31.306207: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-01-18 03:28:31.306251: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2023-01-18 03:28:31.306269: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2023-01-18 03:28:31.306284: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2023-01-18 03:28:31.306299: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2023-01-18 03:28:31.306314: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2023-01-18 03:28:31.306328: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2023-01-18 03:28:31.306343: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2023-01-18 03:28:31.309841: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2023-01-18 03:28:31.309874: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-01-18 03:28:31.835274: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix:
+2023-01-18 03:28:31.835428: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267]      0 
+2023-01-18 03:28:31.835441: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0:   N 
+2023-01-18 03:28:31.839282: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 37414 MB memory) -> physical GPU (device: 0, name: NVIDIA A100-SXM4-40GB, pci bus id: 0000:4d:00.0, compute capability: 8.0)
+2023-01-18 03:29:46.367852: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2)
+2023-01-18 03:29:46.369809: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 1999930000 Hz
+2023-01-18 03:29:46.411675: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2023-01-18 03:29:46.970443: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2023-01-18 03:29:46.972268: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+/home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:69: RuntimeWarning: invalid value encountered in true_divide
+  cur_jsd=jensenshannon(true_counts[idx,:]/np.nansum(true_counts[idx,:]),pred_probs[idx,:])
+/home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/utils/metrics_utils.py:196: RuntimeWarning: invalid value encountered in true_divide
+  profile_prob = profile / np.sum(profile)
+/home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:78: RuntimeWarning: invalid value encountered in true_divide
+  shuffled_labels_prob=shuffled_labels/np.nansum(shuffled_labels)
+/home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:88: RuntimeWarning: invalid value encountered in true_divide
+  curr_jsd_rnd=jensenshannon(true_counts[idx,:]/np.nansum(true_counts[idx,:]),shuffled_labels_prob)
+No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.
+findfont: Font family ['normal'] not found. Falling back to DejaVu Sans.
+findfont: Font family ['normal'] not found. Falling back to DejaVu Sans.
+findfont: Font family ['normal'] not found. Falling back to DejaVu Sans.
+No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.
+No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.
+2023-01-18 03:30:55.604764: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-01-18 03:30:56.974121: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2023-01-18 03:30:56.975255: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1
+2023-01-18 03:30:57.119560: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: 
+pciBusID: 0000:4d:00.0 name: NVIDIA A100-SXM4-40GB computeCapability: 8.0
+coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 39.42GiB deviceMemoryBandwidth: 1.41TiB/s
+2023-01-18 03:30:57.119694: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-01-18 03:30:57.122412: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2023-01-18 03:30:57.122479: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2023-01-18 03:30:57.123669: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2023-01-18 03:30:57.123974: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2023-01-18 03:30:57.126546: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2023-01-18 03:30:57.127208: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2023-01-18 03:30:57.127574: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2023-01-18 03:30:57.131211: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2023-01-18 03:30:57.131543: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+2023-01-18 03:30:57.131631: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2023-01-18 03:30:57.133446: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: 
+pciBusID: 0000:4d:00.0 name: NVIDIA A100-SXM4-40GB computeCapability: 8.0
+coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 39.42GiB deviceMemoryBandwidth: 1.41TiB/s
+2023-01-18 03:30:57.133477: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-01-18 03:30:57.133494: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2023-01-18 03:30:57.133510: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2023-01-18 03:30:57.133524: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2023-01-18 03:30:57.133539: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2023-01-18 03:30:57.133553: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2023-01-18 03:30:57.133568: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2023-01-18 03:30:57.133582: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2023-01-18 03:30:57.137091: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2023-01-18 03:30:57.137128: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-01-18 03:30:57.666964: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix:
+2023-01-18 03:30:57.667098: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267]      0 
+2023-01-18 03:30:57.667111: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0:   N 
+2023-01-18 03:30:57.670929: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 37414 MB memory) -> physical GPU (device: 0, name: NVIDIA A100-SXM4-40GB, pci bus id: 0000:4d:00.0, compute capability: 8.0)
+WARNING:tensorflow:No training configuration found in the save file, so the model was *not* compiled. Compile it manually.
+2023-01-18 03:31:15.373294: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2)
+2023-01-18 03:31:15.373937: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 1999930000 Hz
+2023-01-18 03:31:15.598878: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2023-01-18 03:31:16.275816: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2023-01-18 03:31:16.277828: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+mkdir: cannot create directory ‘/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR594NOE//chrombpnet_model_feb15_fold_2//footprints’: File exists
+2023-01-18 03:33:05.579648: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-01-18 03:33:06.970921: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2023-01-18 03:33:06.972030: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1
+2023-01-18 03:33:07.000552: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: 
+pciBusID: 0000:4d:00.0 name: NVIDIA A100-SXM4-40GB computeCapability: 8.0
+coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 39.42GiB deviceMemoryBandwidth: 1.41TiB/s
+2023-01-18 03:33:07.000669: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-01-18 03:33:07.003380: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2023-01-18 03:33:07.003443: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2023-01-18 03:33:07.004626: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2023-01-18 03:33:07.004930: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2023-01-18 03:33:07.007496: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2023-01-18 03:33:07.008169: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2023-01-18 03:33:07.008543: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2023-01-18 03:33:07.013407: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2023-01-18 03:33:07.013752: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+2023-01-18 03:33:07.013858: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2023-01-18 03:33:07.015762: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: 
+pciBusID: 0000:4d:00.0 name: NVIDIA A100-SXM4-40GB computeCapability: 8.0
+coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 39.42GiB deviceMemoryBandwidth: 1.41TiB/s
+2023-01-18 03:33:07.015795: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-01-18 03:33:07.015813: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2023-01-18 03:33:07.015829: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2023-01-18 03:33:07.015845: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2023-01-18 03:33:07.015860: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2023-01-18 03:33:07.015875: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2023-01-18 03:33:07.015890: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2023-01-18 03:33:07.015905: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2023-01-18 03:33:07.021583: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2023-01-18 03:33:07.021624: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-01-18 03:33:07.815021: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix:
+2023-01-18 03:33:08.647141: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267]      0 
+2023-01-18 03:33:08.647249: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0:   N 
+2023-01-18 03:33:08.662799: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 37414 MB memory) -> physical GPU (device: 0, name: NVIDIA A100-SXM4-40GB, pci bus id: 0000:4d:00.0, compute capability: 8.0)
+2023-01-18 03:33:25.719335: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2)
+2023-01-18 03:33:25.719990: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 1999930000 Hz
+2023-01-18 03:33:25.870910: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2023-01-18 03:33:26.500117: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2023-01-18 03:33:26.502081: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8

fold_2/logs.models.fold_2.ENCSR594NOE/logfile.modelling.fold_2.ENCSR594NOE.stdout.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:880fcece1d33f1868c8cb15667c5fb1bebdd4d82fb43d6bad70db36064dd542f
+size 13330610

fold_2/logs.models.fold_2.ENCSR594NOE/logfile.modelling.fold_2.ENCSR594NOE.stdout_v1.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ce88456d2fb387e88367814c2544c6c150f0946f328f9c3c8089af984a0c914
+size 13328937

fold_2/model.bias_scaled.fold_2.ENCSR594NOE.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b8196eec57d05d56bd3027d7b40997f80be0c84a73c963bfc70e013de49340f
+size 2688440

fold_2/model.bias_scaled.fold_2.ENCSR594NOE.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:02540e7f83e0e15343f1b88cbdc3a386a6060f2afd3990c8c5ff9b6fc0ae55c8
+size 1198080

fold_2/model.chrombpnet.fold_2.ENCSR594NOE.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:825bc4333ecc909d0cc36f12e05412ae94ebbeb71e7350b94eef51e9c56a2c5b
+size 26447928

fold_2/model.chrombpnet.fold_2.ENCSR594NOE.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e3ee934e4afb3afb0bf27784442c80e5175ada09b9a37525c99b254b3570211
+size 27607040

fold_2/model.chrombpnet_nobias.fold_2.ENCSR594NOE.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff14bda5e43cddc1748b1c1c5437ad47c90a43d9d6bc7a703ce9bbea17e02467
+size 25583536

fold_2/model.chrombpnet_nobias.fold_2.ENCSR594NOE.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0011d997bf5af85fdc750278514c6717a49ee656067f74c09073bc4ac02a0b29
+size 26081280

fold_3/logs.models.fold_3.ENCSR594NOE/logfile.modelling.fold_3.ENCSR594NOE.stdout.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:26c6e70120a13ba445fd8805c0a07baac8d2529b2517b67dd112fbc49fe11147
+size 16878825

fold_3/logs.models.fold_3.ENCSR594NOE/logfile.modelling.fold_3.ENCSR594NOE.stdout_v1.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:97cf1d2e9c3400b32a5178bd006b1288fc6a1074d02807facc9c8e5f66d019cf
+size 16877152

fold_3/model.bias_scaled.fold_3.ENCSR594NOE.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6f759da5043e5db5bdf4c18d0a6b31a105f3054739eb6ce1999fcae231ec7773
+size 2688440

fold_3/model.bias_scaled.fold_3.ENCSR594NOE.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:88753c84ffa8c8af9c0a6fe17e0450b46299c3a2ef05f8d508d47bffda19464e
+size 1198080

fold_3/model.chrombpnet.fold_3.ENCSR594NOE.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b082c77bd6df1ffc6fa3148f0f320fece1f8d415aad7c507a09d540e35b9b923
+size 26447928

fold_3/model.chrombpnet.fold_3.ENCSR594NOE.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd5f3054cec7404861d9add178af6825b6d7405bbae78368f50f205f888c979f
+size 27607040

fold_3/model.chrombpnet_nobias.fold_3.ENCSR594NOE.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e55e47a4905b3b6a27adce3d3026e2bba14d824391c5e09b59ee9fa773293190
+size 25583536

fold_3/model.chrombpnet_nobias.fold_3.ENCSR594NOE.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:24328565c3979e5543cce3a94877a65e10e9f6ae058a8744d66f79df607c98d7
+size 26081280

fold_4/logs.models.fold_4.ENCSR594NOE/logfile.modelling.fold_4.ENCSR594NOE.stdout.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e0ff41664b500d73dfcf77bc33c16783eb79ccbb9861a9c28b87093c50846f80
+size 15179064