Hello,
I am trying to test this first tutorial :
I got several errors (KeyError: u’cyclist’), I have probably forgotten something ?
my conf file :
dataset_config {
data_sources: {
tfrecords_path: "/workspace/hd/download/kitti/outputtfrecord/*"
image_directory_path: "/workspace/hd/download/kitti/"
}
image_extension: "png"
target_class_mapping {
key: "car"
value: "car"
}
target_class_mapping {
key: "pedestrian"
value: "pedestrian"
}
target_class_mapping {
key: "cyclist"
value: "cyclist"
}
validation_fold: 0
}
augmentation_config {
preprocessing {
output_image_width: 1248
output_image_height: 384
output_image_channel: 3
min_bbox_width: 1.0
min_bbox_height: 1.0
}
spatial_augmentation {
hflip_probability: 0.5
zoom_min: 1.0
zoom_max: 1.0
translate_max_x: 8.0
translate_max_y: 8.0
}
color_augmentation {
hue_rotation_max: 25.0
saturation_shift_max: 0.2
contrast_scale_max: 0.1
contrast_center: 0.5
}
}
model_config {
pretrained_model_file: "model-download/tlt_resnet18_detectnet_v2_v1/resnet18.hdf5"
num_layers: 18
freeze_blocks: 0
arch: "resnet"
use_batch_norm: true
objective_set {
bbox {
scale: 35.0
offset: 0.5
}
cov {
}
}
training_precision {
backend_floatx: FLOAT32
}
}
training_config {
batch_size_per_gpu: 14
num_epochs: 120
learning_rate {
soft_start_annealing_schedule {
min_learning_rate: 5e-06
max_learning_rate: 0.0005
soft_start: 0.1
annealing: 0.7
}
}
regularizer {
type: L1
weight: 3e-09
}
optimizer {
adam {
epsilon: 9.9e-09
beta1: 0.9
beta2: 0.999
}
}
cost_scaling {
initial_exponent: 20.0
increment: 0.005
decrement: 1.0
}
checkpoint_interval: 10
}
evaluation_config {
average_precision_mode: INTEGRATE
validation_period_during_training: 10
first_validation_epoch: 1
minimum_detection_ground_truth_overlap {
key: "car"
value: 0.7
}
minimum_detection_ground_truth_overlap {
key: "cyclist"
value: 0.5
}
minimum_detection_ground_truth_overlap {
key: "pedestrian"
value: 0.5
}
}
bbox_rasterizer_config {
target_class_config {
key: "car"
value {
cov_center_x: 0.5
cov_center_y: 0.5
cov_radius_x: 0.4
cov_radius_y: 0.4
bbox_min_radius: 1.0
}
}
target_class_config {
key: "cyclist"
value {
cov_center_x: 0.5
cov_center_y: 0.5
cov_radius_x: 1.0
cov_radius_y: 1.0
bbox_min_radius: 1.0
}
}
target_class_config {
key: "pedestrian"
value {
cov_center_x: 0.5
cov_center_y: 0.5
cov_radius_x: 1.0
cov_radius_y: 1.0
bbox_min_radius: 1.0
}
}
deadzone_radius: 0.67
}
postprocessing_config {
target_class_config {
key: "car"
value {
clustering_config {
coverage_threshold: 0.005
dbscan_eps: 0.13
dbscan_min_samples: 0.05
minimum_bounding_box_height: 4
}
}
}
target_class_config {
key: "cyclist"
value {
clustering_config {
coverage_threshold: 0.005
dbscan_eps: 0.15
dbscan_min_samples: 0.05
minimum_bounding_box_height: 4
}
}
}
target_class_config {
key: "pedestrian"
value {
clustering_config {
coverage_threshold: 0.005
dbscan_eps: 0.15
dbscan_min_samples: 0.05
minimum_bounding_box_height: 4
}
}
}
}
cost_function_config {
target_classes {
name: "car"
class_weight: 1.0
coverage_foreground_weight: 0.05
objectives {
name: "cov"
initial_weight: 1.0
weight_target: 1.0
}
objectives {
name: "bbox"
initial_weight: 10.0
weight_target: 10.0
}
}
target_classes {
name: "cyclist"
class_weight: 1.0
coverage_foreground_weight: 0.05
objectives {
name: "cov"
initial_weight: 1.0
weight_target: 1.0
}
objectives {
name: "bbox"
initial_weight: 10.0
weight_target: 10.0
}
}
target_classes {
name: "pedestrian"
class_weight: 1.0
coverage_foreground_weight: 0.05
objectives {
name: "cov"
initial_weight: 1.0
weight_target: 1.0
}
objectives {
name: "bbox"
initial_weight: 10.0
weight_target: 10.0
}
}
}
shell output :
root@c748bd377f66:/workspace/hd/download/kitti# tlt-train detectnet_v2 --gpus 1 -r results/ -e conf -k mykey
Using TensorFlow backend.
2019-10-12 09:48:53.312682: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
2019-10-12 09:48:53.458358: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:998] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2019-10-12 09:48:53.459593: I tensorflow/compiler/xla/service/service.cc:150] XLA service 0x76da2c0 executing computations on platform CUDA. Devices:
2019-10-12 09:48:53.459640: I tensorflow/compiler/xla/service/service.cc:158] StreamExecutor device (0): GeForce RTX 2070, Compute Capability 7.5
2019-10-12 09:48:53.461802: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2496000000 Hz
2019-10-12 09:48:53.462023: I tensorflow/compiler/xla/service/service.cc:150] XLA service 0x7742180 executing computations on platform Host. Devices:
2019-10-12 09:48:53.462047: I tensorflow/compiler/xla/service/service.cc:158] StreamExecutor device (0): <undefined>, <undefined>
2019-10-12 09:48:53.462253: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1433] Found device 0 with properties:
name: GeForce RTX 2070 major: 7 minor: 5 memoryClockRate(GHz): 1.62
pciBusID: 0000:01:00.0
totalMemory: 7.76GiB freeMemory: 7.36GiB
2019-10-12 09:48:53.462282: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1512] Adding visible gpu devices: 0
2019-10-12 09:48:53.463750: I tensorflow/core/common_runtime/gpu/gpu_device.cc:984] Device interconnect StreamExecutor with strength 1 edge matrix:
2019-10-12 09:48:53.463790: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 0
2019-10-12 09:48:53.463805: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1003] 0: N
2019-10-12 09:48:53.464032: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 7158 MB memory) -> physical GPU (device: 0, name: GeForce RTX 2070, pci bus id: 0000:01:00.0, compute capability: 7.5)
2019-10-12 09:48:53,464 [INFO] iva.detectnet_v2.scripts.train: Loading experiment spec at conf.
2019-10-12 09:48:53,465 [INFO] iva.detectnet_v2.spec_handler.spec_loader: Merging specification from conf
WARNING:tensorflow:From ./detectnet_v2/dataloader/utilities.py:114: tf_record_iterator (from tensorflow.python.lib.io.tf_record) is deprecated and will be removed in a future version.
Instructions for updating:
Use eager execution and:
`tf.data.TFRecordDataset(path)`
2019-10-12 09:48:53,518 [WARNING] tensorflow: From ./detectnet_v2/dataloader/utilities.py:114: tf_record_iterator (from tensorflow.python.lib.io.tf_record) is deprecated and will be removed in a future version.
Instructions for updating:
Use eager execution and:
`tf.data.TFRecordDataset(path)`
2019-10-12 09:48:53,613 [INFO] iva.detectnet_v2.scripts.train: Cannot iterate over exactly 5985 samples with a batch size of 14; each epoch will therefore take one extra step.
WARNING:tensorflow:From /usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.
Instructions for updating:
Colocations handled automatically by placer.
2019-10-12 09:48:53,619 [WARNING] tensorflow: From /usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.
Instructions for updating:
Colocations handled automatically by placer.
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
input_1 (InputLayer) (None, 3, 384, 1248) 0
__________________________________________________________________________________________________
conv1 (Conv2D) (None, 64, 192, 624) 9472 input_1[0][0]
__________________________________________________________________________________________________
bn_conv1 (BatchNormalization) (None, 64, 192, 624) 256 conv1[0][0]
__________________________________________________________________________________________________
activation_1 (Activation) (None, 64, 192, 624) 0 bn_conv1[0][0]
__________________________________________________________________________________________________
block_1a_conv_1 (Conv2D) (None, 64, 96, 312) 36928 activation_1[0][0]
__________________________________________________________________________________________________
block_1a_bn_1 (BatchNormalizati (None, 64, 96, 312) 256 block_1a_conv_1[0][0]
__________________________________________________________________________________________________
activation_2 (Activation) (None, 64, 96, 312) 0 block_1a_bn_1[0][0]
__________________________________________________________________________________________________
block_1a_conv_2 (Conv2D) (None, 64, 96, 312) 36928 activation_2[0][0]
__________________________________________________________________________________________________
block_1a_conv_shortcut (Conv2D) (None, 64, 96, 312) 4160 activation_1[0][0]
__________________________________________________________________________________________________
block_1a_bn_2 (BatchNormalizati (None, 64, 96, 312) 256 block_1a_conv_2[0][0]
__________________________________________________________________________________________________
block_1a_bn_shortcut (BatchNorm (None, 64, 96, 312) 256 block_1a_conv_shortcut[0][0]
__________________________________________________________________________________________________
add_1 (Add) (None, 64, 96, 312) 0 block_1a_bn_2[0][0]
block_1a_bn_shortcut[0][0]
__________________________________________________________________________________________________
activation_3 (Activation) (None, 64, 96, 312) 0 add_1[0][0]
__________________________________________________________________________________________________
block_1b_conv_1 (Conv2D) (None, 64, 96, 312) 36928 activation_3[0][0]
__________________________________________________________________________________________________
block_1b_bn_1 (BatchNormalizati (None, 64, 96, 312) 256 block_1b_conv_1[0][0]
__________________________________________________________________________________________________
activation_4 (Activation) (None, 64, 96, 312) 0 block_1b_bn_1[0][0]
__________________________________________________________________________________________________
block_1b_conv_2 (Conv2D) (None, 64, 96, 312) 36928 activation_4[0][0]
__________________________________________________________________________________________________
block_1b_bn_2 (BatchNormalizati (None, 64, 96, 312) 256 block_1b_conv_2[0][0]
__________________________________________________________________________________________________
add_2 (Add) (None, 64, 96, 312) 0 block_1b_bn_2[0][0]
activation_3[0][0]
__________________________________________________________________________________________________
activation_5 (Activation) (None, 64, 96, 312) 0 add_2[0][0]
__________________________________________________________________________________________________
block_2a_conv_1 (Conv2D) (None, 128, 48, 156) 73856 activation_5[0][0]
__________________________________________________________________________________________________
block_2a_bn_1 (BatchNormalizati (None, 128, 48, 156) 512 block_2a_conv_1[0][0]
__________________________________________________________________________________________________
activation_6 (Activation) (None, 128, 48, 156) 0 block_2a_bn_1[0][0]
__________________________________________________________________________________________________
block_2a_conv_2 (Conv2D) (None, 128, 48, 156) 147584 activation_6[0][0]
__________________________________________________________________________________________________
block_2a_conv_shortcut (Conv2D) (None, 128, 48, 156) 8320 activation_5[0][0]
__________________________________________________________________________________________________
block_2a_bn_2 (BatchNormalizati (None, 128, 48, 156) 512 block_2a_conv_2[0][0]
__________________________________________________________________________________________________
block_2a_bn_shortcut (BatchNorm (None, 128, 48, 156) 512 block_2a_conv_shortcut[0][0]
__________________________________________________________________________________________________
add_3 (Add) (None, 128, 48, 156) 0 block_2a_bn_2[0][0]
block_2a_bn_shortcut[0][0]
__________________________________________________________________________________________________
activation_7 (Activation) (None, 128, 48, 156) 0 add_3[0][0]
__________________________________________________________________________________________________
block_2b_conv_1 (Conv2D) (None, 128, 48, 156) 147584 activation_7[0][0]
__________________________________________________________________________________________________
block_2b_bn_1 (BatchNormalizati (None, 128, 48, 156) 512 block_2b_conv_1[0][0]
__________________________________________________________________________________________________
activation_8 (Activation) (None, 128, 48, 156) 0 block_2b_bn_1[0][0]
__________________________________________________________________________________________________
block_2b_conv_2 (Conv2D) (None, 128, 48, 156) 147584 activation_8[0][0]
__________________________________________________________________________________________________
block_2b_bn_2 (BatchNormalizati (None, 128, 48, 156) 512 block_2b_conv_2[0][0]
__________________________________________________________________________________________________
add_4 (Add) (None, 128, 48, 156) 0 block_2b_bn_2[0][0]
activation_7[0][0]
__________________________________________________________________________________________________
activation_9 (Activation) (None, 128, 48, 156) 0 add_4[0][0]
__________________________________________________________________________________________________
block_3a_conv_1 (Conv2D) (None, 256, 24, 78) 295168 activation_9[0][0]
__________________________________________________________________________________________________
block_3a_bn_1 (BatchNormalizati (None, 256, 24, 78) 1024 block_3a_conv_1[0][0]
__________________________________________________________________________________________________
activation_10 (Activation) (None, 256, 24, 78) 0 block_3a_bn_1[0][0]
__________________________________________________________________________________________________
block_3a_conv_2 (Conv2D) (None, 256, 24, 78) 590080 activation_10[0][0]
__________________________________________________________________________________________________
block_3a_conv_shortcut (Conv2D) (None, 256, 24, 78) 33024 activation_9[0][0]
__________________________________________________________________________________________________
block_3a_bn_2 (BatchNormalizati (None, 256, 24, 78) 1024 block_3a_conv_2[0][0]
__________________________________________________________________________________________________
block_3a_bn_shortcut (BatchNorm (None, 256, 24, 78) 1024 block_3a_conv_shortcut[0][0]
__________________________________________________________________________________________________
add_5 (Add) (None, 256, 24, 78) 0 block_3a_bn_2[0][0]
block_3a_bn_shortcut[0][0]
__________________________________________________________________________________________________
activation_11 (Activation) (None, 256, 24, 78) 0 add_5[0][0]
__________________________________________________________________________________________________
block_3b_conv_1 (Conv2D) (None, 256, 24, 78) 590080 activation_11[0][0]
__________________________________________________________________________________________________
block_3b_bn_1 (BatchNormalizati (None, 256, 24, 78) 1024 block_3b_conv_1[0][0]
__________________________________________________________________________________________________
activation_12 (Activation) (None, 256, 24, 78) 0 block_3b_bn_1[0][0]
__________________________________________________________________________________________________
block_3b_conv_2 (Conv2D) (None, 256, 24, 78) 590080 activation_12[0][0]
__________________________________________________________________________________________________
block_3b_bn_2 (BatchNormalizati (None, 256, 24, 78) 1024 block_3b_conv_2[0][0]
__________________________________________________________________________________________________
add_6 (Add) (None, 256, 24, 78) 0 block_3b_bn_2[0][0]
activation_11[0][0]
__________________________________________________________________________________________________
activation_13 (Activation) (None, 256, 24, 78) 0 add_6[0][0]
__________________________________________________________________________________________________
block_4a_conv_1 (Conv2D) (None, 512, 24, 78) 1180160 activation_13[0][0]
__________________________________________________________________________________________________
block_4a_bn_1 (BatchNormalizati (None, 512, 24, 78) 2048 block_4a_conv_1[0][0]
__________________________________________________________________________________________________
activation_14 (Activation) (None, 512, 24, 78) 0 block_4a_bn_1[0][0]
__________________________________________________________________________________________________
block_4a_conv_2 (Conv2D) (None, 512, 24, 78) 2359808 activation_14[0][0]
__________________________________________________________________________________________________
block_4a_conv_shortcut (Conv2D) (None, 512, 24, 78) 131584 activation_13[0][0]
__________________________________________________________________________________________________
block_4a_bn_2 (BatchNormalizati (None, 512, 24, 78) 2048 block_4a_conv_2[0][0]
__________________________________________________________________________________________________
block_4a_bn_shortcut (BatchNorm (None, 512, 24, 78) 2048 block_4a_conv_shortcut[0][0]
__________________________________________________________________________________________________
add_7 (Add) (None, 512, 24, 78) 0 block_4a_bn_2[0][0]
block_4a_bn_shortcut[0][0]
__________________________________________________________________________________________________
activation_15 (Activation) (None, 512, 24, 78) 0 add_7[0][0]
__________________________________________________________________________________________________
block_4b_conv_1 (Conv2D) (None, 512, 24, 78) 2359808 activation_15[0][0]
__________________________________________________________________________________________________
block_4b_bn_1 (BatchNormalizati (None, 512, 24, 78) 2048 block_4b_conv_1[0][0]
__________________________________________________________________________________________________
activation_16 (Activation) (None, 512, 24, 78) 0 block_4b_bn_1[0][0]
__________________________________________________________________________________________________
block_4b_conv_2 (Conv2D) (None, 512, 24, 78) 2359808 activation_16[0][0]
__________________________________________________________________________________________________
block_4b_bn_2 (BatchNormalizati (None, 512, 24, 78) 2048 block_4b_conv_2[0][0]
__________________________________________________________________________________________________
add_8 (Add) (None, 512, 24, 78) 0 block_4b_bn_2[0][0]
activation_15[0][0]
__________________________________________________________________________________________________
activation_17 (Activation) (None, 512, 24, 78) 0 add_8[0][0]
__________________________________________________________________________________________________
output_bbox (Conv2D) (None, 12, 24, 78) 6156 activation_17[0][0]
__________________________________________________________________________________________________
output_cov (Conv2D) (None, 3, 24, 78) 1539 activation_17[0][0]
==================================================================================================
Total params: 11,203,023
Trainable params: 11,183,823
Non-trainable params: 19,200
__________________________________________________________________________________________________
target/truncation is not updated to match the crop areaif the dataset contains target/truncation.
target/truncation is not updated to match the crop areaif the dataset contains target/truncation.
target/truncation is not updated to match the crop areaif the dataset contains target/truncation.
target/truncation is not updated to match the crop areaif the dataset contains target/truncation.
target/truncation is not updated to match the crop areaif the dataset contains target/truncation.
target/truncation is not updated to match the crop areaif the dataset contains target/truncation.
target/truncation is not updated to match the crop areaif the dataset contains target/truncation.
target/truncation is not updated to match the crop areaif the dataset contains target/truncation.
target/truncation is not updated to match the crop areaif the dataset contains target/truncation.
target/truncation is not updated to match the crop areaif the dataset contains target/truncation.
target/truncation is not updated to match the crop areaif the dataset contains target/truncation.
target/truncation is not updated to match the crop areaif the dataset contains target/truncation.
target/truncation is not updated to match the crop areaif the dataset contains target/truncation.
target/truncation is not updated to match the crop areaif the dataset contains target/truncation.
2019-10-12 09:49:12,558 [INFO] iva.detectnet_v2.scripts.train: Found 5985 samples in training set
target/truncation is not updated to match the crop areaif the dataset contains target/truncation.
target/truncation is not updated to match the crop areaif the dataset contains target/truncation.
target/truncation is not updated to match the crop areaif the dataset contains target/truncation.
target/truncation is not updated to match the crop areaif the dataset contains target/truncation.
target/truncation is not updated to match the crop areaif the dataset contains target/truncation.
target/truncation is not updated to match the crop areaif the dataset contains target/truncation.
target/truncation is not updated to match the crop areaif the dataset contains target/truncation.
target/truncation is not updated to match the crop areaif the dataset contains target/truncation.
target/truncation is not updated to match the crop areaif the dataset contains target/truncation.
target/truncation is not updated to match the crop areaif the dataset contains target/truncation.
target/truncation is not updated to match the crop areaif the dataset contains target/truncation.
target/truncation is not updated to match the crop areaif the dataset contains target/truncation.
target/truncation is not updated to match the crop areaif the dataset contains target/truncation.
target/truncation is not updated to match the crop areaif the dataset contains target/truncation.
2019-10-12 09:49:27,033 [INFO] iva.detectnet_v2.scripts.train: Found 1496 samples in validation set
INFO:tensorflow:Create CheckpointSaverHook.
2019-10-12 09:49:39,200 [INFO] tensorflow: Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
2019-10-12 09:49:40,363 [INFO] tensorflow: Graph was finalized.
2019-10-12 09:49:40.363746: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1512] Adding visible gpu devices: 0
2019-10-12 09:49:40.363781: I tensorflow/core/common_runtime/gpu/gpu_device.cc:984] Device interconnect StreamExecutor with strength 1 edge matrix:
2019-10-12 09:49:40.363793: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 0
2019-10-12 09:49:40.363802: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1003] 0: N
2019-10-12 09:49:40.363923: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 7158 MB memory) -> physical GPU (device: 0, name: GeForce RTX 2070, pci bus id: 0000:01:00.0, compute capability: 7.5)
INFO:tensorflow:Running local_init_op.
2019-10-12 09:49:43,652 [INFO] tensorflow: Running local_init_op.
INFO:tensorflow:Done running local_init_op.
2019-10-12 09:49:44,323 [INFO] tensorflow: Done running local_init_op.
INFO:tensorflow:Saving checkpoints for step-0.
2019-10-12 09:50:06,359 [INFO] tensorflow: Saving checkpoints for step-0.
2019-10-12 09:50:48.655312: I tensorflow/stream_executor/dso_loader.cc:152] successfully opened CUDA library libcublas.so.10.0 locally
2019-10-12 09:50:48.949078: I tensorflow/core/kernels/cuda_solvers.cc:159] Creating CudaSolver handles for stream 0x7773840
2019-10-12 09:50:51.718673: E tensorflow/stream_executor/cuda/cuda_driver.cc:806] failed to allocate 2.99G (3207918336 bytes) from device: CUDA_ERROR_OUT_OF_MEMORY: out of memory
INFO:tensorflow:epoch = 0.0, loss = 0.09195674, step = 0
2019-10-12 09:50:53,934 [INFO] tensorflow: epoch = 0.0, loss = 0.09195674, step = 0
2019-10-12 09:50:53,936 [INFO] /usr/local/lib/python2.7/dist-packages/iva/detectnet_v2/tfhooks/task_progress_monitor_hook.pyc: Epoch 0/120: loss: 0.09196 Time taken: 0:00:00 ETA: 0:00:00
INFO:tensorflow:epoch = 0.004672897196261682, loss = 0.08913421, step = 2 (13.194 sec)
2019-10-12 09:51:07,128 [INFO] tensorflow: epoch = 0.004672897196261682, loss = 0.08913421, step = 2 (13.194 sec)
INFO:tensorflow:epoch = 0.02803738317757009, loss = 0.07545308, step = 12 (6.173 sec)
2019-10-12 09:51:13,301 [INFO] tensorflow: epoch = 0.02803738317757009, loss = 0.07545308, step = 12 (6.173 sec)
INFO:tensorflow:epoch = 0.04906542056074766, loss = 0.061266925, step = 21 (5.759 sec)
2019-10-12 09:51:19,060 [INFO] tensorflow: epoch = 0.04906542056074766, loss = 0.061266925, step = 21 (5.759 sec)
2019-10-12 09:51:20,927 [INFO] /usr/local/lib/python2.7/dist-packages/iva/detectnet_v2/tfhooks/sample_counter_hook.pyc: Samples / sec: 9.094
INFO:tensorflow:epoch = 0.07242990654205607, loss = 0.047333784, step = 31 (6.209 sec)
2019-10-12 09:51:25,269 [INFO] tensorflow: epoch = 0.07242990654205607, loss = 0.047333784, step = 31 (6.209 sec)
INFO:tensorflow:epoch = 0.09579439252336448, loss = 0.03620739, step = 41 (6.173 sec)
2019-10-12 09:51:31,442 [INFO] tensorflow: epoch = 0.09579439252336448, loss = 0.03620739, step = 41 (6.173 sec)
INFO:tensorflow:global_step/sec: 0.950732
2019-10-12 09:51:38,112 [INFO] tensorflow: global_step/sec: 0.950732
INFO:tensorflow:epoch = 0.10046728971962617, loss = 0.03373318, step = 43 (7.302 sec)
2019-10-12 09:51:38,744 [INFO] tensorflow: epoch = 0.10046728971962617, loss = 0.03373318, step = 43 (7.302 sec)
2019-10-12 09:51:42,449 [INFO] /usr/local/lib/python2.7/dist-packages/iva/detectnet_v2/tfhooks/sample_counter_hook.pyc: Samples / sec: 16.263
INFO:tensorflow:epoch = 0.12383177570093457, loss = 0.021546174, step = 53 (6.167 sec)
2019-10-12 09:51:44,911 [INFO] tensorflow: epoch = 0.12383177570093457, loss = 0.021546174, step = 53 (6.167 sec)
INFO:tensorflow:epoch = 0.14719626168224298, loss = 0.018093007, step = 63 (6.289 sec)
2019-10-12 09:51:51,200 [INFO] tensorflow: epoch = 0.14719626168224298, loss = 0.018093007, step = 63 (6.289 sec)
INFO:tensorflow:epoch = 0.1705607476635514, loss = 0.011953581, step = 73 (6.191 sec)
2019-10-12 09:51:57,391 [INFO] tensorflow: epoch = 0.1705607476635514, loss = 0.011953581, step = 73 (6.191 sec)
2019-10-12 09:51:57,989 [INFO] /usr/local/lib/python2.7/dist-packages/iva/detectnet_v2/tfhooks/sample_counter_hook.pyc: Samples / sec: 22.523
INFO:tensorflow:epoch = 0.1939252336448598, loss = 0.010523062, step = 83 (6.219 sec)
2019-10-12 09:52:03,610 [INFO] tensorflow: epoch = 0.1939252336448598, loss = 0.010523062, step = 83 (6.219 sec)
INFO:tensorflow:global_step/sec: 1.60067
2019-10-12 09:52:04,351 [INFO] tensorflow: global_step/sec: 1.60067
INFO:tensorflow:epoch = 0.21728971962616822, loss = 0.008785188, step = 93 (6.240 sec)
2019-10-12 09:52:09,850 [INFO] tensorflow: epoch = 0.21728971962616822, loss = 0.008785188, step = 93 (6.240 sec)
2019-10-12 09:52:13,606 [INFO] /usr/local/lib/python2.7/dist-packages/iva/detectnet_v2/tfhooks/sample_counter_hook.pyc: Samples / sec: 22.412
INFO:tensorflow:epoch = 0.2406542056074766, loss = 0.007459309, step = 103 (6.250 sec)
2019-10-12 09:52:16,100 [INFO] tensorflow: epoch = 0.2406542056074766, loss = 0.007459309, step = 103 (6.250 sec)
INFO:tensorflow:epoch = 0.26401869158878505, loss = 0.006734521, step = 113 (6.201 sec)
2019-10-12 09:52:22,300 [INFO] tensorflow: epoch = 0.26401869158878505, loss = 0.006734521, step = 113 (6.201 sec)
INFO:tensorflow:epoch = 0.2850467289719626, loss = 0.0057151946, step = 122 (5.834 sec)
2019-10-12 09:52:28,135 [INFO] tensorflow: epoch = 0.2850467289719626, loss = 0.0057151946, step = 122 (5.834 sec)
2019-10-12 09:52:29,409 [INFO] /usr/local/lib/python2.7/dist-packages/iva/detectnet_v2/tfhooks/sample_counter_hook.pyc: Samples / sec: 22.148
INFO:tensorflow:global_step/sec: 1.59295
2019-10-12 09:52:30,717 [INFO] tensorflow: global_step/sec: 1.59295
INFO:tensorflow:epoch = 0.30607476635514014, loss = 0.0057842317, step = 131 (5.753 sec)
2019-10-12 09:52:33,888 [INFO] tensorflow: epoch = 0.30607476635514014, loss = 0.0057842317, step = 131 (5.753 sec)
INFO:tensorflow:epoch = 0.32710280373831774, loss = 0.0052827913, step = 140 (5.633 sec)
2019-10-12 09:52:39,521 [INFO] tensorflow: epoch = 0.32710280373831774, loss = 0.0052827913, step = 140 (5.633 sec)
2019-10-12 09:52:45,126 [INFO] /usr/local/lib/python2.7/dist-packages/iva/detectnet_v2/tfhooks/sample_counter_hook.pyc: Samples / sec: 22.269
INFO:tensorflow:epoch = 0.35046728971962615, loss = 0.004640116, step = 150 (6.238 sec)
2019-10-12 09:52:45,759 [INFO] tensorflow: epoch = 0.35046728971962615, loss = 0.004640116, step = 150 (6.238 sec)
INFO:tensorflow:epoch = 0.3714953271028037, loss = 0.004948322, step = 159 (5.694 sec)
2019-10-12 09:52:51,453 [INFO] tensorflow: epoch = 0.3714953271028037, loss = 0.004948322, step = 159 (5.694 sec)
INFO:tensorflow:epoch = 0.3925233644859813, loss = 0.0043838015, step = 168 (5.637 sec)
2019-10-12 09:52:57,090 [INFO] tensorflow: epoch = 0.3925233644859813, loss = 0.0043838015, step = 168 (5.637 sec)
INFO:tensorflow:global_step/sec: 1.59246
2019-10-12 09:52:57,091 [INFO] tensorflow: global_step/sec: 1.59246
2019-10-12 09:53:00,847 [INFO] /usr/local/lib/python2.7/dist-packages/iva/detectnet_v2/tfhooks/sample_counter_hook.pyc: Samples / sec: 22.264
INFO:tensorflow:epoch = 0.41355140186915884, loss = 0.004247729, step = 177 (5.719 sec)
2019-10-12 09:53:02,809 [INFO] tensorflow: epoch = 0.41355140186915884, loss = 0.004247729, step = 177 (5.719 sec)
INFO:tensorflow:epoch = 0.43691588785046725, loss = 0.003548611, step = 187 (6.145 sec)
2019-10-12 09:53:08,954 [INFO] tensorflow: epoch = 0.43691588785046725, loss = 0.003548611, step = 187 (6.145 sec)
INFO:tensorflow:epoch = 0.4579439252336448, loss = 0.004053987, step = 196 (5.615 sec)
2019-10-12 09:53:14,569 [INFO] tensorflow: epoch = 0.4579439252336448, loss = 0.004053987, step = 196 (5.615 sec)
2019-10-12 09:53:16,428 [INFO] /usr/local/lib/python2.7/dist-packages/iva/detectnet_v2/tfhooks/sample_counter_hook.pyc: Samples / sec: 22.464
INFO:tensorflow:epoch = 0.4813084112149532, loss = 0.0037323842, step = 206 (6.120 sec)
2019-10-12 09:53:20,689 [INFO] tensorflow: epoch = 0.4813084112149532, loss = 0.0037323842, step = 206 (6.120 sec)
INFO:tensorflow:global_step/sec: 1.60885
2019-10-12 09:53:23,197 [INFO] tensorflow: global_step/sec: 1.60885
INFO:tensorflow:epoch = 0.5023364485981308, loss = 0.0035405941, step = 215 (5.625 sec)
2019-10-12 09:53:26,314 [INFO] tensorflow: epoch = 0.5023364485981308, loss = 0.0035405941, step = 215 (5.625 sec)
INFO:tensorflow:epoch = 0.5233644859813084, loss = 0.0033171896, step = 224 (5.629 sec)
2019-10-12 09:53:31,943 [INFO] tensorflow: epoch = 0.5233644859813084, loss = 0.0033171896, step = 224 (5.629 sec)
2019-10-12 09:53:31,944 [INFO] /usr/local/lib/python2.7/dist-packages/iva/detectnet_v2/tfhooks/sample_counter_hook.pyc: Samples / sec: 22.558
INFO:tensorflow:epoch = 0.5467289719626168, loss = 0.0028232406, step = 234 (6.104 sec)
2019-10-12 09:53:38,047 [INFO] tensorflow: epoch = 0.5467289719626168, loss = 0.0028232406, step = 234 (6.104 sec)
INFO:tensorflow:epoch = 0.5677570093457943, loss = 0.0031780032, step = 243 (5.696 sec)
2019-10-12 09:53:43,743 [INFO] tensorflow: epoch = 0.5677570093457943, loss = 0.0031780032, step = 243 (5.696 sec)
2019-10-12 09:53:47,317 [INFO] /usr/local/lib/python2.7/dist-packages/iva/detectnet_v2/tfhooks/sample_counter_hook.pyc: Samples / sec: 22.767
INFO:tensorflow:global_step/sec: 1.61957
2019-10-12 09:53:49,130 [INFO] tensorflow: global_step/sec: 1.61957
INFO:tensorflow:epoch = 0.5911214953271028, loss = 0.0031626946, step = 253 (6.022 sec)
2019-10-12 09:53:49,764 [INFO] tensorflow: epoch = 0.5911214953271028, loss = 0.0031626946, step = 253 (6.022 sec)
INFO:tensorflow:epoch = 0.6121495327102803, loss = 0.0032624006, step = 262 (5.905 sec)
2019-10-12 09:53:55,669 [INFO] tensorflow: epoch = 0.6121495327102803, loss = 0.0032624006, step = 262 (5.905 sec)
INFO:tensorflow:epoch = 0.6355140186915887, loss = 0.002991713, step = 272 (6.107 sec)
2019-10-12 09:54:01,776 [INFO] tensorflow: epoch = 0.6355140186915887, loss = 0.002991713, step = 272 (6.107 sec)
2019-10-12 09:54:03,116 [INFO] /usr/local/lib/python2.7/dist-packages/iva/detectnet_v2/tfhooks/sample_counter_hook.pyc: Samples / sec: 22.153
INFO:tensorflow:epoch = 0.6565420560747663, loss = 0.0030114937, step = 281 (5.786 sec)
2019-10-12 09:54:07,563 [INFO] tensorflow: epoch = 0.6565420560747663, loss = 0.0030114937, step = 281 (5.786 sec)
INFO:tensorflow:epoch = 0.6775700934579438, loss = 0.0026158905, step = 290 (5.619 sec)
2019-10-12 09:54:13,181 [INFO] tensorflow: epoch = 0.6775700934579438, loss = 0.0026158905, step = 290 (5.619 sec)
INFO:tensorflow:global_step/sec: 1.57824
2019-10-12 09:54:15,741 [INFO] tensorflow: global_step/sec: 1.57824
INFO:tensorflow:epoch = 0.6985981308411214, loss = 0.0028272897, step = 299 (5.745 sec)
2019-10-12 09:54:18,927 [INFO] tensorflow: epoch = 0.6985981308411214, loss = 0.0028272897, step = 299 (5.745 sec)
2019-10-12 09:54:18,927 [INFO] /usr/local/lib/python2.7/dist-packages/iva/detectnet_v2/tfhooks/sample_counter_hook.pyc: Samples / sec: 22.137
INFO:tensorflow:epoch = 0.7219626168224299, loss = 0.0025348759, step = 309 (6.216 sec)
2019-10-12 09:54:25,142 [INFO] tensorflow: epoch = 0.7219626168224299, loss = 0.0025348759, step = 309 (6.216 sec)
INFO:tensorflow:epoch = 0.7429906542056074, loss = 0.0033877161, step = 318 (5.645 sec)
2019-10-12 09:54:30,787 [INFO] tensorflow: epoch = 0.7429906542056074, loss = 0.0033877161, step = 318 (5.645 sec)
2019-10-12 09:54:34,667 [INFO] /usr/local/lib/python2.7/dist-packages/iva/detectnet_v2/tfhooks/sample_counter_hook.pyc: Samples / sec: 22.237
INFO:tensorflow:epoch = 0.764018691588785, loss = 0.0025100755, step = 327 (5.823 sec)
2019-10-12 09:54:36,611 [INFO] tensorflow: epoch = 0.764018691588785, loss = 0.0025100755, step = 327 (5.823 sec)
INFO:tensorflow:global_step/sec: 1.58561
2019-10-12 09:54:42,230 [INFO] tensorflow: global_step/sec: 1.58561
INFO:tensorflow:epoch = 0.7873831775700934, loss = 0.0021754005, step = 337 (6.244 sec)
2019-10-12 09:54:42,855 [INFO] tensorflow: epoch = 0.7873831775700934, loss = 0.0021754005, step = 337 (6.244 sec)
INFO:tensorflow:epoch = 0.808411214953271, loss = 0.0023828594, step = 346 (5.708 sec)
2019-10-12 09:54:48,563 [INFO] tensorflow: epoch = 0.808411214953271, loss = 0.0023828594, step = 346 (5.708 sec)
2019-10-12 09:54:50,416 [INFO] /usr/local/lib/python2.7/dist-packages/iva/detectnet_v2/tfhooks/sample_counter_hook.pyc: Samples / sec: 22.224
INFO:tensorflow:epoch = 0.8317757009345794, loss = 0.0030601362, step = 356 (6.164 sec)
2019-10-12 09:54:54,727 [INFO] tensorflow: epoch = 0.8317757009345794, loss = 0.0030601362, step = 356 (6.164 sec)
INFO:tensorflow:epoch = 0.8528037383177569, loss = 0.002007912, step = 365 (5.767 sec)
2019-10-12 09:55:00,494 [INFO] tensorflow: epoch = 0.8528037383177569, loss = 0.002007912, step = 365 (5.767 sec)
INFO:tensorflow:epoch = 0.8738317757009345, loss = 0.0019262063, step = 374 (5.660 sec)
2019-10-12 09:55:06,155 [INFO] tensorflow: epoch = 0.8738317757009345, loss = 0.0019262063, step = 374 (5.660 sec)
2019-10-12 09:55:06,155 [INFO] /usr/local/lib/python2.7/dist-packages/iva/detectnet_v2/tfhooks/sample_counter_hook.pyc: Samples / sec: 22.239
INFO:tensorflow:global_step/sec: 1.59245
2019-10-12 09:55:08,604 [INFO] tensorflow: global_step/sec: 1.59245
INFO:tensorflow:epoch = 0.8971962616822429, loss = 0.0030834365, step = 384 (6.192 sec)
2019-10-12 09:55:12,347 [INFO] tensorflow: epoch = 0.8971962616822429, loss = 0.0030834365, step = 384 (6.192 sec)
INFO:tensorflow:epoch = 0.9205607476635513, loss = 0.0025320747, step = 394 (6.127 sec)
2019-10-12 09:55:18,474 [INFO] tensorflow: epoch = 0.9205607476635513, loss = 0.0025320747, step = 394 (6.127 sec)
2019-10-12 09:55:21,733 [INFO] /usr/local/lib/python2.7/dist-packages/iva/detectnet_v2/tfhooks/sample_counter_hook.pyc: Samples / sec: 22.467
INFO:tensorflow:epoch = 0.9415887850467289, loss = 0.003184855, step = 403 (5.825 sec)
2019-10-12 09:55:24,299 [INFO] tensorflow: epoch = 0.9415887850467289, loss = 0.003184855, step = 403 (5.825 sec)
INFO:tensorflow:epoch = 0.9649532710280373, loss = 0.002503721, step = 413 (6.233 sec)
2019-10-12 09:55:30,532 [INFO] tensorflow: epoch = 0.9649532710280373, loss = 0.002503721, step = 413 (6.233 sec)
INFO:tensorflow:global_step/sec: 1.59423
2019-10-12 09:55:34,949 [INFO] tensorflow: global_step/sec: 1.59423
INFO:tensorflow:epoch = 0.9859813084112149, loss = 0.00204749, step = 422 (5.670 sec)
2019-10-12 09:55:36,202 [INFO] tensorflow: epoch = 0.9859813084112149, loss = 0.00204749, step = 422 (5.670 sec)
2019-10-12 09:55:37,485 [INFO] /usr/local/lib/python2.7/dist-packages/iva/detectnet_v2/tfhooks/sample_counter_hook.pyc: Samples / sec: 22.221
2019-10-12 09:55:39,515 [INFO] iva.detectnet_v2.evaluation.evaluation: step 0 / 106, 0.00s/step
/usr/local/lib/python2.7/dist-packages/iva/detectnet_v2/evaluation/metadata.py:38: UserWarning: One or more metadata field(s) are missing from ground_truth batch_data, and will be replaced with defaults: ['frame/camera_location']
2019-10-12 09:56:36,621 [INFO] iva.detectnet_v2.evaluation.evaluation: step 10 / 106, 5.71s/step
2019-10-12 09:57:28,627 [INFO] iva.detectnet_v2.evaluation.evaluation: step 20 / 106, 5.20s/step
2019-10-12 09:58:21,125 [INFO] iva.detectnet_v2.evaluation.evaluation: step 30 / 106, 5.25s/step
2019-10-12 09:59:13,356 [INFO] iva.detectnet_v2.evaluation.evaluation: step 40 / 106, 5.22s/step
2019-10-12 10:00:05,858 [INFO] iva.detectnet_v2.evaluation.evaluation: step 50 / 106, 5.25s/step
2019-10-12 10:00:57,414 [INFO] iva.detectnet_v2.evaluation.evaluation: step 60 / 106, 5.16s/step
2019-10-12 10:01:49,237 [INFO] iva.detectnet_v2.evaluation.evaluation: step 70 / 106, 5.18s/step
2019-10-12 10:02:41,832 [INFO] iva.detectnet_v2.evaluation.evaluation: step 80 / 106, 5.26s/step
2019-10-12 10:03:34,250 [INFO] iva.detectnet_v2.evaluation.evaluation: step 90 / 106, 5.24s/step
2019-10-12 10:04:26,300 [INFO] iva.detectnet_v2.evaluation.evaluation: step 100 / 106, 5.21s/step
Traceback (most recent call last):
File "/usr/local/bin/tlt-train-g1", line 10, in <module>
sys.exit(main())
File "./common/magnet_train.py", line 37, in main
File "</usr/local/lib/python2.7/dist-packages/decorator.pyc:decorator-gen-2>", line 2, in main
File "./detectnet_v2/utilities/timer.py", line 46, in wrapped_fn
File "./detectnet_v2/scripts/train.py", line 632, in main
File "./detectnet_v2/scripts/train.py", line 556, in run_experiment
File "./detectnet_v2/scripts/train.py", line 490, in train_gridbox
File "./detectnet_v2/scripts/train.py", line 136, in run_training_loop
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/training/monitored_session.py", line 676, in run
run_metadata=run_metadata)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/training/monitored_session.py", line 1270, in run
raise six.reraise(*original_exc_info)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/training/monitored_session.py", line 1255, in run
return self._sess.run(*args, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/training/monitored_session.py", line 1335, in run
run_metadata=run_metadata))
File "./detectnet_v2/tfhooks/validation_hook.py", line 69, in after_run
File "./detectnet_v2/tfhooks/validation_hook.py", line 75, in validate
File "./detectnet_v2/evaluation/evaluation.py", line 179, in evaluate
File "./detectnet_v2/evaluation/compute_metrics.py", line 161, in __init__
File "./detectnet_v2/evaluation/compute_metrics.py", line 331, in _prepare_internal_structures
File "./detectnet_v2/evaluation/compute_metrics.py", line 301, in _check_if_bbox_is_valid
KeyError: u'cyclist'