Semantic Segmentation in DS 4

Hi,

I am running the deepstream segmenation application in DS 4.0. I would like to write the output to a file and visualize it. Are there any examples or code snippets to view the output segmentation results for this example?

-SN

What’s platform you used, you should observe the result from the output screen when you running the app.

I am running the application in a docker on a remote server (Tesla V100). It is ideal to write the output to a file. I have set the properties as: g_object_set (G_OBJECT (sink), “location”, “segmentation-out.mjpeg”, NULL); Any thoughts?

If you set the output sink to a file, then you can play the file to see the result.

BTW, you can add dump in tiler_src_pad_buffer_probe, paste the code for your reference.

#ifdef DUMP_JPG
static unsigned char class2BGR[] = {
    0, 0, 0,        0, 0, 128,      0, 128, 0,
    0, 128, 128,    128, 0, 0,      128, 0, 128,
    128, 128, 0,    128, 128, 128,  0, 0, 64,
    0, 0, 192,      0, 128, 64,     0, 128, 192,
    128, 0, 64,     128, 0, 192,    128, 128, 64,
    128, 128, 192,  0, 64, 0,       0, 64, 128,
    0, 192, 0,     0, 192, 128,    128, 64, 0,
    192, 192, 0
};

static cv::Mat overlayColor(int* mask, int height, int width) {
    unsigned char* buffer_R = reinterpret_cast<unsigned char*>(malloc(sizeof(unsigned char) * height * width));
    unsigned char* buffer_G = reinterpret_cast<unsigned char*>(malloc(sizeof(unsigned char) * height * width));
    unsigned char* buffer_B = reinterpret_cast<unsigned char*>(malloc(sizeof(unsigned char) * height * width));
    unsigned char* buffer_A = reinterpret_cast<unsigned char*>(malloc(sizeof(unsigned char) * height * width));

    g_print("buffer size = %d\n", (height * width));

    for(int pix_id = 0; pix_id < width * height; pix_id++) {
        g_print("mask[pix_id] = %d\n", mask[pix_id]);
        unsigned char* color = class2BGR + ((mask[pix_id] + 3)* 3);
        buffer_B[pix_id] = color[0];
        buffer_G[pix_id] = color[1];
        buffer_R[pix_id] = color[2];
        buffer_A[pix_id] = 255;
    }
    std::vector<cv::Mat> channels;
    channels.push_back(cv::Mat(height, width, CV_8UC1, buffer_B));
    channels.push_back(cv::Mat(height, width, CV_8UC1, buffer_G));
    channels.push_back(cv::Mat(height, width, CV_8UC1, buffer_R));
    channels.push_back(cv::Mat(height, width, CV_8UC1, buffer_A));
    cv::Mat merged;
    cv::merge(channels, merged);

    free(buffer_B);
    free(buffer_G);
    free(buffer_R);

    return merged;
}

static void dump_jpg(char* src_data, int src_width, int src_height, NvDsInferSegmentationMeta* seg_meta, int source_id, int frame_number) {
    char file_name[128];

    cv::Mat map_mat = overlayColor((int*)seg_meta->class_map, seg_meta->height, seg_meta->width);
    sprintf(file_name, "dump_map_stream%2d_frame%03d.jpg", source_id, frame_number);
    cv::Mat map_resized;
    cv::resize(map_mat, map_resized, cv::Size(src_width, src_height));
    cv::imwrite(file_name, map_resized);

    // NV12 source data
    cv::Mat src_mat(src_height *3/2, src_width, CV_8UC1, (void*)src_data);
    cv::Mat src_mat_BGRA;
    cv::cvtColor(src_mat, src_mat_BGRA, CV_YUV2BGRA_NV12);
    sprintf(file_name, "dump_orig_stream%2d_frame%03d.jpg", source_id, frame_number);
    cv::imwrite(file_name, src_mat_BGRA);

    cv::Mat dst_mat;

    float alpha = 0.5;
    float beta = 1.0 - alpha;
    cv::addWeighted(src_mat_BGRA , alpha, map_resized, beta, 0.0, dst_mat);
    sprintf(file_name, "dump_merged_stream%2d_%03d.jpg", source_id, frame_number);
    cv::imwrite(file_name, dst_mat);
}
#endif

/* tiler_sink_pad_buffer_probe  will extract metadata received on segmentation
 *  src pad */
static GstPadProbeReturn
tiler_src_pad_buffer_probe (GstPad * pad, GstPadProbeInfo * info,
    gpointer u_data)
{
#ifdef DUMP_JPG
    GstBuffer *buf = (GstBuffer *) info->data;
    NvDsMetaList * l_frame = NULL;
    NvDsMetaList * l_user_meta = NULL;
    NvDsUserMeta *user_meta = NULL;
    NvDsInferSegmentationMeta* seg_meta_data = NULL;
    // Get original raw data
    GstMapInfo in_map_info;
    char* src_data = NULL;
    if (!gst_buffer_map (buf, &in_map_info, GST_MAP_READ)) {
        g_print ("Error: Failed to map gst buffer\n");
        gst_buffer_unmap (buf, &in_map_info);
        return GST_PAD_PROBE_OK;
    }
    NvBufSurface *surface = (NvBufSurface *)in_map_info.data;

    NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta (buf);

    for (l_frame = batch_meta->frame_meta_list; l_frame != NULL;
      l_frame = l_frame->next) {
        NvDsFrameMeta *frame_meta = (NvDsFrameMeta *) (l_frame->data);
        /* Validate user meta */
        for (l_user_meta = frame_meta->frame_user_meta_list; l_user_meta != NULL;
            l_user_meta = l_user_meta->next) {
            user_meta = (NvDsUserMeta *) (l_user_meta->data);
            if (user_meta && user_meta->base_meta.meta_type == NVDSINFER_SEGMENTATION_META) {
                seg_meta_data = (NvDsInferSegmentationMeta*)user_meta->user_meta_data;
            }
        }

        src_data = (char*) malloc(surface->surfaceList[frame_meta->batch_id].dataSize);
        if(src_data == NULL) {
            g_print("Error: failed to malloc src_data \n");
            continue;
        }
        cudaMemcpy((void*)src_data,
                   (void*)surface->surfaceList[frame_meta->batch_id].dataPtr,
                   surface->surfaceList[frame_meta->batch_id].dataSize,
                   cudaMemcpyDeviceToHost);
        dump_jpg(src_data,
                 surface->surfaceList[frame_meta->batch_id].width,
                 surface->surfaceList[frame_meta->batch_id].height,
                 seg_meta_data, frame_meta->source_id, frame_meta->frame_num);

        if(src_data != NULL) {
            free(src_data);
            src_data = NULL;
        }
    }
    gst_buffer_unmap (buf, &in_map_info);
#endif
    return GST_PAD_PROBE_OK;
}

Thank you, I will try this out and post an update.

Hi bcao,

I am trying to use your advice given above. I tried using opencv along with deepstream-app. Please find the Makefile below :

APP:= deepstream-app

#TARGET_DEVICE = $(shell gcc -dumpmachine | cut -f1 -d -)
TARGET_DEVICE = $(shell g++ -dumpmachine | cut -f1 -d -)

NVDS_VERSION:=4.0

LIB_INSTALL_DIR?=/opt/nvidia/deepstream/deepstream-$(NVDS_VERSION)/lib/

ifeq ($(TARGET_DEVICE),aarch64)
  CFLAGS:= -DPLATFORM_TEGRA
endif

CC := gcc
CXX := g++

SRCS:= $(wildcard *.c)
SRCS+= $(wildcard ../../apps-common/src/*.c)

SRCS_CXX:= $(wildcard *.cpp)
INCS:= $(wildcard *.h)

PKGS:= gstreamer-1.0 gstreamer-video-1.0 x11

OBJS:= $(SRCS:.c=.o)
OBJS_CXX:= $(SRCS:.cpp=.co)

CFLAGS+= -I../../apps-common/includes -I../../../includes  -I/usr/src/nvidia/tegra_multimedia_api/include/ -I/usr/local/cuda-10.0/targets/aarch64-linux/include/ -DDS_VERSION_MINOR=0 -DDS_VERSION_MAJOR=4

LIBS+= -L$(LIB_INSTALL_DIR)  -L/usr/local/cuda-10.0/targets/aarch64-linux/lib -lnppicc -lnvdsgst_meta -lnvds_meta -lnvdsgst_helper -lnvds_utils -lm -lcurl \
       -lgstrtspserver-1.0 -Wl,-rpath,$(LIB_INSTALL_DIR)

CFLAGS+= `pkg-config --cflags $(PKGS)`

LIBS+= `pkg-config --libs $(PKGS)`

all: $(APP)

%.o: %.c $(INCS) Makefile
	$(CC) -c -o $@ $(CFLAGS) $<

%.co: %.cpp $(INCS) Makefile
	$(CXX) -c -o $@ $(CFLAGS) $<

$(APP): $(OBJS_CXX) $(OBJS)  Makefile
	$(CXX) -o $(APP) $(OBJS_CXX) $(OBJS)  $(LIBS)

clean:
	rm -rf $(OBJS) $(APP)

However, I get the following error when I build the application:

deepstream_app.c:23:10: fatal error: gst/gst.h: No such file or directory
 #include <gst/gst.h>
          ^~~~~~~~~~~
compilation terminated.
In file included from deepstream_app_main.c:23:0:
deepstream_app.h:31:10: fatal error: gst/gst.h: No such file or directory
 #include <gst/gst.h>
          ^~~~~~~~~~~
compilation terminated.
In file included from deepstream_app_config_parser.c:23:0:
deepstream_app.h:31:10: fatal error: gst/gst.h: No such file or directory
 #include <gst/gst.h>
          ^~~~~~~~~~~
compilation terminated.
../../apps-common/src/deepstream_tracker_bin.c:23:10: fatal error: deepstream_common.h: No such file or directory
 #include "deepstream_common.h"
          ^~~~~~~~~~~~~~~~~~~~~
compilation terminated.
../../apps-common/src/deepstream_primary_gie_bin.c:26:10: fatal error: deepstream_common.h: No such file or directory
 #include "deepstream_common.h"
          ^~~~~~~~~~~~~~~~~~~~~
compilation terminated.
../../apps-common/src/deepstream_source_bin.c:25:10: fatal error: gstnvdsmeta.h: No such file or directory
 #include "gstnvdsmeta.h"
          ^~~~~~~~~~~~~~~
compilation terminated.
../../apps-common/src/deepstream_config_file_parser.c:23:10: fatal error: deepstream_common.h: No such file or directory
 #include "deepstream_common.h"
          ^~~~~~~~~~~~~~~~~~~~~
compilation terminated.
../../apps-common/src/deepstream_common.c:23:10: fatal error: gst/gst.h: No such file or directory
 #include <gst/gst.h>
          ^~~~~~~~~~~
compilation terminated.
../../apps-common/src/deepstream_sink_bin.c:30:10: fatal error: deepstream_common.h: No such file or directory
 #include "deepstream_common.h"
          ^~~~~~~~~~~~~~~~~~~~~
compilation terminated.
../../apps-common/src/deepstream_perf.c:29:10: fatal error: gstnvdsmeta.h: No such file or directory
 #include "gstnvdsmeta.h"
          ^~~~~~~~~~~~~~~
compilation terminated.
../../apps-common/src/deepstream_dewarper_bin.c:23:10: fatal error: deepstream_common.h: No such file or directory
 #include "deepstream_common.h"
          ^~~~~~~~~~~~~~~~~~~~~
compilation terminated.
../../apps-common/src/deepstream_dsexample.c:23:10: fatal error: deepstream_common.h: No such file or directory
 #include "deepstream_common.h"
          ^~~~~~~~~~~~~~~~~~~~~
compilation terminated.
../../apps-common/src/deepstream_secondary_gie_bin.c:26:10: fatal error: deepstream_common.h: No such file or directory
 #include "deepstream_common.h"
          ^~~~~~~~~~~~~~~~~~~~~
compilation terminated.
../../apps-common/src/deepstream_tiled_display_bin.c:23:10: fatal error: deepstream_common.h: No such file or directory
 #include "deepstream_common.h"
          ^~~~~~~~~~~~~~~~~~~~~
compilation terminated.
../../apps-common/src/deepstream_osd_bin.c:23:10: fatal error: deepstream_common.h: No such file or directory
 #include "deepstream_common.h"
          ^~~~~~~~~~~~~~~~~~~~~
compilation terminated.
../../apps-common/src/deepstream_streammux.c:23:10: fatal error: deepstream_common.h: No such file or directory
 #include "deepstream_common.h"
          ^~~~~~~~~~~~~~~~~~~~~
compilation terminated.
Makefile:69: recipe for target 'deepstream-app' failed

Kindly let me know how to build with opencv. If you can provide the Makefile it will be great help.

Since opencv did not work I used libjpeg to convert only the Y channel of the buffer. However, the output was almost completely black, not representative of my input.

Code:

int write_jpeg_file( char *filename, unsigned char* rgb_image , int width, int height, int bytes_per_pixel, J_COLOR_SPACE color_space )
{

	//parse_rgb_row_major(width,height,rgb_image,width*3);

	struct jpeg_compress_struct cinfo;
	struct jpeg_error_mgr jerr;
	
	JSAMPROW row_pointer[1];
	FILE *outfile = fopen( filename, "wb" );
	
	if ( !outfile )
	{
		printf("Error opening output jpeg file %s\n!", filename );
		return -1;
	}
	cinfo.err = jpeg_std_error( &jerr );
	jpeg_create_compress(&cinfo);
	jpeg_stdio_dest(&cinfo, outfile);

	cinfo.image_width = width;	
	cinfo.image_height = height;
	cinfo.input_components = bytes_per_pixel;
	cinfo.in_color_space = color_space; //JCS_RGB

	jpeg_set_defaults( &cinfo );

	jpeg_start_compress( &cinfo, TRUE );

	while( cinfo.next_scanline < cinfo.image_height )
	{
		row_pointer[0] = &rgb_image[ cinfo.next_scanline * cinfo.image_width *  cinfo.input_components];
		jpeg_write_scanlines( &cinfo, row_pointer, 1 );
	}

	jpeg_finish_compress( &cinfo );
	jpeg_destroy_compress( &cinfo );
	fclose( outfile );

	return 1;
}

/**
 * Buffer probe function after tracker.
 */
static GstPadProbeReturn
tracking_done_buf_prob (GstPad * pad, GstPadProbeInfo * info, gpointer u_data)
{
  NvDsInstanceBin *bin = (NvDsInstanceBin *) u_data;
  guint index = bin->index;
  AppCtx *appCtx = bin->appCtx;
  GstBuffer *buf = (GstBuffer *) info->data;
  NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta (buf);
  if (!batch_meta) {
    NVGSTDS_WARN_MSG_V ("Batch meta not found for buffer %p", buf);
    return GST_PAD_PROBE_OK;
  }

  GstMapInfo in_map_info;
  NvBufSurface *surface = NULL;

  memset (&in_map_info, 0, sizeof (in_map_info));
  if (!gst_buffer_map (buf, &in_map_info, GST_MAP_READ)) {
    g_print ("Error: Failed to map gst buffer\n");
  }

surface = (NvBufSurface *) in_map_info.data;  
    NvDsMetaList * l_frame = NULL;
    NvDsMetaList * l_user_meta = NULL;
    NvDsUserMeta *user_meta = NULL;

    for (l_frame = batch_meta->frame_meta_list; l_frame != NULL;
      l_frame = l_frame->next) {
        NvDsFrameMeta *frame_meta = (NvDsFrameMeta *) (l_frame->data);
        char *src_data = (char*) malloc(surface->surfaceList[frame_meta->batch_id].dataSize);
        if(src_data == NULL) {
            g_print("Error: failed to malloc src_data \n");
            continue;
        }

        cudaError_t err =cudaMemcpy((void*)src_data,
                   (void*)surface->surfaceList[frame_meta->batch_id].dataPtr,
                   surface->surfaceList[frame_meta->batch_id].dataSize,
                   cudaMemcpyDeviceToHost);

	uint32_t frame_width  =  surface->surfaceList[frame_meta->batch_id].width;
	uint32_t frame_height =  surface->surfaceList[frame_meta->batch_id].height;

	char filename[200];
	sprintf(filename,"file_y_%dx%d.jpg",frame_width,frame_height);

        write_jpeg_file( filename, src_data , frame_width, frame_height, 1, JCS_GRAYSCALE );

if(src_data != NULL) {
            free(src_data);
            src_data = NULL;
        }
    }
    gst_buffer_unmap (buf, &in_map_info);

/*
   * Output KITTI labels with tracking ID if configured to do so.
   */
  write_kitti_track_output(appCtx, batch_meta);

  if (appCtx->primary_bbox_generated_cb)
    appCtx->primary_bbox_generated_cb (appCtx, buf, batch_meta, index);
  return GST_PAD_PROBE_OK;
}

I also get cudaError_t 11

I have attached the output. Kindly let me know where I must be going wrong. I have tested the libjpeg based code and it works fine separately.

Thanks.

Which platform and DS version did you use?

Hi bcao,

I am working on Jetson Nano and with deepstream 4.0 sdk.

Thanks

You need to find the header files if make sure you had set up the environment correctly

Hi bcao,

I am working on Jetson Nano and with deepstream 4.0.1 sdk.
I am trying to use your advice given above “dump in tiler_src_pad_buffer_probe example”.
I also get seg_meta_data == NULL. Kindly let me know where I must be going wrong.

Thanks
deepstream_infer_tensor_meta_test.cpp (30.6 KB)

You need to add the probe call back to the right element(nvinfer src or the after element).