Acquire multi stream output in a single stream using CVB++

Hello all,
I am working with Jai SW4000Q line scan camera. This camera gives both RGB and NIR images. Previously I was accessing both the images as two separate streams (DS0 and DS1). Now, I would like to combine all 4 planes into one stream and read the image with 4 planes in a single stream output. I know this is possible with setting the pixel format to RGBa8 as given in the manual (image attached below).

Using genicam browser, when I set the pixel format to “RGBa8”, I can see that the amount of data received from the camera goes from 800 to 1000 MB/sec which indicates that there are 4 planes in the image.

I tried to do the same thing using CVB++ using the below code

#include <cvb/device_factory.hpp>
#include <cvb/utilities/system_info.hpp>
#include <cvb/driver/stream.hpp>
#include <cvb/global.hpp>
#include <cvb/driver/composite_stream.hpp>
#include <cvb/genapi/node_map_enumerator.hpp>
#include <cvb/driver/image_rect.hpp>

#include<chrono>
#include <time.h>
#include <stdlib.h>
#include <thread>
#include <memory>
#include<filesystem>
#include<cmath>

#define ACQUISITION_TIME 10
#define MAC_ID_LINE_SCAN "00-0C-DF-0A-6B-6F"
#define LINE_SCAN_DRIVER_TYPE "FILTER" // "FILTER" or "SOCKET"

using namespace Cvb;
using namespace std::chrono;

static std::map<Cvb::WaitStatus, const char*>
WAIT_ERROR_STATES
{
  {Cvb::WaitStatus::Timeout, "timeout"},
  {Cvb::WaitStatus::Abort, "abort"}
};


void SetIntegerNodeMapValue(Cvb::GenApi::IntegerNodePtr fromNode, int toValue) {
	fromNode->SetValue(toValue);
	std::cout << "Set " << fromNode->Name() << " to value: " << fromNode->Value() << std::endl;
}

void SetFloatNodeMapValue(Cvb::GenApi::FloatNodePtr fromNode, float toValue) {
	fromNode->SetValue(toValue);
	std::cout << "Set " << fromNode->Name() << " to value: " << fromNode->Value() << std::endl;
}

void SetFEnumNodeMapValue(Cvb::GenApi::EnumerationNodePtr fromNode, std::string toValue) {
	fromNode->SetValue(toValue);
	std::cout << "Set " << fromNode->Name() << " to value: " << fromNode->Value() << std::endl;
}


int main(int argc, char* argv[])
{
    std::string json_config_path;
    std::string save_directory_path_RGB, save_directory_path_NIR;
    
    json_config_path = argv[1];
    
    LineScanConfig line_scan_config;
    line_scan_config = parse_json(json_config_path);
    try
    {
        auto path = Cvb::InstallPath();
        path += CVB_LIT("drivers\\GenICam.vin");
    
        std::cout<<"GeniCam Driver Path : "<<path<<std::endl;
        auto flags = Cvb::DiscoverFlags::IgnoreVins | Cvb::DiscoverFlags::IgnoreGevSD;
        //const auto timeout = std::chrono::seconds(2);
        auto discovery_tokens = Cvb::DeviceFactory::Discover(flags);
        std::cout<<"Number of Devices Discovered :"<<discovery_tokens.size()<<std::endl;
        std::cout<<"Type of Discovery Tokens : "<<typeid(discovery_tokens).name()<<std::endl;
        unsigned int device_index =  0; 
        for(auto &token : discovery_tokens)
        {    
            std::string device_mac, interface_driver;

            if(token.TryGetProperty(Cvb::DiscoveryProperties::DeviceMac, device_mac) && \
            token.TryGetProperty(Cvb::DiscoveryProperties::InterfaceDriverType , interface_driver) )
            {
                std::cout<<"Found device MAC and Interface Driver : "<< device_mac<< " : " <<interface_driver<<std::endl;
                std::cout<<"Device Index : "<<device_index<<std::endl;
            }

            if(!device_mac.compare(MAC_ID_LINE_SCAN) && ! interface_driver.compare(LINE_SCAN_DRIVER_TYPE))
            {
                    std::cout<<"Opening Device : " << device_mac << " using "<<interface_driver<<" driver"<<std::endl;
                    //token.SetParameter("PixelFormat", "0");
                    //token.SetParameter("NumBuffer", "50");
                    break;
            }
    
            device_index++;
        }
        
        auto line_scan_device = Cvb::DeviceFactory::Open(discovery_tokens[device_index].AccessToken(), Cvb::AcquisitionStack::GenTL);

        auto device_node_map = line_scan_device->NodeMap(CVB_LIT("Device"));
        auto old_width = device_node_map->Node<IntegerNode>("Std::Width"); 
        auto old_height = device_node_map->Node<IntegerNode>("Std::Height"); 
        auto frame_rate = device_node_map->Node<FloatNode>("Std::AcquisitionFrameRate"); 
        auto pixelFormat = device_node_map->Node<EnumerationNode>("Std::PixelFormat");
        auto componentSelector = device_node_map->Node<EnumerationNode>("Std::ComponentSelector");
        auto line_rate = device_node_map->Node<FloatNode>("Std::AcquisitionLineRate");
        
        SetIntegerNodeMapValue(device_node_map->Node<IntegerNode>("Std::Width"), line_scan_config.Parameters.image_width);
        SetIntegerNodeMapValue(device_node_map->Node<IntegerNode>("Std::Height"), line_scan_config.Parameters.image_height);
        SetFEnumNodeMapValue(device_node_map->Node<EnumerationNode>("Std::PixelFormat"),"RGBa8");

        auto set_width = device_node_map->Node<IntegerNode>("Std::Width"); 
        auto set_height = device_node_map->Node<IntegerNode>("Std::Height"); 
        std::cout<<"Set Width : " <<set_width->Value() <<std::endl;
        std::cout<<"Set Height : " <<set_height->Value() <<std::endl;
        std::cout<<"Changed Frame Rate : "<<device_node_map->Node<FloatNode>("Std::AcquisitionFrameRate")->Value()<<std::endl;
        std::cout<<"Changed Pixel Format : "<<device_node_map->Node<EnumerationNode>("Std::PixelFormat")->Value()<<std::endl;


        auto image_stream = line_scan_device->Stream<Cvb::ImageStream>(0);
        image_stream->Start();
        auto start = high_resolution_clock::now();
        static const constexpr auto TIMEOUT = std::chrono::milliseconds(3000);
        unsigned int i = 0;
        while(duration_cast<seconds>(high_resolution_clock::now() - start).count() < ACQUISITION_TIME)
        {
            Cvb::MultiPartImagePtr multiPartImage;
            Cvb::WaitStatus waitStatus;
            Cvb::NodeMapEnumerator enumerator;
            std::tie(multiPartImage, waitStatus, enumerator) = image_stream->WaitFor(TIMEOUT);
            switch (waitStatus)
            {
                default:
                    std::cout << "unknown wait status\n";
                case Cvb::WaitStatus::Abort:
                case Cvb::WaitStatus::Timeout:
                {
                    std::cout << "wait status not ok: " << WAIT_ERROR_STATES[waitStatus] << "\n";;
                    continue;
                }
                case Cvb::WaitStatus::Ok: break;
            }
        
            auto element1 = multiPartImage->GetPartAt(0);
            if (!Cvb::holds_alternative<Cvb::ImagePtr>(element1))
            {
                std::cout << "multi part image does not contain an image as first element\n";
                continue;
            }
            auto image = Cvb::get<Cvb::ImagePtr>(element1);
   
            auto linearAccess = image->Plane(0).LinearAccess();
            std::cout<<"Number of Planes in the image : " <<image->PlanesCount() <<std::endl;
            std::cout << "Acquired image: #" << i << ", at address " << linearAccess.BasePtr() << "\n";
            i ++ ;
        }

        image_stream->Stop();
    }
                
    catch(const std::exception& error)
    {
        std::cout<<" Camera config : "<<error.what()<<std::endl;

    }

    return 0;
}

Now, using the above code I tried to print out the number of planes received by the image and it says 3 instead of 4. Is there a way to get all 4 planes in a single output stream ?

Hi,

You should set the pixel format acquisition stack explicitly before opening the device:

discovery_tokens[device_index].SetParameter("PixelFormat", "0");

more details can be found here:
https://help.commonvisionblox.com/NextGen/14.0/cvbpp/db/db5/class_cvb_1_1_driver_1_1_discovery_information.html#a0174ff782739a2fc3faefeae3b12f1ac

Pixel format of the CVB Image
0 = Raw image what ever the device delivers
1 = Mono 8Bit
2 = RGB 8Bit
3 = Mono 16Bit
4 = RGB 16Bit
5 = Auto

Hi @MandanaS

I tried changing the pixel format to raw format and still the images planes are only 3 instead of 4. Is there a different way to get all 4 planes together

@keerthitheja Would you please send us a few lines above and below setting the pixel format?

Hi @MandanaS,
Here is the complete code, I am using

Discovering the device :


        auto flags = Cvb::DiscoverFlags::IgnoreVins | Cvb::DiscoverFlags::IgnoreGevSD;
        //const auto timeout = std::chrono::seconds(2);
        auto discovery_tokens = Cvb::DeviceFactory::Discover(flags);
        std::cout<<"Number of Devices Discovered :"<<discovery_tokens.size()<<std::endl;
        std::cout<<"Type of Discovery Tokens : "<<typeid(discovery_tokens).name()<<std::endl;
        unsigned int device_index =  0; 
        for(auto &token : discovery_tokens)
        {    
            std::string device_mac, interface_driver;

            if(token.TryGetProperty(Cvb::DiscoveryProperties::DeviceMac, device_mac) && \
            token.TryGetProperty(Cvb::DiscoveryProperties::InterfaceDriverType , interface_driver) )
            {
                std::cout<<"Found device MAC and Interface Driver : "<< device_mac<< " : " <<interface_driver<<std::endl;
                std::cout<<"Device Index : "<<device_index<<std::endl;
            }

            if(!device_mac.compare(MAC_ID_LINE_SCAN) && ! interface_driver.compare(LINE_SCAN_DRIVER_TYPE))
            {
                    std::cout<<"Opening Device : " << device_mac << " using "<<interface_driver<<" driver"<<std::endl;
                    break;
            }
    
            device_index++;
        }

Setting Pixel Format and streaming:

        discovery_tokens[device_index].SetParameter("PixelFormat", "0");
        auto line_scan_device = Cvb::DeviceFactory::Open(discovery_tokens[device_index].AccessToken(), Cvb::AcquisitionStack::GenTL);

        auto device_node_map = line_scan_device->NodeMap(CVB_LIT("Device"));
        auto old_width = device_node_map->Node<IntegerNode>("Std::Width"); 
        auto old_height = device_node_map->Node<IntegerNode>("Std::Height"); 
        auto frame_rate = device_node_map->Node<FloatNode>("Std::AcquisitionFrameRate"); 
        auto pixelFormat = device_node_map->Node<EnumerationNode>("Std::PixelFormat");
        auto componentSelector = device_node_map->Node<EnumerationNode>("Std::ComponentSelector");
        auto line_rate = device_node_map->Node<FloatNode>("Std::AcquisitionLineRate");
        
        SetIntegerNodeMapValue(device_node_map->Node<IntegerNode>("Std::Width"), line_scan_config.Parameters.image_width);
        SetIntegerNodeMapValue(device_node_map->Node<IntegerNode>("Std::Height"), line_scan_config.Parameters.image_height);
        SetFEnumNodeMapValue(device_node_map->Node<EnumerationNode>("Std::PixelFormat"),"RGBa8");

        auto set_width = device_node_map->Node<IntegerNode>("Std::Width"); 
        auto set_height = device_node_map->Node<IntegerNode>("Std::Height"); 
        std::cout<<"Set Width : " <<set_width->Value() <<std::endl;
        std::cout<<"Set Height : " <<set_height->Value() <<std::endl;
        std::cout<<"Changed Frame Rate : "<<device_node_map->Node<FloatNode>("Std::AcquisitionFrameRate")->Value()<<std::endl;
        std::cout<<"Changed Pixel Format : "<<device_node_map->Node<EnumerationNode>("Std::PixelFormat")->Value()<<std::endl;


        auto image_stream = line_scan_device->Stream<Cvb::ImageStream>(0);
        image_stream->Start();
        auto start = high_resolution_clock::now();
        static const constexpr auto TIMEOUT = std::chrono::milliseconds(3000);
        unsigned int i = 0;
        while(duration_cast<seconds>(high_resolution_clock::now() - start).count() < ACQUISITION_TIME)
        {
            Cvb::MultiPartImagePtr multiPartImage;
            Cvb::WaitStatus waitStatus;
            Cvb::NodeMapEnumerator enumerator;
            std::tie(multiPartImage, waitStatus, enumerator) = image_stream->WaitFor(TIMEOUT);
            switch (waitStatus)
            {
                default:
                    std::cout << "unknown wait status\n";
                case Cvb::WaitStatus::Abort:
                case Cvb::WaitStatus::Timeout:
                {
                    std::cout << "wait status not ok: " << WAIT_ERROR_STATES[waitStatus] << "\n";;
                    continue;
                }
                case Cvb::WaitStatus::Ok: break;
            }
        
            auto element1 = multiPartImage->GetPartAt(0);
            if (!Cvb::holds_alternative<Cvb::ImagePtr>(element1))
            {
                std::cout << "multi part image does not contain an image as first element\n";
                continue;
            }
            auto image = Cvb::get<Cvb::ImagePtr>(element1);
   
            auto linearAccess = image->Plane(0).LinearAccess();
            std::cout<<"Number of Planes in the image : " <<image->PlanesCount() <<std::endl;
            std::cout << "Acquired image: #" << i << ", at address " << linearAccess.BasePtr() << "\n";
            i ++ ;
        }

        image_stream->Stop();
    }
                
    catch(const std::exception& error)
    {
        std::cout<<" Camera config : "<<error.what()<<std::endl;

    }

    return 0;
}

I am setting the pixel format to raw just before opening the camera.

Hi @keerthitheja ,

could you do me a favour and check what value you have for the CVB Color Format in the GenICam Browser when you right click the configured device and look into the settings.

If this is set to RAW and everything works, including changing the PixelFormat in the camera, we might have to take a closer look into what is going on there.

Cheers
Chris

Hi @Chris

I checked the color format in the GeniCam Browser and it is set to Raw. I again ran the same code and it still gives me 3 planes.

1 Like

@c.hartmann I`m a bit puzzled here.
Setting the colorformat to RAW in GCBrowser and then changing it in the NodeMap of the camera seems to work.
Doing the same in code does not work.
Are we overseeing something here?

Hi @Chris , @keerthitheja

Hm … this is very confusing.

from our internal code:
switch (pfncCode)
{
case RGBa8:
// … many more formats.
return 4;
}
So this seems to be correct.

In the logging for the genicam.vin, there should be a message about “MakeCompositeFromSingleImageBuffer” (on trace level).
This should (partially) describe what is going on.

Could you send this?

AND send me the number of the Std::Pixelformat:

Hi @c.hartmann

Here you go, the numbers for Std::PixelFormat
pixel_format

Also Where can i find the genicam.vin logging ?

Hi @keerthitheja

the number for the pixelformat is correct, that’s good.

The logging is available via the “LogGui”, like i said the message is dropped on trace.

https://help.commonvisionblox.com/NextGen/14.0/md_theory_of_operation_apps__log_g_u_i.html

Also i really expected the correct “4” to appear. This may be a bug.

Hi @c.hartmann

I configured the log GUI, but strangely it doesn’t log anything. I do not see any messages


I started logging from geni cam browser. You can see from the configuration that I had added geni cam browser with trace level.

Is it possible that the CVB package installed in my PC is broken ?

Regarding the Logging:
Did you restart the GenICambrowser? (The log config is only read at loading our libs/applications).

Regarding your initial problem:
Could you check the value of linearAccess.XInc()?

Hi @c.hartmann

The returns a value of 4.
Also, I was able to log the messages. Previously I had the genicam browser open and after closing and opening the browser, the messages started appearing on the log GUI.

But, I do not see the message.

Here are the configurations, I am logging currently

Ok. Logging aside, I think the heart of the problem here is that :cvb: does not really have a handling for alpha planes. Looking at @keerthitheja’s first post, the NIR bytes get interleaved with the BGR bytes and the result is designated as RGBa8. In other words: The NIR information gets stored in an alpha channel. As a result, 4 bytes per pixel are transmitted and received by the GenICam.vin. These are then mapped to :cvb:'s concept of image planes and image access (by building a description of width, height, # of planes and the necessary access data). But as :cvb: simply doesn’t know what to do with an alpha channel this one is silently ignored and becomes invisble.

The good news is that the data is most likely there - wedged between the R bytes and the B bytes. Judging by the image you helpfully supplied, I would assume that

  • XInc is 4 (this has already been confirmed now)
  • YInc is >= 4 * width (>= instead of > because padding bytes might be present - unlikely, but still… - at line ends)
  • <base pointer of plane 0> == <base pointer of plane 1> + 1
  • <base pointer of plane 1> == <base pointer of plane 2> + 1

If we can confirm bullet points 2 to 4 then I believe we can “cheat” and construct an image description that does have 4 planes (R, G, B, nir) that can be visualized and processed. But we aware that e. g. a :cvb: tool that expects RGB will usually not be able to directly digest an image with 4 planes.

The code for this is a bit messy as it mixes use of CVB++ with use of the C-API. “Don’t do this at home” is probably the advice that should go with this:

Cvb::ImagePtr AccessNirPlane(const std::shared_ptr<Cvb::Image>& src)
{
  auto planes = src->Planes();
  std::vector<Cvb::LinearAccessData> linAcc;
  linAcc.reserve(planes.size());
  std::for_each(planes.begin(), planes.end(), [&linAcc](auto& plane) { linAcc.push_back(plane.LinearAccess()); });

  // probe whether prerequisites are met
  // note that this is not fool-proof! the main prerequisite - that there
  // actually is one plane of data preceding the B plane in memory - cannot
  // be verified and if that expectation is not met, thi will result in
  // UNDEFIND BEHAVIOR!
  if (linAcc.size() != 3)
    throw std::runtime_error("expected 3 plane image");
  std::for_each(linAcc.begin(), linAcc.end(), [](auto acc)
    {
      if (acc.XInc() != 4)
        throw std::runtime_error("x increment does not hint at a hidden channel");
      if (acc.YInc() < acc.Width() * 4)
        throw std::runtime_error("y increment inconsistent with a hidden channel");
    });
  for (auto i = 1; i < linAcc.size(); ++i)
  {
    if (linAcc[1].Width() != linAcc[0].Width())
      throw std::runtime_error("image width differs between planes");
    if (linAcc[1].YInc() != linAcc[0].YInc())
      throw std::runtime_error("y increment differs between planes");
    if (linAcc[i - 1].BasePtr() - 1 != linAcc[i].BasePtr())
      throw std::runtime_error("base pointers do not have the expected order");
  }

  // create a new image that makes the hidden data visible
  Cvb::CExports::IMG tmpHandle = nullptr;
  Cvb::CExports::cvbval_t planeOrder[] = { 3, 2, 1, 0 };
  Cvb::CExports::CreateImageFromPointer(linAcc[2].BasePtr() - 1, src->Width() * src->Height() * 4,
    src->Width(), src->Height(), 4, 8, linAcc[0].XInc(), linAcc[0].YInc(), 1, planeOrder,
    ReleaseNirImage, src->Handle(),
    tmpHandle);
  if (tmpHandle)
    Cvb::CExports::ShareObject(src->Handle());

  // create unique ptr and return
  return Cvb::Image::FromHandle(Cvb::HandleGuard<Cvb::Image>(tmpHandle));
}

Lifetime handling of the returned image must absolutely be tied to that of the input image src, which is why we call ShareObject on the source image once we have created the NIR-enabled view to the image. The function to sever the bond between the new view and its source is fairly simple:

void __stdcall ReleaseNirImage(void*, void* parentImg)
{
  Cvb::CExports::ReleaseObject(parentImg);
}

That should do the trick. As I do not have your camera my capacity for verifying this is somewhat limited, so I tested this with a dummy image modeled the way I understand your RGBnir Image looks. The code for generating my test data was:

  // construct a buffer to work with and fill it with data
  constexpr int width = 1920;
  constexpr int height = 1080;
  std::vector<uint8_t> pixels(width * height * 4);
  intptr_t xInc = 4;
  intptr_t yInc = width * 4;
  std::for_each(pixels.begin(), pixels.end(), [](uint8_t& p)
    {
      static int v = -1;
      p = 80 * ((++v) & 0b00000011); // just to have some easily distinguishable content
    });
  // the pixels in the nir channel are now set to 0, those in the blue
  // channel are at 80, those in the green channel at 160 and those in the
  // red channel at 240

  // build the source image around that buffer
  // (this image assumes the role of your RGBnir camera output)
  using namespace Cvb::CExports;
  Cvb::CExports::cvbval_t planeOrder[] = { 2, 1, 0 };
  Cvb::CExports::IMG srcHandle = nullptr;
  CreateImageFromPointer(pixels.data() + 1, // +1 because we want to skip the NIR bytes
    pixels.size(), width, height, 3,        // 3 because we want to emulate what happens to RGBa
    8,                                      // this means 8 bits per pixel
    xInc, yInc, 1, planeOrder,              // planeOrder array inverts RGB <-> BGR
    nullptr, nullptr, srcHandle);           // normally we would supply a release callback here...
                                            // but in this sample/test we won't worry about lifetime
  auto srcImg = Cvb::Image::FromHandle(Cvb::HandleGuard<Cvb::Image>(srcHandle));
  srcHandle = nullptr;
1 Like