跳到主要内容

5. 快速入门k230 AI推理流程

本章教学视频:勘智K230开发板使用教程-快速入门AI推理流程_哔哩哔哩_bilibili

本章介绍了K230 AI推理的完整流程,让大家对基于K230的AI推理过程有个大概印象,它包括视频采集,图像预处理,模型推理、后处理、显示等过程。

  • 视频输入:VI(Video Input),视频采集
  • 视频输出:VO(Video Output),显示

本章包含了K230 AI推理流程的两种实现:基于OpenCV(c++)的AI推理、基于ulab(MicroPython)的AI推理,分别对应两个子章节,具体实现详子章节介绍。

详细代码实现k230_AI_Demo_Code_Flow_Introduction

5.1 基于OpenCV(C++)的AI推理流程

基于OpenCV(C++)的K230 AI推理,简要介绍了使用c++语言实现的视频采集,图像预处理(OpenCV),模型推理、后处理、显示等过程。

5.1.1 视频采集

视频采集:(视频输入,VI)与摄像头相关,本小节简要介绍基于c++的摄像头设置、摄像头启动、从摄像头中获取一帧数据、摄像头停止的整体流程;详细介绍见K230_VICAP_API参考.mdK230_VICAP_SENSOR_参数分区参考.mdK230_Camera_Sensor适配指南.md

1. 设置摄像头属性

  • 设置的sensor两路输出;
  • 一路输出用于显示,输出大小设置1080p,图像格式为PIXEL_FORMAT_YVU_PLANAR_420,直接绑定到vo;
  • 另一路输出用于AI计算,输出大小720p,图像格式为PIXEL_FORMAT_BGR_888_PLANAR(实际为rgb,chw,uint8);

创建摄像头(sensor)输出缓存(VB):用于存放摄像头两路输出

//vi_vo.h
/***************************unfixed:不同AI Demo可能需要修改***********************/
#define SENSOR_CHANNEL (3) // 通道数
#define SENSOR_HEIGHT (720) // sensor ch1输出高度,AI输入
#define SENSOR_WIDTH (1280) // sensor ch1输出宽度,AI输入
#define ISP_CHN0_WIDTH (1920) // sensor ch0输出宽度,vo
#define ISP_CHN0_HEIGHT (1080) // sensor ch0输出高度,vo
/*****************************************************************************/

/***************************fixed:无需修改***********************************/
memset(&config, 0, sizeof(config));
config.max_pool_cnt = 64;
//VB for YUV420SP output
config.comm_pool[0].blk_cnt = 5;
config.comm_pool[0].mode = VB_REMAP_MODE_NOCACHE;
config.comm_pool[0].blk_size = VICAP_ALIGN_UP((ISP_CHN0_WIDTH * ISP_CHN0_HEIGHT * 3 / 2), VICAP_ALIGN_1K);

//VB for RGB888 output
config.comm_pool[1].blk_cnt = 5;
config.comm_pool[1].mode = VB_REMAP_MODE_NOCACHE;
config.comm_pool[1].blk_size = VICAP_ALIGN_UP((SENSOR_HEIGHT * SENSOR_WIDTH * 3 ), VICAP_ALIGN_1K);

ret = kd_mpi_vb_set_config(&config);
if (ret) {
printf("vb_set_config failed ret:%d\n", ret);
return ret;
}
/*****************************************************************************/

设置摄像头属性:设置sensor_type;一般无需换摄像头,无需修改。

//vi_vo.h
/***************************fixed:无需修改***********************************/
vicap_dev = VICAP_DEV_ID_0;
ret = kd_mpi_vicap_get_sensor_info(sensor_type, &sensor_info);
if (ret) {
printf("sample_vicap, the sensor type not supported!\n");
return ret;
}

......

dev_attr.cpature_frame = 0;
memcpy(&dev_attr.sensor_info, &sensor_info, sizeof(k_vicap_sensor_info));

ret = kd_mpi_vicap_set_dev_attr(vicap_dev, dev_attr);
if (ret) {
printf("sample_vicap, kd_mpi_vicap_set_dev_attr failed.\n");
return ret;
}
/*****************************************************************************/

设置摄像头通道0属性:设置摄像头通道0分辨率为1080p,格式为PIXEL_FORMAT_YVU_PLANAR_420;并将摄像头通道0绑定到显示;一般只需要关注chn_attr.out_win.widthchn_attr.out_win.heightchn_attr.pix_format即可,其它不用修改。

//vi_vo.h
/***************************unfixed:不同AI Demo可能需要修改***********************/
#define ISP_CHN0_WIDTH (1920) // sensor ch0输出宽度,vo
#define ISP_CHN0_HEIGHT (1080) // sensor ch0输出高度,vo
/*****************************************************************************/

/***************************unfixed:不同AI Demo可能需要修改******************/
//set chn0 output yuv420sp
......
chn_attr.out_win.width = ISP_CHN0_WIDTH;
chn_attr.out_win.height = ISP_CHN0_HEIGHT;
......
chn_attr.chn_enable = K_TRUE;
chn_attr.pix_format = PIXEL_FORMAT_YVU_PLANAR_420;
/*****************************************************************************/
......
/***************************fixed:无需修改***********************************/
chn_attr.buffer_num = VICAP_MAX_FRAME_COUNT; //at least 3 buffers for isp
chn_attr.buffer_size = config.comm_pool[0].blk_size;
vicap_chn = VICAP_CHN_ID_0;

printf("sample_vicap ...kd_mpi_vicap_set_chn_attr, buffer_size[%d]\n", chn_attr.buffer_size);
ret = kd_mpi_vicap_set_chn_attr(vicap_dev, vicap_chn, chn_attr);
if (ret) {
printf("sample_vicap, kd_mpi_vicap_set_chn_attr failed.\n");
return ret;
}

//bind vicap chn 0 to vo
vicap_mpp_chn.mod_id = K_ID_VI;
vicap_mpp_chn.dev_id = vicap_dev;
vicap_mpp_chn.chn_id = vicap_chn;

vo_mpp_chn.mod_id = K_ID_VO;
vo_mpp_chn.dev_id = K_VO_DISPLAY_DEV_ID;
vo_mpp_chn.chn_id = K_VO_DISPLAY_CHN_ID1;

sample_vicap_bind_vo(vicap_mpp_chn, vo_mpp_chn);
printf("sample_vicap ...dwc_dsi_init\n");
/*****************************************************************************/

**设置摄像头通道1属性:**设置通道1分辨率为720p,格式为PIXEL_FORMAT_BGR_888_PLANAR。

//vi_vo.h
/***************************unfixed:不同AI Demo可能需要修改***********************/
#define SENSOR_CHANNEL (3) // 通道数
#define SENSOR_HEIGHT (720) // sensor ch1输出高度,AI输入
#define SENSOR_WIDTH (1280) // sensor ch1输出宽度,AI输入
/*****************************************************************************/
......
/***************************unfixed:不同AI Demo可能需要修改******************/
//set chn1 output rgb888p
....
chn_attr.out_win.width = SENSOR_WIDTH ;
chn_attr.out_win.height = SENSOR_HEIGHT;

......
chn_attr.chn_enable = K_TRUE;
chn_attr.pix_format = PIXEL_FORMAT_BGR_888_PLANAR;
/*****************************************************************************/

/***************************fixed:无需修改***********************************/
chn_attr.buffer_num = VICAP_MAX_FRAME_COUNT;//at least 3 buffers for isp
chn_attr.buffer_size = config.comm_pool[1].blk_size;

printf("sample_vicap ...kd_mpi_vicap_set_chn_attr, buffer_size[%d]\n", chn_attr.buffer_size);
ret = kd_mpi_vicap_set_chn_attr(vicap_dev, VICAP_CHN_ID_1, chn_attr);
if (ret) {
printf("sample_vicap, kd_mpi_vicap_set_chn_attr failed.\n");
return ret;
}
/*****************************************************************************/

设置初始化、并启动摄像头:

//vi_vo.h
/***************************fixed:无需修改***********************************/
ret = kd_mpi_vicap_init(vicap_dev);
if (ret) {
printf("sample_vicap, kd_mpi_vicap_init failed.\n");
// goto err_exit;
}

printf("sample_vicap ...kd_mpi_vicap_start_stream\n");
ret = kd_mpi_vicap_start_stream(vicap_dev);
if (ret) {
printf("sample_vicap, kd_mpi_vicap_init failed.\n");
// goto err_exit;
}
/*****************************************************************************/

使用示例

//main.cc
vivcap_start();

2. 获取摄像头图像

摄像头数据临时地址:创建vaddr,临时存放摄像头最新数据,它与摄像头通道1大小相同。

// main.cc
/***************************fixed:无需修改***********************************/
// alloc memory for sensor
size_t paddr = 0;
void *vaddr = nullptr;
size_t size = SENSOR_CHANNEL * SENSOR_HEIGHT * SENSOR_WIDTH;
int ret = kd_mpi_sys_mmz_alloc_cached(&paddr, &vaddr, "allocate", "anonymous", size);
if (ret)
{
std::cerr << "physical_memory_block::allocate failed: ret = " << ret << ", errno = " << strerror(errno) << std::endl;
std::abort();
}
/*****************************************************************************/

读取最新帧

  • dump:从摄像头通道1读取一帧图像,即从VB中dump一帧数据到dump_info
  • 映射:将dump_info对应DDR地址(物理地址)映射到当前系统地址(虚拟地址)进行访问
  • 转cv::Mat:将摄像头数据转换为cv::Mat,sensor(rgb,chw)->cv::Mat(bgr,hwc);将摄像头数据转换为为cv::Mat不是必须的,这里只是为了以大家比较熟悉的方式(cv::Mat)进行讲解。
//vi_vo.h
/***************************fixed:无需修改***********************************/
//VB for RGB888 output
config.comm_pool[1].blk_cnt = 5;
config.comm_pool[1].mode = VB_REMAP_MODE_NOCACHE;
config.comm_pool[1].blk_size = VICAP_ALIGN_UP((SENSOR_HEIGHT * SENSOR_WIDTH * 3 ), VICAP_ALIGN_1K);
/*****************************************************************************/

//main.cc
while (!isp_stop)
{
cv::Mat ori_img;
//sensor to cv::Mat
{
/***************************fixed:无需修改***********************************/
//从摄像头通道1读取一帧图像,即从VB中dump一帧数据到dump_info
memset(&dump_info, 0 , sizeof(k_video_frame_info));
ret = kd_mpi_vicap_dump_frame(vicap_dev, VICAP_CHN_ID_1, VICAP_DUMP_YUV, &dump_info, 1000);
if (ret) {
printf("sample_vicap...kd_mpi_vicap_dump_frame failed.\n");
continue;
}

//将dump_info对应DDR地址(物理地址)映射到当前系统(虚拟地址)进行访问
//vbvaddr是实时改变的,因此我们最好把最新数据拷贝到【固定地址】vaddr,以便其它部分进行访问
auto vbvaddr = kd_mpi_sys_mmap_cached(dump_info.v_frame.phys_addr[0], size);
memcpy(vaddr, (void *)vbvaddr, SENSOR_HEIGHT * SENSOR_WIDTH * 3);
kd_mpi_sys_munmap(vbvaddr, size);
/*****************************************************************************/

//将摄像头数据转换为为cv::Mat,sensor(rgb,chw)->cv::Mat(bgr,hwc)
cv::Mat image_r = cv::Mat(SENSOR_HEIGHT,SENSOR_WIDTH, CV_8UC1, vaddr);
cv::Mat image_g = cv::Mat(SENSOR_HEIGHT,SENSOR_WIDTH, CV_8UC1, vaddr+SENSOR_HEIGHT*SENSOR_WIDTH);
cv::Mat image_b = cv::Mat(SENSOR_HEIGHT,SENSOR_WIDTH, CV_8UC1, vaddr+2*SENSOR_HEIGHT*SENSOR_WIDTH);
std::vector<cv::Mat> color_vec(3);
color_vec.clear();
color_vec.push_back(image_b);
color_vec.push_back(image_g);
color_vec.push_back(image_r);
cv::merge(color_vec, ori_img);
}
//使用当前帧数据
......
......
{
/***************************fixed:无需修改***********************************/
// 释放sensor当前帧
ret = kd_mpi_vicap_dump_release(vicap_dev, VICAP_CHN_ID_1, &dump_info);
if (ret) {
printf("sample_vicap...kd_mpi_vicap_dump_release failed.\n");
}
/*****************************************************************************/
}
}

3. 停止摄像头

//vi_vo.h
/***************************fixed:无需修改***********************************/
int vivcap_stop()
{
// 摄像头停止
printf("sample_vicap ...kd_mpi_vicap_stop_stream\n");
int ret = kd_mpi_vicap_stop_stream(vicap_dev);
if (ret) {
printf("sample_vicap, kd_mpi_vicap_init failed.\n");
return ret;
}

// 摄像头资源释放
ret = kd_mpi_vicap_deinit(vicap_dev);
if (ret) {
printf("sample_vicap, kd_mpi_vicap_deinit failed.\n");
return ret;
}

kd_mpi_vo_disable_video_layer(K_VO_LAYER1);

vicap_mpp_chn.mod_id = K_ID_VI;
vicap_mpp_chn.dev_id = vicap_dev;
vicap_mpp_chn.chn_id = vicap_chn;

vo_mpp_chn.mod_id = K_ID_VO;
vo_mpp_chn.dev_id = K_VO_DISPLAY_DEV_ID;
vo_mpp_chn.chn_id = K_VO_DISPLAY_CHN_ID1;

// vi vo解绑
sample_vicap_unbind_vo(vicap_mpp_chn, vo_mpp_chn);

/*Allow one frame time for the VO to release the VB block*/
k_u32 display_ms = 1000 / 33;
usleep(1000 * display_ms);

// 退出vb
ret = kd_mpi_vb_exit();
if (ret) {
printf("sample_vicap, kd_mpi_vb_exit failed.\n");
return ret;
}

return 0;
}
/*****************************************************************************/

只需在main.cc中调用:

//main.cc
vivcap_start();
......
vivcap_stop();

5.1.2 预处理

对当前帧数据进行resize处理。

//main.cc
while (!isp_stop)
{
cv::Mat ori_img; //ori:uint8,chw,rgb
//sensor to cv::Mat
{
......
}
/***************************unfixed:不同AI Demo可能需要修改******************/
// pre_process,cv::Mat((1280,720),bgr,hwc)->kmodel((224,224),rgb,hwc)
cv::Mat pre_process_img;
{
cv::Mat rgb_img;
cv::cvtColor(ori_img, rgb_img, cv::COLOR_BGR2RGB);
cv::resize(rgb_img, pre_process_img, cv::Size(kmodel_input_width, kmodel_input_height), cv::INTER_LINEAR);
}
/*****************************************************************************/

//已在ai_base.cc中给出通用实现,并且将在第6章给出开源代码
/***************************fixed:无需修改***********************************/
// set kmodel input
{
runtime_tensor tensor0 = kmodel_interp.input_tensor(0).expect("cannot get input tensor");
auto in_buf = tensor0.impl()->to_host().unwrap()->buffer().as_host().unwrap().map(map_access_::map_write).unwrap().buffer();
memcpy(reinterpret_cast<unsigned char *>(in_buf.data()), pre_process_img.data,sizeof(uint8_t)* kmodel_input_height * kmodel_input_width * 3);
hrt::sync(tensor0, sync_op_t::sync_write_back, true).expect("sync write_back failed");
}
/*****************************************************************************/
......
}

5.1.3 模型推理

设置好模型输入后,进行模型推理,得到模型推理结果。

//main.cc
string kmodel_path = argv[1];
cout<<kmodel_path<<endl;
float cls_thresh=0.5;

//已在ai_base.cc中给出通用实现,并且将在第6章给出开源代码
/***************************fixed:无需修改***********************************/
// kmodel解释器,从kmodel文件构建,负责模型的加载、输入输出设置和推理
interpreter kmodel_interp;
// load model
std::ifstream ifs(kmodel_path, std::ios::binary);
kmodel_interp.load_model(ifs).expect("Invalid kmodel");

// inputs init
for (size_t i = 0; i < kmodel_interp.inputs_size(); i++)
{
auto desc = kmodel_interp.input_desc(i);
auto shape = kmodel_interp.input_shape(i);
auto tensor = host_runtime_tensor::create(desc.datatype, shape, hrt::pool_shared).expect("cannot create input tensor");
kmodel_interp.input_tensor(i, tensor).expect("cannot set input tensor");
}
auto shape0 = kmodel_interp.input_shape(0); //nhwc
int kmodel_input_height = shape0[1];
int kmodel_input_width = shape0[2];

// outputs init
for (size_t i = 0; i < kmodel_interp.outputs_size(); i++)
{
auto desc = kmodel_interp.output_desc(i);
auto shape = kmodel_interp.output_shape(i);
auto tensor = host_runtime_tensor::create(desc.datatype, shape, hrt::pool_shared).expect("cannot create output tensor");
kmodel_interp.output_tensor(i, tensor).expect("cannot set output tensor");
}
/*****************************************************************************/

while (!isp_stop)
{
cv::Mat ori_img;
//sensor to cv::Mat
{
......
}

// pre_process
cv::Mat pre_process_img;
{
......
}

// set kmodel input
{
......
}

//已在ai_base.cc中给出通用实现,并且将在第6章给出开源代码
/***************************fixed:无需修改***********************************/
// kmodel run
kmodel_interp.run().expect("error occurred in running model");

// get kmodel output
vector<float *> k_outputs;
{
for (int i = 0; i < kmodel_interp.outputs_size(); i++)
{
auto out = kmodel_interp.output_tensor(i).expect("cannot get output tensor");
auto buf = out.impl()->to_host().unwrap()->buffer().as_host().unwrap().map(map_access_::map_read).unwrap().buffer();
float *p_out = reinterpret_cast<float *>(buf.data());
k_outputs.push_back(p_out);
}
}
/*****************************************************************************/
}

5.1.4 后处理

对模型结果进行后处理,并结果放到results中。

//main.cc
vector<cls_res> results;
while (!isp_stop)
{
cv::Mat ori_img;
//sensor to cv::Mat
{
......
}

// pre_process
cv::Mat pre_process_img;
{
......
}

// set kmodel input
{
......
}

// kmodel run
......

// get kmodel output
vector<float *> k_outputs;
{
......
}

//post process
results.clear();
{
/***************************unfixed:不同AI Demo可能需要修改******************/
float* output0 = k_outputs[0];
//softmax
float sum = 0.0;
for (int i = 0; i < labels.size(); i++){
sum += exp(output0[i]);
}

int max_index;
for (int i = 0; i < labels.size(); i++)
{
output0[i] = exp(output0[i]) / sum;
}
max_index = std::max_element(output0,output0+labels.size()) - output0;
cls_res b;
if (output0[max_index] >= cls_thresh)
{
b.label = labels[max_index];
b.score = output0[max_index];
results.push_back(b);
}
/*****************************************************************************/
}
}

5.1.5 显示

显示(视频输出,VO)与display相关,本小节简要介绍基于c++的显示设置、显示叠加的整体流程;详细介绍参见K230_视频输出_API参考.md

  • 显示设置:设置显示大小,格式
  • 显示叠加:显示由2个图层构成,其中下边的图层(原图图层)直接显示摄像头输出,上边的图层(osd图层)用于画框、画点,写文字等。

1. 显示设置:设置显示大小,格式。

//vi_vo.h
/***************************fixed:无需修改***********************************/
static k_s32 sample_connector_init(void)
{
......
k_connector_type connector_type = LT9611_MIPI_4LAN_1920X1080_30FPS;
......
}

static k_s32 vo_layer_vdss_bind_vo_config(void)
{
......
sample_connector_init();

// config lyaer
info.act_size.width = ISP_CHN0_WIDTH;//ISP_CHN0_HEIGHT;//1080;//640;//1080;
info.act_size.height = ISP_CHN0_HEIGHT;//ISP_CHN0_WIDTH;//1920;//480;//1920;
info.format = PIXEL_FORMAT_YVU_PLANAR_420;
......
}
/*****************************************************************************/

2. 显示叠加:由于摄像头和显示的通道进行了绑定,我们无法对vo进行直接操作,因此采用叠加的方式进行显示。

原图图层:由于摄像头(vi)通道0绑定了显示(vo)的通道1;随着摄像头的启动,摄像头通道0的数据会自动流到vo的通道1。

//vi_vo.h
......
/***************************fixed:无需修改***********************************/
//bind vicap chn 0 to vo
vicap_mpp_chn.mod_id = K_ID_VI;
vicap_mpp_chn.dev_id = vicap_dev;
vicap_mpp_chn.chn_id = vicap_chn;

vo_mpp_chn.mod_id = K_ID_VO;
vo_mpp_chn.dev_id = K_VO_DISPLAY_DEV_ID;
vo_mpp_chn.chn_id = K_VO_DISPLAY_CHN_ID1;

sample_vicap_bind_vo(vicap_mpp_chn, vo_mpp_chn);
printf("sample_vicap ...dwc_dsi_init\n");
/*****************************************************************************/
......

osd图层:cv::Mat上画框、画点、写文字之后,将数据插入vo对应通道。

// vi_vo.h
#define osd_id K_VO_OSD3
/***************************unfixed:不同AI Demo可能需要修改******************/
#define ISP_CHN0_WIDTH (1920) // sensor ch0输出宽度,vo
#define ISP_CHN0_HEIGHT (1080) // sensor ch0输出高度,vo
#define osd_width (1920)
#define osd_height (1080)
/*****************************************************************************/
.....
/***************************fixed:无需修改***********************************/
k_vb_blk_handle vo_insert_frame(k_video_frame_info *vf_info, void **pic_vaddr)
{
k_u64 phys_addr = 0;
k_u32 *virt_addr;
k_vb_blk_handle handle;
k_s32 size;

if (vf_info == NULL)
return K_FALSE;

if (vf_info->v_frame.pixel_format == PIXEL_FORMAT_ABGR_8888 || vf_info->v_frame.pixel_format == PIXEL_FORMAT_ARGB_8888)
size = vf_info->v_frame.height * vf_info->v_frame.width * 4;
else if (vf_info->v_frame.pixel_format == PIXEL_FORMAT_RGB_565 || vf_info->v_frame.pixel_format == PIXEL_FORMAT_BGR_565)
size = vf_info->v_frame.height * vf_info->v_frame.width * 2;
else if (vf_info->v_frame.pixel_format == PIXEL_FORMAT_ABGR_4444 || vf_info->v_frame.pixel_format == PIXEL_FORMAT_ARGB_4444)
size = vf_info->v_frame.height * vf_info->v_frame.width * 2;
else if (vf_info->v_frame.pixel_format == PIXEL_FORMAT_RGB_888 || vf_info->v_frame.pixel_format == PIXEL_FORMAT_BGR_888)
size = vf_info->v_frame.height * vf_info->v_frame.width * 3;
else if (vf_info->v_frame.pixel_format == PIXEL_FORMAT_ARGB_1555 || vf_info->v_frame.pixel_format == PIXEL_FORMAT_ABGR_1555)
size = vf_info->v_frame.height * vf_info->v_frame.width * 2;
else if (vf_info->v_frame.pixel_format == PIXEL_FORMAT_YVU_PLANAR_420)
size = vf_info->v_frame.height * vf_info->v_frame.width * 3 / 2;

size = size + 4096; // 强制4K ,后边得删了

printf("vb block size is %x \n", size);

handle = kd_mpi_vb_get_block(g_pool_id, size, NULL);
if (handle == VB_INVALID_HANDLE)
{
printf("%s get vb block error\n", __func__);
return K_FAILED;
}

phys_addr = kd_mpi_vb_handle_to_phyaddr(handle);
if (phys_addr == 0)
{
printf("%s get phys addr error\n", __func__);
return K_FAILED;
}

virt_addr = (k_u32 *)kd_mpi_sys_mmap(phys_addr, size);
// virt_addr = (k_u32 *)kd_mpi_sys_mmap_cached(phys_addr, size);

if (virt_addr == NULL)
{
printf("%s mmap error\n", __func__);
return K_FAILED;
}

vf_info->mod_id = K_ID_VO;
vf_info->pool_id = g_pool_id;
vf_info->v_frame.phys_addr[0] = phys_addr;
if (vf_info->v_frame.pixel_format == PIXEL_FORMAT_YVU_PLANAR_420)
vf_info->v_frame.phys_addr[1] = phys_addr + (vf_info->v_frame.height * vf_info->v_frame.stride[0]);
*pic_vaddr = virt_addr;

printf("phys_addr is %lx g_pool_id is %d \n", phys_addr, g_pool_id);

return handle;
}
/*****************************************************************************/

使用示例

// main.cc
/***************************fixed:无需修改***********************************/
// osd create
k_video_frame_info vf_info;
void *pic_vaddr = NULL;
memset(&vf_info, 0, sizeof(vf_info));
vf_info.v_frame.width = osd_width;
vf_info.v_frame.height = osd_height;
vf_info.v_frame.stride[0] = osd_width;
vf_info.v_frame.pixel_format = PIXEL_FORMAT_ARGB_8888;
block = vo_insert_frame(&vf_info, &pic_vaddr);
/*****************************************************************************/

while (!isp_stop)
{
cv::Mat ori_img;
// sensor to cv::Mat
// pre_process
// set kmodel input
// kmodel run
// get kmodel output
// post process
results.clear();
{
......
}

// draw result to vo
{
// draw osd
{
cv::Mat osd_frame(osd_height, osd_width, CV_8UC4, cv::Scalar(0, 0, 0, 0));
/***************************unfixed:不同AI Demo可能需要修改******************/
{
//draw cls
double fontsize = (osd_frame.cols * osd_frame.rows * 1.0) / (1100 * 1200);
for(int i = 0; i < results.size(); i++)
{
std::string text = "class: " + results[i].label + ", score: " + std::to_string(round(results[i].score * 100) / 100.0).substr(0, 4);

cv::putText(osd_frame, text, cv::Point(1, 40), cv::FONT_HERSHEY_SIMPLEX, 0.8, cv::Scalar(255, 255, 255, 0), 2);

std::cout << text << std::endl;
}
}

/*************************************************************************/

/***************************fixed:无需修改***********************************/
memcpy(pic_vaddr, osd_frame.data, osd_width * osd_height * 4);
}

// insert osd to vo
{
kd_mpi_vo_chn_insert_frame(osd_id+3, &vf_info);
printf("kd_mpi_vo_chn_insert_frame success \n");
}
/*****************************************************************************/
}
......
}

5.1.6 完整代码

具体怎么操作运行,请参考第6章,这里着重介绍代码流程。

vi_vo.h

/* Copyright (c) 2023, Canaan Bright Sight Co., Ltd
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
* NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/

#include <chrono>
#include <fstream>
#include <iostream>
#include <thread>
#include <atomic>

#include "mpi_sys_api.h"

/* vicap */
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include <sys/mman.h>

#include "k_module.h"
#include "k_type.h"
#include "k_vb_comm.h"
#include "k_video_comm.h"
#include "k_sys_comm.h"
#include "mpi_vb_api.h"
#include "mpi_vicap_api.h"
#include "mpi_isp_api.h"
#include "mpi_sys_api.h"
#include "k_vo_comm.h"
#include "mpi_vo_api.h"

#include "vo_test_case.h"

#include "k_connector_comm.h"
#include "mpi_connector_api.h"
#include "k_autoconf_comm.h"

/***************************unfixed:不同AI Demo可能需要修改*******************/
#if defined(CONFIG_BOARD_K230_CANMV)
#define SENSOR_CHANNEL (3) // 通道数
#define SENSOR_HEIGHT (720) // sensor ch1输出高度,AI输入
#define SENSOR_WIDTH (1280) // sensor ch1输出宽度,AI输入
#define ISP_CHN0_WIDTH (1920) // sensor ch0输出宽度,vo
#define ISP_CHN0_HEIGHT (1080) // sensor ch0输出高度,vo
#define vicap_install_osd (1)
#define osd_id K_VO_OSD3
#define osd_width (1920)
#define osd_height (1080)
#else
#define SENSOR_CHANNEL (3)
#define SENSOR_HEIGHT (1280)
#define SENSOR_WIDTH (720)
#define ISP_CHN0_WIDTH (1088)
#define ISP_CHN0_HEIGHT (1920)
#define vicap_install_osd (1)
#define osd_id K_VO_OSD3
#define osd_width (1080)
#define osd_height (1920)
#endif
/*****************************************************************************/

/***************************fixed:无需修改***********************************/
k_vb_config config;
k_vicap_dev vicap_dev;
k_vicap_chn vicap_chn;
k_vicap_dev_attr dev_attr;
k_vicap_chn_attr chn_attr;
k_vicap_sensor_info sensor_info;
k_vicap_sensor_type sensor_type;
k_mpp_chn vicap_mpp_chn;
k_mpp_chn vo_mpp_chn;

k_video_frame_info dump_info;

k_vo_draw_frame vo_frame = (k_vo_draw_frame) {
1,
16,
16,
128,
128,
1
};

static k_vb_blk_handle block;
k_u32 g_pool_id;

int vo_creat_layer_test(k_vo_layer chn_id, layer_info *info)
{
k_vo_video_layer_attr attr;

// check layer
if ((chn_id >= K_MAX_VO_LAYER_NUM) || ((info->func & K_VO_SCALER_ENABLE) && (chn_id != K_VO_LAYER0))
|| ((info->func != 0) && (chn_id == K_VO_LAYER2)))
{
printf("input layer num failed \n");
return -1 ;
}

// check scaler

// set offset
attr.display_rect = info->offset;
// set act
attr.img_size = info->act_size;
// sget size
info->size = info->act_size.height * info->act_size.width * 3 / 2;
//set pixel format
attr.pixel_format = info->format;
if (info->format != PIXEL_FORMAT_YVU_PLANAR_420)
{
printf("input pix format failed \n");
return -1;
}
// set stride
attr.stride = (info->act_size.width / 8 - 1) + ((info->act_size.height - 1) << 16);
// set function
attr.func = info->func;
// set scaler attr
attr.scaler_attr = info->attr;

// set video layer atrr
kd_mpi_vo_set_video_layer_attr(chn_id, &attr);

// enable layer
kd_mpi_vo_enable_video_layer(chn_id);

return 0;
}

k_vb_blk_handle vo_insert_frame(k_video_frame_info *vf_info, void **pic_vaddr)
{
k_u64 phys_addr = 0;
k_u32 *virt_addr;
k_vb_blk_handle handle;
k_s32 size;

if (vf_info == NULL)
return K_FALSE;

if (vf_info->v_frame.pixel_format == PIXEL_FORMAT_ABGR_8888 || vf_info->v_frame.pixel_format == PIXEL_FORMAT_ARGB_8888)
size = vf_info->v_frame.height * vf_info->v_frame.width * 4;
else if (vf_info->v_frame.pixel_format == PIXEL_FORMAT_RGB_565 || vf_info->v_frame.pixel_format == PIXEL_FORMAT_BGR_565)
size = vf_info->v_frame.height * vf_info->v_frame.width * 2;
else if (vf_info->v_frame.pixel_format == PIXEL_FORMAT_ABGR_4444 || vf_info->v_frame.pixel_format == PIXEL_FORMAT_ARGB_4444)
size = vf_info->v_frame.height * vf_info->v_frame.width * 2;
else if (vf_info->v_frame.pixel_format == PIXEL_FORMAT_RGB_888 || vf_info->v_frame.pixel_format == PIXEL_FORMAT_BGR_888)
size = vf_info->v_frame.height * vf_info->v_frame.width * 3;
else if (vf_info->v_frame.pixel_format == PIXEL_FORMAT_ARGB_1555 || vf_info->v_frame.pixel_format == PIXEL_FORMAT_ABGR_1555)
size = vf_info->v_frame.height * vf_info->v_frame.width * 2;
else if (vf_info->v_frame.pixel_format == PIXEL_FORMAT_YVU_PLANAR_420)
size = vf_info->v_frame.height * vf_info->v_frame.width * 3 / 2;

size = size + 4096; // 强制4K ,后边得删了

printf("vb block size is %x \n", size);

handle = kd_mpi_vb_get_block(g_pool_id, size, NULL);
if (handle == VB_INVALID_HANDLE)
{
printf("%s get vb block error\n", __func__);
return K_FAILED;
}

phys_addr = kd_mpi_vb_handle_to_phyaddr(handle);
if (phys_addr == 0)
{
printf("%s get phys addr error\n", __func__);
return K_FAILED;
}

virt_addr = (k_u32 *)kd_mpi_sys_mmap(phys_addr, size);
// virt_addr = (k_u32 *)kd_mpi_sys_mmap_cached(phys_addr, size);

if (virt_addr == NULL)
{
printf("%s mmap error\n", __func__);
return K_FAILED;
}

vf_info->mod_id = K_ID_VO;
vf_info->pool_id = g_pool_id;
vf_info->v_frame.phys_addr[0] = phys_addr;
if (vf_info->v_frame.pixel_format == PIXEL_FORMAT_YVU_PLANAR_420)
vf_info->v_frame.phys_addr[1] = phys_addr + (vf_info->v_frame.height * vf_info->v_frame.stride[0]);
*pic_vaddr = virt_addr;

printf("phys_addr is %lx g_pool_id is %d \n", phys_addr, g_pool_id);

return handle;
}

k_u32 vo_creat_osd_test(k_vo_osd osd, osd_info *info)
{
k_vo_video_osd_attr attr;

// set attr
attr.global_alptha = info->global_alptha;

if (info->format == PIXEL_FORMAT_ABGR_8888 || info->format == PIXEL_FORMAT_ARGB_8888)
{
info->size = info->act_size.width * info->act_size.height * 4;
info->stride = info->act_size.width * 4 / 8;
}
else if (info->format == PIXEL_FORMAT_RGB_565 || info->format == PIXEL_FORMAT_BGR_565)
{
info->size = info->act_size.width * info->act_size.height * 2;
info->stride = info->act_size.width * 2 / 8;
}
else if (info->format == PIXEL_FORMAT_RGB_888 || info->format == PIXEL_FORMAT_BGR_888)
{
info->size = info->act_size.width * info->act_size.height * 3;
info->stride = info->act_size.width * 3 / 8;
}
else if(info->format == PIXEL_FORMAT_ARGB_4444 || info->format == PIXEL_FORMAT_ABGR_4444)
{
info->size = info->act_size.width * info->act_size.height * 2;
info->stride = info->act_size.width * 2 / 8;
}
else if(info->format == PIXEL_FORMAT_ARGB_1555 || info->format == PIXEL_FORMAT_ABGR_1555)
{
info->size = info->act_size.width * info->act_size.height * 2;
info->stride = info->act_size.width * 2 / 8;
}
else
{
printf("set osd pixel format failed \n");
}

attr.stride = info->stride;
attr.pixel_format = info->format;
attr.display_rect = info->offset;
attr.img_size = info->act_size;
kd_mpi_vo_set_video_osd_attr(osd, &attr);

kd_mpi_vo_osd_enable(osd);

return 0;
}

void sample_vicap_install_osd(void)
{
osd_info osd;

osd.act_size.width = osd_width ;
osd.act_size.height = osd_height;
osd.offset.x = 0;
osd.offset.y = 0;
osd.global_alptha = 0xff;
// osd.global_alptha = 0x32;
osd.format = PIXEL_FORMAT_ARGB_8888;//PIXEL_FORMAT_ARGB_4444; //PIXEL_FORMAT_ARGB_1555;//PIXEL_FORMAT_ARGB_8888;

vo_creat_osd_test(osd_id, &osd);
}

void vo_osd_release_block(void)
{
if(vicap_install_osd == 1)
{
kd_mpi_vo_osd_disable(osd_id);
kd_mpi_vb_release_block(block);
}

}

static k_s32 sample_connector_init(void)
{
k_u32 ret = 0;
k_s32 connector_fd;
#if defined(CONFIG_BOARD_K230_CANMV)
k_connector_type connector_type = LT9611_MIPI_4LAN_1920X1080_30FPS;// HX8377_V2_MIPI_4LAN_1080X1920_30FPS;
#else
k_connector_type connector_type = HX8377_V2_MIPI_4LAN_1080X1920_30FPS;
#endif
k_connector_info connector_info;

memset(&connector_info, 0, sizeof(k_connector_info));

//connector get sensor info
ret = kd_mpi_get_connector_info(connector_type, &connector_info);
if (ret) {
printf("sample_vicap, the sensor type not supported!\n");
return ret;
}

connector_fd = kd_mpi_connector_open(connector_info.connector_name);
if (connector_fd < 0) {
printf("%s, connector open failed.\n", __func__);
return K_ERR_VO_NOTREADY;
}

// set connect power
kd_mpi_connector_power_set(connector_fd, K_TRUE);
// connector init
kd_mpi_connector_init(connector_fd, connector_info);

return 0;
}

static k_s32 vo_layer_vdss_bind_vo_config(void)
{
layer_info info;

k_vo_layer chn_id = K_VO_LAYER1;

memset(&info, 0, sizeof(info));

sample_connector_init();

// config lyaer
info.act_size.width = ISP_CHN0_WIDTH;//ISP_CHN0_HEIGHT;//1080;//640;//1080;
info.act_size.height = ISP_CHN0_HEIGHT;//ISP_CHN0_WIDTH;//1920;//480;//1920;
info.format = PIXEL_FORMAT_YVU_PLANAR_420;
info.func = 0;//K_ROTATION_180;////K_ROTATION_90;
info.global_alptha = 0xff;
info.offset.x = 0;//(1080-w)/2,
info.offset.y = 0;//(1920-h)/2;
vo_creat_layer_test(chn_id, &info);

if(vicap_install_osd == 1)
sample_vicap_install_osd();

//exit ;
return 0;
}

static void sample_vicap_bind_vo(k_mpp_chn vicap_mpp_chn, k_mpp_chn vo_mpp_chn)
{
k_s32 ret;

ret = kd_mpi_sys_bind(&vicap_mpp_chn, &vo_mpp_chn);
if (ret) {
printf("kd_mpi_sys_unbind failed:0x%x\n", ret);
}

return;
}

static void sample_vicap_unbind_vo(k_mpp_chn vicap_mpp_chn, k_mpp_chn vo_mpp_chn)
{
k_s32 ret;

ret = kd_mpi_sys_unbind(&vicap_mpp_chn, &vo_mpp_chn);
if (ret) {
printf("kd_mpi_sys_unbind failed:0x%x\n", ret);
}

return;
}

int vivcap_start()
{
k_s32 ret = 0;

k_u32 pool_id;
k_vb_pool_config pool_config;

printf("sample_vicap ...\n");

#if defined(CONFIG_BOARD_K230_CANMV)
sensor_type = OV_OV5647_MIPI_CSI0_1920X1080_30FPS_10BIT_LINEAR;
kd_mpi_vicap_set_mclk(VICAP_MCLK0, VICAP_PLL0_CLK_DIV4, 16, 1);
#else
sensor_type = IMX335_MIPI_2LANE_RAW12_2592X1944_30FPS_LINEAR;
#endif
vicap_dev = VICAP_DEV_ID_0;

memset(&config, 0, sizeof(config));
config.max_pool_cnt = 64;
//VB for YUV420SP output
config.comm_pool[0].blk_cnt = 5;
config.comm_pool[0].mode = VB_REMAP_MODE_NOCACHE;
config.comm_pool[0].blk_size = VICAP_ALIGN_UP((ISP_CHN0_WIDTH * ISP_CHN0_HEIGHT * 3 / 2), VICAP_ALIGN_1K);

//VB for RGB888 output
config.comm_pool[1].blk_cnt = 5;
config.comm_pool[1].mode = VB_REMAP_MODE_NOCACHE;
config.comm_pool[1].blk_size = VICAP_ALIGN_UP((SENSOR_HEIGHT * SENSOR_WIDTH * 3 ), VICAP_ALIGN_1K);

ret = kd_mpi_vb_set_config(&config);
if (ret) {
printf("vb_set_config failed ret:%d\n", ret);
return ret;
}

k_vb_supplement_config supplement_config;
memset(&supplement_config, 0, sizeof(supplement_config));
supplement_config.supplement_config |= VB_SUPPLEMENT_JPEG_MASK;

ret = kd_mpi_vb_set_supplement_config(&supplement_config);
if (ret) {
printf("vb_set_supplement_config failed ret:%d\n", ret);
return ret;
}

ret = kd_mpi_vb_init();
if (ret) {
printf("vb_init failed ret:%d\n", ret);
return ret;
}
printf("sample_vicap ...kd_mpi_vicap_get_sensor_info\n");

// dwc_dsi_init();
vo_layer_vdss_bind_vo_config();

if(vicap_install_osd == 1)
{
memset(&pool_config, 0, sizeof(pool_config));
pool_config.blk_size = VICAP_ALIGN_UP((osd_width * osd_height * 4 * 2), VICAP_ALIGN_1K);
pool_config.blk_cnt = 4;
pool_config.mode = VB_REMAP_MODE_NOCACHE;
pool_id = kd_mpi_vb_create_pool(&pool_config); // osd0 - 3 argb 320 x 240
g_pool_id = pool_id;

printf("--------aa--------------g_pool_id is %d pool_id is %d \n",g_pool_id, pool_id);
}

memset(&sensor_info, 0, sizeof(k_vicap_sensor_info));
ret = kd_mpi_vicap_get_sensor_info(sensor_type, &sensor_info);
if (ret) {
printf("sample_vicap, the sensor type not supported!\n");
return ret;
}

memset(&dev_attr, 0, sizeof(k_vicap_dev_attr));
dev_attr.acq_win.h_start = 0;
dev_attr.acq_win.v_start = 0;
#if defined (CONFIG_BOARD_K230_CANMV)
dev_attr.acq_win.width = ISP_CHN0_WIDTH;
dev_attr.acq_win.height = ISP_CHN0_HEIGHT;
#else
dev_attr.acq_win.width = 2592;//SENSOR_HEIGHT;
dev_attr.acq_win.height = 1944;//SENSOR_WIDTH;
#endif
dev_attr.mode = VICAP_WORK_ONLINE_MODE;

dev_attr.pipe_ctrl.data = 0xFFFFFFFF;
dev_attr.pipe_ctrl.bits.af_enable = 0;
dev_attr.pipe_ctrl.bits.ahdr_enable = 0;


dev_attr.cpature_frame = 0;
memcpy(&dev_attr.sensor_info, &sensor_info, sizeof(k_vicap_sensor_info));

ret = kd_mpi_vicap_set_dev_attr(vicap_dev, dev_attr);
if (ret) {
printf("sample_vicap, kd_mpi_vicap_set_dev_attr failed.\n");
return ret;
}

memset(&chn_attr, 0, sizeof(k_vicap_chn_attr));

//set chn0 output yuv420sp
chn_attr.out_win.h_start = 0;
chn_attr.out_win.v_start = 0;
chn_attr.out_win.width = ISP_CHN0_WIDTH;
chn_attr.out_win.height = ISP_CHN0_HEIGHT;


#if defined(CONFIG_BOARD_K230_CANMV)
chn_attr.crop_win = dev_attr.acq_win;
#else
// chn_attr.crop_win = dev_attr.acq_win;
chn_attr.crop_win.h_start = 768;
chn_attr.crop_win.v_start = 16;
chn_attr.crop_win.width = ISP_CHN0_WIDTH;
chn_attr.crop_win.height = ISP_CHN0_HEIGHT;
#endif

chn_attr.scale_win = chn_attr.out_win;
chn_attr.crop_enable = K_FALSE;
chn_attr.scale_enable = K_FALSE;
// chn_attr.dw_enable = K_FALSE;
chn_attr.chn_enable = K_TRUE;
chn_attr.pix_format = PIXEL_FORMAT_YVU_PLANAR_420;
chn_attr.buffer_num = VICAP_MAX_FRAME_COUNT;//at least 3 buffers for isp
chn_attr.buffer_size = config.comm_pool[0].blk_size;
vicap_chn = VICAP_CHN_ID_0;

printf("sample_vicap ...kd_mpi_vicap_set_chn_attr, buffer_size[%d]\n", chn_attr.buffer_size);
ret = kd_mpi_vicap_set_chn_attr(vicap_dev, vicap_chn, chn_attr);
if (ret) {
printf("sample_vicap, kd_mpi_vicap_set_chn_attr failed.\n");
return ret;
}

//bind vicap chn 0 to vo
vicap_mpp_chn.mod_id = K_ID_VI;
vicap_mpp_chn.dev_id = vicap_dev;
vicap_mpp_chn.chn_id = vicap_chn;

vo_mpp_chn.mod_id = K_ID_VO;
vo_mpp_chn.dev_id = K_VO_DISPLAY_DEV_ID;
vo_mpp_chn.chn_id = K_VO_DISPLAY_CHN_ID1;

sample_vicap_bind_vo(vicap_mpp_chn, vo_mpp_chn);
printf("sample_vicap ...dwc_dsi_init\n");

//set chn1 output rgb888p
chn_attr.out_win.h_start = 0;
chn_attr.out_win.v_start = 0;
chn_attr.out_win.width = SENSOR_WIDTH ;
chn_attr.out_win.height = SENSOR_HEIGHT;
// chn_attr.crop_win = dev_attr.acq_win;

#if defined(CONFIG_BOARD_K230_CANMV)
chn_attr.crop_win = dev_attr.acq_win;
#else
chn_attr.crop_win.h_start = 768;
chn_attr.crop_win.v_start = 16;
chn_attr.crop_win.width = ISP_CHN0_WIDTH;
chn_attr.crop_win.height = ISP_CHN0_HEIGHT;
#endif

chn_attr.scale_win = chn_attr.out_win;
chn_attr.crop_enable = K_FALSE;
chn_attr.scale_enable = K_FALSE;
// chn_attr.dw_enable = K_FALSE;
chn_attr.chn_enable = K_TRUE;
chn_attr.pix_format = PIXEL_FORMAT_BGR_888_PLANAR;
chn_attr.buffer_num = VICAP_MAX_FRAME_COUNT;//at least 3 buffers for isp
chn_attr.buffer_size = config.comm_pool[1].blk_size;

printf("sample_vicap ...kd_mpi_vicap_set_chn_attr, buffer_size[%d]\n", chn_attr.buffer_size);
ret = kd_mpi_vicap_set_chn_attr(vicap_dev, VICAP_CHN_ID_1, chn_attr);
if (ret) {
printf("sample_vicap, kd_mpi_vicap_set_chn_attr failed.\n");
return ret;
}

printf("sample_vicap ...kd_mpi_vicap_init\n");
ret = kd_mpi_vicap_init(vicap_dev);
if (ret) {
printf("sample_vicap, kd_mpi_vicap_init failed.\n");
// goto err_exit;
}

printf("sample_vicap ...kd_mpi_vicap_start_stream\n");
ret = kd_mpi_vicap_start_stream(vicap_dev);
if (ret) {
printf("sample_vicap, kd_mpi_vicap_init failed.\n");
// goto err_exit;
}

return ret;
}

int vivcap_stop()
{
printf("sample_vicap ...kd_mpi_vicap_stop_stream\n");
int ret = kd_mpi_vicap_stop_stream(vicap_dev);
if (ret) {
printf("sample_vicap, kd_mpi_vicap_init failed.\n");
return ret;
}

ret = kd_mpi_vicap_deinit(vicap_dev);
if (ret) {
printf("sample_vicap, kd_mpi_vicap_deinit failed.\n");
return ret;
}

kd_mpi_vo_disable_video_layer(K_VO_LAYER1);

vicap_mpp_chn.mod_id = K_ID_VI;
vicap_mpp_chn.dev_id = vicap_dev;
vicap_mpp_chn.chn_id = vicap_chn;

vo_mpp_chn.mod_id = K_ID_VO;
vo_mpp_chn.dev_id = K_VO_DISPLAY_DEV_ID;
vo_mpp_chn.chn_id = K_VO_DISPLAY_CHN_ID1;

sample_vicap_unbind_vo(vicap_mpp_chn, vo_mpp_chn);

/*Allow one frame time for the VO to release the VB block*/
k_u32 display_ms = 1000 / 33;
usleep(1000 * display_ms);

ret = kd_mpi_vb_exit();
if (ret) {
printf("sample_vicap, kd_mpi_vb_exit failed.\n");
return ret;
}

return 0;
}

void yuv_rotate_90(char *des, char *src,int width,int height)
{
int n = 0;
int hw = width>>1;
int hh = height>>1;
int size = width * height;
int hsize = size>>2;

int pos = 0;

for(int i = width-1;i >= 0;i--)
{
pos = 0;
for(int j= 0;j < height;j++)
{
des[n++]= src[pos+i];
pos += width;
}
}

}
/****************************************************************************/

main.cc

/* Copyright (c) 2023, Canaan Bright Sight Co., Ltd
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
* NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/

#include <iostream>
#include <thread>
#include <map>
#include <nncase/runtime/runtime_tensor.h>
#include <nncase/runtime/interpreter.h>
#include <nncase/runtime/runtime_op_utility.h>
#include <opencv2/highgui.hpp>
#include <opencv2/imgcodecs.hpp>
#include <opencv2/imgproc.hpp>

#include "vi_vo.h"

using namespace nncase;
using namespace nncase::runtime;

using cv::Mat;
using std::cerr;
using std::cout;
using std::endl;
using namespace std;

auto cache = cv::Mat::zeros(1, 1, CV_32FC1);
/**
* @brief 分类结果结构
*/
typedef struct cls_res
{
float score;//分类分数
string label;//分类标签结果
}cls_res;

std::atomic<bool> isp_stop(false);

void video_proc_cls(char *argv[])
{

/***************************fixed:无需修改***********************************/
vivcap_start();

// osd set
k_video_frame_info vf_info;
void *pic_vaddr = NULL;

memset(&vf_info, 0, sizeof(vf_info));

vf_info.v_frame.width = osd_width;
vf_info.v_frame.height = osd_height;
vf_info.v_frame.stride[0] = osd_width;
vf_info.v_frame.pixel_format = PIXEL_FORMAT_ARGB_8888;
block = vo_insert_frame(&vf_info, &pic_vaddr);

// alloc memory for sensor
size_t paddr = 0;
void *vaddr = nullptr;
size_t size = SENSOR_CHANNEL * SENSOR_HEIGHT * SENSOR_WIDTH;
int ret = kd_mpi_sys_mmz_alloc_cached(&paddr, &vaddr, "allocate", "anonymous", size);
if (ret)
{
std::cerr << "physical_memory_block::allocate failed: ret = " << ret << ", errno = " << strerror(errno) << std::endl;
std::abort();
}
/*****************************************************************************/

string kmodel_path = argv[1];
cout<<"kmodel_path : "<<kmodel_path<<endl;
float cls_thresh=0.5;

/***************************fixed:无需修改***********************************/
// 我们已经把相关实现封装到ai_base.cc,这里只是为了介绍起来比较简单
interpreter kmodel_interp;
// load model
std::ifstream ifs(kmodel_path, std::ios::binary);
kmodel_interp.load_model(ifs).expect("Invalid kmodel");

// inputs init
for (size_t i = 0; i < kmodel_interp.inputs_size(); i++)
{
auto desc = kmodel_interp.input_desc(i);
auto shape = kmodel_interp.input_shape(i);
auto tensor = host_runtime_tensor::create(desc.datatype, shape, hrt::pool_shared).expect("cannot create input tensor");
kmodel_interp.input_tensor(i, tensor).expect("cannot set input tensor");
}
auto shape0 = kmodel_interp.input_shape(0); //nhwc
int kmodel_input_height = shape0[1];
int kmodel_input_width = shape0[2];

// outputs init
for (size_t i = 0; i < kmodel_interp.outputs_size(); i++)
{
auto desc = kmodel_interp.output_desc(i);
auto shape = kmodel_interp.output_shape(i);
auto tensor = host_runtime_tensor::create(desc.datatype, shape, hrt::pool_shared).expect("cannot create output tensor");
kmodel_interp.output_tensor(i, tensor).expect("cannot set output tensor");
}
/*****************************************************************************/

vector<cls_res> results;
std::vector<std::string> labels = {"bocai","changqiezi","huluobo","xihongshi","xilanhua"};

while (!isp_stop)
{
cv::Mat ori_img;
//sensor to cv::Mat
{
/***************************fixed:无需修改***********************************/
//从摄像头读取一帧图像
memset(&dump_info, 0 , sizeof(k_video_frame_info));
ret = kd_mpi_vicap_dump_frame(vicap_dev, VICAP_CHN_ID_1, VICAP_DUMP_YUV, &dump_info, 1000);
if (ret) {
printf("sample_vicap...kd_mpi_vicap_dump_frame failed.\n");
continue;
}

//将摄像头当前帧对应DDR地址映射到当前系统进行访问
auto vbvaddr = kd_mpi_sys_mmap_cached(dump_info.v_frame.phys_addr[0], size);
memcpy(vaddr, (void *)vbvaddr, SENSOR_HEIGHT * SENSOR_WIDTH * 3);
kd_mpi_sys_munmap(vbvaddr, size);
/*****************************************************************************/

//将摄像头数据转换为为cv::Mat,sensor(rgb,chw)->cv::Mat(bgr,hwc)
cv::Mat image_r = cv::Mat(SENSOR_HEIGHT,SENSOR_WIDTH, CV_8UC1, vaddr);
cv::Mat image_g = cv::Mat(SENSOR_HEIGHT,SENSOR_WIDTH, CV_8UC1, vaddr+SENSOR_HEIGHT*SENSOR_WIDTH);
cv::Mat image_b = cv::Mat(SENSOR_HEIGHT,SENSOR_WIDTH, CV_8UC1, vaddr+2*SENSOR_HEIGHT*SENSOR_WIDTH);
std::vector<cv::Mat> color_vec(3);
color_vec.clear();
color_vec.push_back(image_b);
color_vec.push_back(image_g);
color_vec.push_back(image_r);
cv::merge(color_vec, ori_img);
}

/***************************unfixed:不同AI Demo可能需要修改******************/
// pre_process
cv::Mat pre_process_img;
{
cv::Mat rgb_img;
cv::cvtColor(ori_img, rgb_img, cv::COLOR_BGR2RGB);
cv::resize(rgb_img, pre_process_img, cv::Size(kmodel_input_width, kmodel_input_height), cv::INTER_LINEAR);
}
/*****************************************************************************/

/***************************fixed:无需修改***********************************/
// set kmodel input
{
runtime_tensor tensor0 = kmodel_interp.input_tensor(0).expect("cannot get input tensor");
auto in_buf = tensor0.impl()->to_host().unwrap()->buffer().as_host().unwrap().map(map_access_::map_write).unwrap().buffer();
memcpy(reinterpret_cast<unsigned char *>(in_buf.data()), pre_process_img.data,sizeof(uint8_t)* kmodel_input_height * kmodel_input_width * 3);
hrt::sync(tensor0, sync_op_t::sync_write_back, true).expect("sync write_back failed");
}


// kmodel run
kmodel_interp.run().expect("error occurred in running model");

// get kmodel output
vector<float *> k_outputs;
{
for (int i = 0; i < kmodel_interp.outputs_size(); i++)
{
auto out = kmodel_interp.output_tensor(i).expect("cannot get output tensor");
auto buf = out.impl()->to_host().unwrap()->buffer().as_host().unwrap().map(map_access_::map_read).unwrap().buffer();
float *p_out = reinterpret_cast<float *>(buf.data());
k_outputs.push_back(p_out);
}
}
/***************************fixed:无需修改***********************************/

/***************************unfixed:不同AI Demo可能需要修改******************/
//post process
results.clear();
{

float* output0 = k_outputs[0];
float sum = 0.0;
for (int i = 0; i < labels.size(); i++){
sum += exp(output0[i]);
}

int max_index;
for (int i = 0; i < labels.size(); i++)
{
output0[i] = exp(output0[i]) / sum;
}
max_index = std::max_element(output0,output0+labels.size()) - output0;
cls_res b;
if (output0[max_index] >= cls_thresh)
{
b.label = labels[max_index];
b.score = output0[max_index];
results.push_back(b);
}
}
/*****************************************************************************/

// draw result to vo
{
{
cv::Mat osd_frame(osd_height, osd_width, CV_8UC4, cv::Scalar(0, 0, 0, 0));
{
/***************************unfixed:不同AI Demo可能需要修改******************/
//draw cls
double fontsize = (osd_frame.cols * osd_frame.rows * 1.0) / (1100 * 1200);
for(int i = 0; i < results.size(); i++)
{
std::string text = "class: " + results[i].label + ", score: " + std::to_string(round(results[i].score * 100) / 100.0).substr(0, 4);

cv::putText(osd_frame, text, cv::Point(1, 40), cv::FONT_HERSHEY_SIMPLEX, 0.8, cv::Scalar(255, 255, 255, 0), 2);

std::cout << text << std::endl;
}
/*****************************************************************************/
}

/***************************fixed:无需修改***********************************/
memcpy(pic_vaddr, osd_frame.data, osd_width * osd_height * 4);
}

// insert osd to vo
{
kd_mpi_vo_chn_insert_frame(osd_id+3, &vf_info); //K_VO_OSD0
printf("kd_mpi_vo_chn_insert_frame success \n");
}

}

{
// 释放sensor当前帧
ret = kd_mpi_vicap_dump_release(vicap_dev, VICAP_CHN_ID_1, &dump_info);
if (ret) {
printf("sample_vicap...kd_mpi_vicap_dump_release failed.\n");
}
}
/*****************************************************************************/
}

/***************************fixed:无需修改***********************************/
vo_osd_release_block();
vivcap_stop();

// free memory
ret = kd_mpi_sys_mmz_free(paddr, vaddr);
if (ret)
{
std::cerr << "free failed: ret = " << ret << ", errno = " << strerror(errno) << std::endl;
std::abort();
}
/*****************************************************************************/
}

int main(int argc, char *argv[])
{
std::cout << "case " << argv[0] << " built at " << __DATE__ << " " << __TIME__ << std::endl;
if (argc != 2)
{
cout << "模型推理时传参说明:"
<< "<kmodel_path>" << endl
<< "Options:" << endl
<< " kmodel_path Kmodel的路径\n"
<< "\n"
<< endl;
return -1;
}

/***************************fixed:无需修改***********************************/
std::thread thread_isp(video_proc_cls, argv);
while (getchar() != 'q')
{
usleep(10000);
}

isp_stop = true;
thread_isp.join();
/*****************************************************************************/
return 0;
}

5.2 基于ulab(MicroPython)的AI推理流程

基于ulab(MicroPython)的K230 AI推理,简要介绍了使用MicroPython语言实现的视频采集,图像预处理(ulab),模型推理、后处理(ulab)、显示等过程。

5.2.1 视频采集

视频采集:(视频输入,VI)与摄像头相关,本小节简要介绍基于Micropython的摄像头设置、摄像头启动、从摄像头中获取一帧数据、摄像头停止的整体流程。

创建摄像头sensor对象,并设置多路输出格式和大小

  • 设置的sensor两路输出;
  • 一路输出用于显示,输出大小设置1080p,图像格式为PIXEL_FORMAT_YVU_PLANAR_420,直接绑定到vo;
  • 另一路输出用于AI计算,输出大小(1280,720),图像格式为PIXEL_FORMAT_RGB_888_PLANAR;
import os
from media.sensor import * #导入sensor模块,使用sensor相关接口
from media.display import * #导入display模块,使用display相关接口
from media.media import * #导入media模块,使用meida相关接口
from time import *
import gc

# 显示输出大小
DISPLAY_WIDTH = ALIGN_UP(1920, 16)
DISPLAY_HEIGHT = 1080
# AI获取的RGB888P的图像大小
OUT_RGB888P_WIDTH = ALIGN_UP(1280, 16)
OUT_RGB888P_HEIGH = 720

def sensor_init():
# 初始化并配置sensor
sensor = Sensor()
sensor.reset()
# 设置镜像
sensor.set_hmirror(False)
# 设置翻转
sensor.set_vflip(False)
# 通道0直接给到显示VO,格式为YUV420
sensor.set_framesize(width = DISPLAY_WIDTH, height = DISPLAY_HEIGHT)
sensor.set_pixformat(PIXEL_FORMAT_YUV_SEMIPLANAR_420)
# 通道2给到AI做算法处理,格式为RGB888
sensor.set_framesize(width = OUT_RGB888P_WIDTH , height = OUT_RGB888P_HEIGH, chn=CAM_CHN_ID_2)
sensor.set_pixformat(PIXEL_FORMAT_RGB_888_PLANAR, chn=CAM_CHN_ID_2)
# 绑定通道0的输出到vo
sensor_bind_info = sensor.bind_info(x = 0, y = 0, chn = CAM_CHN_ID_0)
Display.bind_layer(**sensor_bind_info, layer = Display.LAYER_VIDEO1)
# 设置为LT9611显示,默认1920x1080
Display.init(Display.LT9611, to_ide = True)
try:
# media初始化
MediaManager.init()
# 启动sensor
sensor.run()
rgb888p_img = None
while True:
#捕获摄像头数据
rgb888p_img = sensor.snapshot(chn=CAM_CHN_ID_2)
#************拿到一帧图像后可以进行后续AI过程************
# ......
#***************************************************
except Exception as e:
print(f"An error occurred during running: {e}")
finally:
os.exitpoint(os.EXITPOINT_ENABLE_SLEEP)
#停止摄像头输出
sensor.stop()
#去初始化显示设备
Display.deinit()
#释放媒体缓冲区
MediaManager.deinit()
gc.collect()
nn.shrink_memory_pool()
return 0

sensor_init()

5.2.2 预处理

import os
from media.sensor import * #导入camera模块,使用camera相关接口
from media.display import * #导入display模块,使用display相关接口
from media.media import * #导入media模块,使用meida相关接口
from time import *
import time
import nncase_runtime as nn #导入nn模块,使用nn相关接口
import ulab.numpy as np #导入np模块,使用np相关接口
import gc

# 显示输出大小
DISPLAY_WIDTH = ALIGN_UP(1920, 16)
DISPLAY_HEIGHT = 1080
# AI获取的RGB888P的图像大小
OUT_RGB888P_WIDTH = ALIGN_UP(1280, 16)
OUT_RGB888P_HEIGH = 720

def cls_test():
print("cls_test start")
#初始化AI2D
ai2d = nn.ai2d()
# 设置ai2d的输入输出数据格式和数据类型
ai2d.set_dtype(nn.ai2d_format.NCHW_FMT,nn.ai2d_format.NCHW_FMT,np.uint8, np.uint8)
# 设置resize预处理
ai2d.set_resize_param(True, nn.interp_method.tf_bilinear, nn.interp_mode.half_pixel )
# 构建预处理,参数为输入输出tensor的shape
ai2d_builder = ai2d.build([1,3,OUT_RGB888P_HEIGH,OUT_RGB888P_WIDTH], [1,3,224,224])

# 初始化并配置sensor
sensor = Sensor()
sensor.reset()
# 设置镜像
sensor.set_hmirror(False)
# 设置翻转
sensor.set_vflip(False)
# 通道0直接给到显示VO,格式为YUV420
sensor.set_framesize(width = DISPLAY_WIDTH, height = DISPLAY_HEIGHT)
sensor.set_pixformat(PIXEL_FORMAT_YUV_SEMIPLANAR_420)
# 通道2给到AI做算法处理,格式为RGB888
sensor.set_framesize(width = OUT_RGB888P_WIDTH , height = OUT_RGB888P_HEIGH, chn=CAM_CHN_ID_2)
sensor.set_pixformat(PIXEL_FORMAT_RGB_888_PLANAR, chn=CAM_CHN_ID_2)
# 绑定通道0的输出到vo
sensor_bind_info = sensor.bind_info(x = 0, y = 0, chn = CAM_CHN_ID_0)
Display.bind_layer(**sensor_bind_info, layer = Display.LAYER_VIDEO1)
# 设置为LT9611显示,默认1920x1080
Display.init(Display.LT9611, to_ide = True)

try:
# media初始化
MediaManager.init()
# 启动sensor
sensor.run()
rgb888p_img = None
while True:
#捕获摄像头数据
rgb888p_img = sensor.snapshot(chn=CAM_CHN_ID_2)
# for rgb888planar
if rgb888p_img.format() == image.RGBP888:
# image转numpy.ndarray
ai2d_input = rgb888p_img.to_numpy_ref()
# 从numpy.ndarray创建tensor
ai2d_input_tensor = nn.from_numpy(ai2d_input)
# 初始化预处理输出
data = np.ones((1,3,224,224),dtype=np.uint8)
# numpy.ndarray转tensor
ai2d_out = nn.from_numpy(data)
# 执行预处理
ai2d_builder.run(ai2d_input_tensor, ai2d_out)
# tensor转numpy.ndarray
ai2d_out_np=ai2d_out.to_numpy()
# 打印预处理形状
print(ai2d_out_np.shape)
except Exception as e:
print(f"An error occurred during running: {e}")
finally:
os.exitpoint(os.EXITPOINT_ENABLE_SLEEP)
#停止摄像头输出
sensor.stop()
#去初始化显示设备
Display.deinit()
#释放媒体缓冲区
MediaManager.deinit()
gc.collect()
time.sleep(1)
nn.shrink_memory_pool()
print("cls_test end")
return 0

cls_test()

:对于该分类模型来说,需要先拿到原图,再将原图resize到(224,224)大小,之后再喂给模型;对于micropython开发,resize可以使用ai2d模块实现。

我们也可以将喂给AI的摄像头数据分辨率直接设置为(224,224),代码如下:

import os
from media.sensor import * #导入camera模块,使用camera相关接口
from media.display import * #导入display模块,使用display相关接口
from media.media import * #导入media模块,使用meida相关接口
from time import *
import time
import nncase_runtime as nn #导入nn模块,使用nn相关接口
import ulab.numpy as np #导入np模块,使用np相关接口
import gc

DISPLAY_WIDTH = ALIGN_UP(1920, 16)
DISPLAY_HEIGHT = 1080

OUT_RGB888P_WIDTH = ALIGN_UP(224, 16)
OUT_RGB888P_HEIGH = 224

def cls_test():
print("cls_test start")
# 初始化并配置sensor
sensor = Sensor()
sensor.reset()
# 设置镜像
sensor.set_hmirror(False)
# 设置翻转
sensor.set_vflip(False)
# 通道0直接给到显示VO,格式为YUV420
sensor.set_framesize(width = DISPLAY_WIDTH, height = DISPLAY_HEIGHT)
sensor.set_pixformat(PIXEL_FORMAT_YUV_SEMIPLANAR_420)
# 通道2给到AI做算法处理,格式为RGB888
sensor.set_framesize(width = OUT_RGB888P_WIDTH , height = OUT_RGB888P_HEIGH, chn=CAM_CHN_ID_2)
sensor.set_pixformat(PIXEL_FORMAT_RGB_888_PLANAR, chn=CAM_CHN_ID_2)
# 绑定通道0的输出到vo
sensor_bind_info = sensor.bind_info(x = 0, y = 0, chn = CAM_CHN_ID_0)
Display.bind_layer(**sensor_bind_info, layer = Display.LAYER_VIDEO1)
# 设置为LT9611显示,默认1920x1080
Display.init(Display.LT9611, to_ide = True)

try:
# media初始化
MediaManager.init()
# 启动sensor
sensor.run()
rgb888p_img = None
while True:
#捕获摄像头数据
rgb888p_img = sensor.snapshot(chn=CAM_CHN_ID_2)
# for rgb888planar
if rgb888p_img.format() == image.RGBP888:
input_img=rgb888p_img.to_numpy_ref()
# 打印形状
print(input_img.shape)
except Exception as e:
print(f"An error occurred during running: {e}")
finally:
os.exitpoint(os.EXITPOINT_ENABLE_SLEEP)
#停止摄像头输出
sensor.stop()
#去初始化显示设备
Display.deinit()
#释放媒体缓冲区
MediaManager.deinit()
gc.collect()
time.sleep(1)
nn.shrink_memory_pool()
print("cls_test end")
return 0

cls_test()

5.2.3 模型推理

import os
from media.sensor import * #导入camera模块,使用camera相关接口
from media.display import * #导入display模块,使用display相关接口
from media.media import * #导入media模块,使用meida相关接口
from time import *
import time
import nncase_runtime as nn #导入nn模块,使用nn相关接口
import ulab.numpy as np #导入np模块,使用np相关接口
import gc

# 显示输出大小
DISPLAY_WIDTH = ALIGN_UP(1920, 16)
DISPLAY_HEIGHT = 1080
# AI获取的RGB888P的图像大小
OUT_RGB888P_WIDTH = ALIGN_UP(1280, 16)
OUT_RGB888P_HEIGH = 720

kmodel_path="/sdcard/app/tests/ai_test_kmodel/veg_cls.kmodel"

def cls_test():
print("cls_test start")
#初始化AI2D
ai2d = nn.ai2d()
# 设置ai2d的输入输出数据格式和数据类型
ai2d.set_dtype(nn.ai2d_format.NCHW_FMT,nn.ai2d_format.NCHW_FMT,np.uint8, np.uint8)
# 设置resize预处理
ai2d.set_resize_param(True, nn.interp_method.tf_bilinear, nn.interp_mode.half_pixel )
# 构建预处理,参数为输入输出tensor的shape
ai2d_builder = ai2d.build([1,3,OUT_RGB888P_HEIGH,OUT_RGB888P_WIDTH], [1,3,224,224])
# 初始化kpu
kpu=nn.kpu()
# 加载模型
kpu.load_kmodel(kmodel_path)

# 初始化并配置sensor
sensor = Sensor()
sensor.reset()
# 设置镜像
sensor.set_hmirror(False)
# 设置翻转
sensor.set_vflip(False)
# 通道0直接给到显示VO,格式为YUV420
sensor.set_framesize(width = DISPLAY_WIDTH, height = DISPLAY_HEIGHT)
sensor.set_pixformat(PIXEL_FORMAT_YUV_SEMIPLANAR_420)
# 通道2给到AI做算法处理,格式为RGB888
sensor.set_framesize(width = OUT_RGB888P_WIDTH , height = OUT_RGB888P_HEIGH, chn=CAM_CHN_ID_2)
sensor.set_pixformat(PIXEL_FORMAT_RGB_888_PLANAR, chn=CAM_CHN_ID_2)
# 绑定通道0的输出到vo
sensor_bind_info = sensor.bind_info(x = 0, y = 0, chn = CAM_CHN_ID_0)
Display.bind_layer(**sensor_bind_info, layer = Display.LAYER_VIDEO1)
# 设置为LT9611显示,默认1920x1080
Display.init(Display.LT9611, to_ide = True)

try:
# media初始化
MediaManager.init()
# 启动sensor
sensor.run()
rgb888p_img = None

# 初始化预处理输出
data = np.ones((1,3,224,224),dtype=np.uint8)
# numpy.ndarray转tensor
ai2d_out = nn.from_numpy(data)
# 将ai2d预处理的输出绑定到kmodel的输入上,即ai2d预处理输出和kpu的输出共用一个tensor
kpu.set_input_tensor(0,ai2d_out)
while True:
#捕获摄像头数据
rgb888p_img = sensor.snapshot(chn=CAM_CHN_ID_2)
# for rgb888planar
if rgb888p_img.format() == image.RGBP888:
# image转numpy.ndarray
ai2d_input = rgb888p_img.to_numpy_ref()
# 从numpy.ndarray创建tensor
ai2d_input_tensor = nn.from_numpy(ai2d_input)
# 执行预处理
ai2d_builder.run(ai2d_input_tensor, ai2d_out)
# kpu推理
kpu.run()

# 获取kmodel的推理输出tensor,输出可能为多个,因此返回的是一个列表
results=[]
for i in range(kpu.outputs_size()):
# 获取kmodel的第i个输出
output_data = kpu.get_output_tensor(i)
# tensor转numpy.ndarray
result = output_data.to_numpy()
# 打印形状
print(result.shape)
# 加入列表
results.append(result)
del output_data

except Exception as e:
print(f"An error occurred during running: {e}")
finally:
os.exitpoint(os.EXITPOINT_ENABLE_SLEEP)
del ai2d_input_tensor
del ai2d_out
#停止摄像头输出
sensor.stop()
#去初始化显示设备
Display.deinit()
#释放媒体缓冲区
MediaManager.deinit()
gc.collect()
time.sleep(1)
nn.shrink_memory_pool()
print("cls_test end")
return 0

cls_test()

注意:对于tensor类型的数据,申请后请手动释放。

5.2.4 后处理

对模型结果进行后处理,这里以分类为例,即对输出先做softmax再做argmax得到类别索引。

import os
from media.sensor import * #导入camera模块,使用camera相关接口
from media.display import * #导入display模块,使用display相关接口
from media.media import * #导入media模块,使用meida相关接口
from time import *
import time
import nncase_runtime as nn #导入nn模块,使用nn相关接口
import ulab.numpy as np #导入np模块,使用np相关接口
import gc

# 显示输出大小
DISPLAY_WIDTH = ALIGN_UP(1920, 16)
DISPLAY_HEIGHT = 1080
# AI获取的RGB888P的图像大小
OUT_RGB888P_WIDTH = ALIGN_UP(1280, 16)
OUT_RGB888P_HEIGH = 720

kmodel_path="/sdcard/app/tests/ai_test_kmodel/veg_cls.kmodel"

# softmax函数
def softmax(x):
exp_x = np.exp(x - np.max(x))
return exp_x / np.sum(exp_x)

def cls_test():
print("cls_test start")
#初始化AI2D
ai2d = nn.ai2d()
# 设置ai2d的输入输出数据格式和数据类型
ai2d.set_dtype(nn.ai2d_format.NCHW_FMT,nn.ai2d_format.NCHW_FMT,np.uint8, np.uint8)
# 设置resize预处理
ai2d.set_resize_param(True, nn.interp_method.tf_bilinear, nn.interp_mode.half_pixel )
# 构建预处理,参数为输入输出tensor的shape
ai2d_builder = ai2d.build([1,3,OUT_RGB888P_HEIGH,OUT_RGB888P_WIDTH], [1,3,224,224])
# 初始化kpu
kpu=nn.kpu()
# 加载模型
kpu.load_kmodel(kmodel_path)

# 初始化并配置sensor
sensor = Sensor()
sensor.reset()
# 设置镜像
sensor.set_hmirror(False)
# 设置翻转
sensor.set_vflip(False)
# 通道0直接给到显示VO,格式为YUV420
sensor.set_framesize(width = DISPLAY_WIDTH, height = DISPLAY_HEIGHT)
sensor.set_pixformat(PIXEL_FORMAT_YUV_SEMIPLANAR_420)
# 通道2给到AI做算法处理,格式为RGB888
sensor.set_framesize(width = OUT_RGB888P_WIDTH , height = OUT_RGB888P_HEIGH, chn=CAM_CHN_ID_2)
sensor.set_pixformat(PIXEL_FORMAT_RGB_888_PLANAR, chn=CAM_CHN_ID_2)
# 绑定通道0的输出到vo
sensor_bind_info = sensor.bind_info(x = 0, y = 0, chn = CAM_CHN_ID_0)
Display.bind_layer(**sensor_bind_info, layer = Display.LAYER_VIDEO1)
# 设置为LT9611显示,默认1920x1080
Display.init(Display.LT9611, to_ide = True)

try:
# media初始化
MediaManager.init()
# 启动sensor
sensor.run()
rgb888p_img = None

# 初始化预处理输出
data = np.ones((1,3,224,224),dtype=np.uint8)
# numpy.ndarray转tensor
ai2d_out = nn.from_numpy(data)
# 将ai2d预处理的输出绑定到kmodel的输入上,即ai2d预处理输出和kpu的输出共用一个tensor
kpu.set_input_tensor(0,ai2d_out)
while True:
#捕获摄像头数据
rgb888p_img = sensor.snapshot(chn=CAM_CHN_ID_2)
# for rgb888planar
if rgb888p_img.format() == image.RGBP888:
# image转numpy.ndarray
ai2d_input = rgb888p_img.to_numpy_ref()
# 从numpy.ndarray创建tensor
ai2d_input_tensor = nn.from_numpy(ai2d_input)
# 执行预处理
ai2d_builder.run(ai2d_input_tensor, ai2d_out)
# kpu推理
kpu.run()

# 获取kmodel的推理输出tensor,输出可能为多个,因此返回的是一个列表
results=[]
for i in range(kpu.outputs_size()):
# 获取kmodel的第i个输出
output_data = kpu.get_output_tensor(i)
# tensor转numpy.ndarray
result = output_data.to_numpy()
# 加入列表
results.append(result)
del output_data
#******************后处理********************
# softmax+argmax
softmax_res=softmax(results[0][0])
res_idx=np.argmax(softmax_res)
# 打印类别索引
print(res_idx)
except Exception as e:
print(f"An error occurred during running: {e}")
finally:
os.exitpoint(os.EXITPOINT_ENABLE_SLEEP)
del ai2d_input_tensor
del ai2d_out
#停止摄像头输出
sensor.stop()
#去初始化显示设备
Display.deinit()
#释放媒体缓冲区
MediaManager.deinit()
gc.collect()
time.sleep(1)
nn.shrink_memory_pool()
print("cls_test end")
return 0

cls_test()

5.2.5 显示

显示(视频输出,VO)与display相关,本小节简要介绍基于Micropython的显示设置、资源释放、显示叠加的整体流程。

  • 显示设置:设置显示大小,格式
  • 显示叠加:显示由2个图层构成,其中下边的图层(原图图层)直接显示摄像头输出,上边的图层(osd图层)用于画框、画点,写文字等。
  • 资源释放:释放显示相关资源

显示示例

import os
from media.sensor import * #导入camera模块,使用camera相关接口
from media.display import * #导入display模块,使用display相关接口
from media.media import * #导入media模块,使用meida相关接口
from time import *
import time
import nncase_runtime as nn #导入nn模块,使用nn相关接口
import ulab.numpy as np #导入np模块,使用np相关接口
import gc

# 显示输出大小
DISPLAY_WIDTH = ALIGN_UP(1920, 16)
DISPLAY_HEIGHT = 1080
# AI获取的RGB888P的图像大小
OUT_RGB888P_WIDTH = ALIGN_UP(1280, 16)
OUT_RGB888P_HEIGH = 720

kmodel_path="/sdcard/app/tests/ai_test_kmodel/veg_cls.kmodel"
labels=["菠菜","长茄子","红苋菜","胡萝卜","西红柿","西蓝花"]

# softmax函数
def softmax(x):
exp_x = np.exp(x - np.max(x))
return exp_x / np.sum(exp_x)


def cls_test():
print("cls_test start")
#初始化AI2D
ai2d = nn.ai2d()
# 设置ai2d的输入输出数据格式和数据类型
ai2d.set_dtype(nn.ai2d_format.NCHW_FMT,nn.ai2d_format.NCHW_FMT,np.uint8, np.uint8)
# 设置resize预处理
ai2d.set_resize_param(True, nn.interp_method.tf_bilinear, nn.interp_mode.half_pixel )
# 构建预处理,参数为输入输出tensor的shape
ai2d_builder = ai2d.build([1,3,OUT_RGB888P_HEIGH,OUT_RGB888P_WIDTH], [1,3,224,224])
# 初始化kpu
kpu=nn.kpu()
# 加载模型
kpu.load_kmodel(kmodel_path)

# 初始化并配置sensor
sensor = Sensor()
sensor.reset()
# 设置镜像
sensor.set_hmirror(False)
# 设置翻转
sensor.set_vflip(False)
# 通道0直接给到显示VO,格式为YUV420
sensor.set_framesize(width = DISPLAY_WIDTH, height = DISPLAY_HEIGHT)
sensor.set_pixformat(PIXEL_FORMAT_YUV_SEMIPLANAR_420)
# 通道2给到AI做算法处理,格式为RGB888
sensor.set_framesize(width = OUT_RGB888P_WIDTH , height = OUT_RGB888P_HEIGH, chn=CAM_CHN_ID_2)
sensor.set_pixformat(PIXEL_FORMAT_RGB_888_PLANAR, chn=CAM_CHN_ID_2)
# 绑定通道0的输出到vo
sensor_bind_info = sensor.bind_info(x = 0, y = 0, chn = CAM_CHN_ID_0)
Display.bind_layer(**sensor_bind_info, layer = Display.LAYER_VIDEO1)
# 设置为LT9611显示,默认1920x1080
Display.init(Display.LT9611, to_ide = True)

#创建OSD图像
osd_img = image.Image(DISPLAY_WIDTH, DISPLAY_HEIGHT, image.ARGB8888)

try:
# media初始化
MediaManager.init()
# 启动sensor
sensor.run()
rgb888p_img = None

# 初始化预处理输出
data = np.ones((1,3,224,224),dtype=np.uint8)
# numpy.ndarray转tensor
ai2d_out = nn.from_numpy(data)
# 将ai2d预处理的输出绑定到kmodel的输入上,即ai2d预处理输出和kpu的输出共用一个tensor
kpu.set_input_tensor(0,ai2d_out)
while True:
#捕获摄像头数据
rgb888p_img = sensor.snapshot(chn=CAM_CHN_ID_2)
# for rgb888planar
if rgb888p_img.format() == image.RGBP888:
#*************************预处理***********************
# image转numpy.ndarray
ai2d_input = rgb888p_img.to_numpy_ref()
# 从numpy.ndarray创建tensor
ai2d_input_tensor = nn.from_numpy(ai2d_input)
# 执行预处理
ai2d_builder.run(ai2d_input_tensor, ai2d_out)
#************************模型推理**********************
# kpu推理
kpu.run()

# 获取kmodel的推理输出tensor,输出可能为多个,因此返回的是一个列表
results=[]
for i in range(kpu.outputs_size()):
# 获取kmodel的第i个输出
output_data = kpu.get_output_tensor(i)
# tensor转numpy.ndarray
result = output_data.to_numpy()
# 加入列表
results.append(result)
del output_data
#******************后处理********************
# softmax+argmax
softmax_res=softmax(results[0][0])
res_idx=np.argmax(softmax_res)
#******************显示*********************
# 类别名称
label=labels[res_idx]
# 类别分数
score=softmax_res[res_idx]
# 绘制文字
text=label+" "+str(score)
# 清理osd_img
osd_img.clear()
# 调用draw_string_advanced()在(5,5)位置,绘制text,文字大小为32,颜色为(0,255,0)
osd_img.draw_string_advanced(5,5,32,text,color=(0,255,0))
Display.show_image(osd_img, 0, 0, Display.LAYER_OSD3)
except Exception as e:
print(f"An error occurred during running: {e}")
finally:
os.exitpoint(os.EXITPOINT_ENABLE_SLEEP)
del ai2d_input_tensor
del ai2d_out
del kpu
#停止摄像头输出
sensor.stop()
#去初始化显示设备
Display.deinit()
#释放媒体缓冲区
MediaManager.deinit()
gc.collect()
time.sleep(1)
nn.shrink_memory_pool()
print("cls_test end")
return 0

cls_test()

5.2.6 资源释放

大核内存包括两个部分,一个是系统内存,一个是 GC 内存,前者主要用来给模型还有系统内的一些功能使用,包括摄像头和屏幕的缓冲区、kmodel及其输入输出都来自这里(mmz,用del释放);后者是解析器层面申请的内存,可以给代码的变量使用(用gc.collect()释放)。

1. gc资源释放

#classification.py
import gc
import os

def func_a():
a = []
for i in range(10000):
a.append(i)

func_a()
print(gc.mem_free() / 1024 / 1024) #stack mem
print(gc.mem_alloc() / 1024 / 1024)
gc.collect()
print(gc.mem_alloc() / 1024 / 1024)

2.系统内存释放

kmodel模块部分,kmodel的input_tensor、output_tensor,及其本身都是mmz申请的内存,需要手动释放

del ai2d_input_tensor
del ai2d_out
del kpu
nn.shrink_memory_pool() #以免漏掉某个del,遍历所有kmodel相关内存,并释放

摄像头、显示及其media缓冲区释放:

#停止摄像头输出
sensor.stop()
#去初始化显示设备
Display.deinit()
#释放媒体缓冲区
MediaManager.deinit()

5.2.7 完整代码

这里给出完整代码,请参考上述每一步代码之间的变化。对于AI开发的步骤我们进行了封装,相关类在/sdcard/app/libs下面,开发步骤请参考:AI Demo说明文档 — K230 CanMV 文档 (canaan-creative.com)

import os
from media.sensor import * #导入camera模块,使用camera相关接口
from media.display import * #导入display模块,使用display相关接口
from media.media import * #导入media模块,使用meida相关接口
from time import *
import time
import nncase_runtime as nn #导入nn模块,使用nn相关接口
import ulab.numpy as np #导入np模块,使用np相关接口
import gc

# 显示输出大小
DISPLAY_WIDTH = ALIGN_UP(1920, 16)
DISPLAY_HEIGHT = 1080
# AI获取的RGB888P的图像大小
OUT_RGB888P_WIDTH = ALIGN_UP(1280, 16)
OUT_RGB888P_HEIGH = 720

kmodel_path="/sdcard/app/tests/ai_test_kmodel/veg_cls.kmodel"
labels=["菠菜","长茄子","红苋菜","胡萝卜","西红柿","西蓝花"]

# softmax函数
def softmax(x):
exp_x = np.exp(x - np.max(x))
return exp_x / np.sum(exp_x)


def cls_test():
print("cls_test start")
#初始化AI2D
ai2d = nn.ai2d()
# 设置ai2d的输入输出数据格式和数据类型
ai2d.set_dtype(nn.ai2d_format.NCHW_FMT,nn.ai2d_format.NCHW_FMT,np.uint8, np.uint8)
# 设置resize预处理
ai2d.set_resize_param(True, nn.interp_method.tf_bilinear, nn.interp_mode.half_pixel )
# 构建预处理,参数为输入输出tensor的shape
ai2d_builder = ai2d.build([1,3,OUT_RGB888P_HEIGH,OUT_RGB888P_WIDTH], [1,3,224,224])
# 初始化kpu
kpu=nn.kpu()
# 加载模型
kpu.load_kmodel(kmodel_path)

# 初始化并配置sensor
sensor = Sensor()
sensor.reset()
# 设置镜像
sensor.set_hmirror(False)
# 设置翻转
sensor.set_vflip(False)
# 通道0直接给到显示VO,格式为YUV420
sensor.set_framesize(width = DISPLAY_WIDTH, height = DISPLAY_HEIGHT)
sensor.set_pixformat(PIXEL_FORMAT_YUV_SEMIPLANAR_420)
# 通道2给到AI做算法处理,格式为RGB888
sensor.set_framesize(width = OUT_RGB888P_WIDTH , height = OUT_RGB888P_HEIGH, chn=CAM_CHN_ID_2)
sensor.set_pixformat(PIXEL_FORMAT_RGB_888_PLANAR, chn=CAM_CHN_ID_2)
# 绑定通道0的输出到vo
sensor_bind_info = sensor.bind_info(x = 0, y = 0, chn = CAM_CHN_ID_0)
Display.bind_layer(**sensor_bind_info, layer = Display.LAYER_VIDEO1)
# 设置为LT9611显示,默认1920x1080
Display.init(Display.LT9611, to_ide = True)

#创建OSD图像
osd_img = image.Image(DISPLAY_WIDTH, DISPLAY_HEIGHT, image.ARGB8888)

try:
# media初始化
MediaManager.init()
# 启动sensor
sensor.run()
rgb888p_img = None

# 初始化预处理输出
data = np.ones((1,3,224,224),dtype=np.uint8)
# numpy.ndarray转tensor
ai2d_out = nn.from_numpy(data)
# 将ai2d预处理的输出绑定到kmodel的输入上,即ai2d预处理输出和kpu的输出共用一个tensor
kpu.set_input_tensor(0,ai2d_out)
while True:
#捕获摄像头数据
rgb888p_img = sensor.snapshot(chn=CAM_CHN_ID_2)
# for rgb888planar
if rgb888p_img.format() == image.RGBP888:
#*************************预处理***********************
# image转numpy.ndarray
ai2d_input = rgb888p_img.to_numpy_ref()
# 从numpy.ndarray创建tensor
ai2d_input_tensor = nn.from_numpy(ai2d_input)
# 执行预处理
ai2d_builder.run(ai2d_input_tensor, ai2d_out)
#************************模型推理**********************
# kpu推理
kpu.run()

# 获取kmodel的推理输出tensor,输出可能为多个,因此返回的是一个列表
results=[]
for i in range(kpu.outputs_size()):
# 获取kmodel的第i个输出
output_data = kpu.get_output_tensor(i)
# tensor转numpy.ndarray
result = output_data.to_numpy()
# 加入列表
results.append(result)
del output_data
#******************后处理********************
# softmax+argmax
softmax_res=softmax(results[0][0])
res_idx=np.argmax(softmax_res)
#******************显示*********************
# 类别名称
label=labels[res_idx]
# 类别分数
score=softmax_res[res_idx]
# 绘制文字
text=label+" "+str(score)
# 清理osd_img
osd_img.clear()
# 调用draw_string_advanced()在(5,5)位置,绘制text,文字大小为32,颜色为(0,255,0)
osd_img.draw_string_advanced(5,5,32,text,color=(0,255,0))
Display.show_image(osd_img, 0, 0, Display.LAYER_OSD3)
except Exception as e:
print(f"An error occurred during running: {e}")
finally:
os.exitpoint(os.EXITPOINT_ENABLE_SLEEP)
del ai2d_input_tensor
del ai2d_out
#停止摄像头输出
sensor.stop()
#去初始化显示设备
Display.deinit()
#释放媒体缓冲区
MediaManager.deinit()
gc.collect()
time.sleep(1)
print("cls_test end")
return 0

cls_test()