diff --git a/APP_Framework/Applications/knowing_app/Kconfig b/APP_Framework/Applications/knowing_app/Kconfig index ece7acb3..8e0305b6 100755 --- a/APP_Framework/Applications/knowing_app/Kconfig +++ b/APP_Framework/Applications/knowing_app/Kconfig @@ -1,4 +1,6 @@ menu "knowing app" source "$APP_DIR/Applications/knowing_app/mnist/Kconfig" source "$APP_DIR/Applications/knowing_app/face_detect/Kconfig" + source "$APP_DIR/Applications/knowing_app/instrusion_detect/Kconfig" + source "$APP_DIR/Applications/knowing_app/helmet_detect/Kconfig" endmenu diff --git a/APP_Framework/Applications/knowing_app/face_detect/detect.json b/APP_Framework/Applications/knowing_app/face_detect/detect.json index f3d181ef..b59731aa 100644 --- a/APP_Framework/Applications/knowing_app/face_detect/detect.json +++ b/APP_Framework/Applications/knowing_app/face_detect/detect.json @@ -26,6 +26,11 @@ ], "kmodel_path": "/kmodel/detect.kmodel", "kmodel_size": 388776, - "obj_thresh": 0.7, + "obj_thresh": [ + 0.7 + ], + "labels": [ + "face" + ], "nms_thresh": 0.3 } \ No newline at end of file diff --git a/APP_Framework/Applications/knowing_app/face_detect/face_detect.c b/APP_Framework/Applications/knowing_app/face_detect/face_detect.c index 3ce3319a..dc9c6568 100644 --- a/APP_Framework/Applications/knowing_app/face_detect/face_detect.c +++ b/APP_Framework/Applications/knowing_app/face_detect/face_detect.c @@ -8,24 +8,17 @@ #define JSON_FILE_PATH "/kmodel/detect.json" #define JSON_BUFFER_SIZE (4 * 1024) -// params from json -float anchor[ANCHOR_NUM * 2] = {}; -int net_output_shape[3] = {}; -int net_input_size[2] = {}; -int sensor_output_size[2] = {}; -char kmodel_path[127] = ""; -int kmodel_size = 0; -float obj_thresh = 1.0; -float nms_thresh = 0.0; - -// float anchor[ANCHOR_NUM * 2] = {1.889, 2.5245, 2.9465, 3.94056, 3.99987, 5.3658, 5.155437, 6.92275, 6.718375, 9.01025}; -// int net_output_shape[3] = {20, 15, 30}; -// int net_input_size[2] = {240, 320}; -// int sensor_output_size[2] = {240, 320}; -// char kmodel_path[127] = "/kmodel/detect.kmodel"; -// int kmodel_size = 388776; -// float obj_thresh = 0.7; -// float nms_thresh = 0.3; +// params from json +static float anchor[ANCHOR_NUM * 2] = {}; +static int net_output_shape[3] = {}; +static int net_input_size[2] = {}; +static int sensor_output_size[2] = {}; +static char kmodel_path[127] = ""; +static int kmodel_size = 0; +static float obj_thresh[20] = {}; +static float nms_thresh = 0.0; +static char labels[20][32] = {}; +static int class_num = 0; #define THREAD_PRIORITY_FACE_D (11) static pthread_t facetid = 0; @@ -141,10 +134,34 @@ static void param_parse() json_item = cJSON_GetObjectItem(json_obj, "kmodel_size"); kmodel_size = json_item->valueint; printf("Got kmodel_size: %d\n", kmodel_size); + // labels + json_item = cJSON_GetObjectItem(json_obj, "labels"); + class_num = cJSON_GetArraySize(json_item); + if (0 >= class_num) { + printf("No labels!"); + exit(-1); + } else { + printf("Got %d labels\n", class_num); + } + for (int i = 0; i < class_num; i++) { + json_array_item = cJSON_GetArrayItem(json_item, i); + memcpy(labels[i], json_array_item->valuestring, strlen(json_array_item->valuestring)); + printf("%d: %s\n", i, labels[i]); + } // obj_thresh json_item = cJSON_GetObjectItem(json_obj, "obj_thresh"); - obj_thresh = json_item->valuedouble; - printf("Got obj_thresh: %f\n", obj_thresh); + array_size = cJSON_GetArraySize(json_item); + if (class_num != array_size) { + printf("label number and thresh number mismatch! label number : %d, obj thresh number %d", class_num, array_size); + exit(-1); + } else { + printf("Got %d obj_thresh\n", array_size); + } + for (int i = 0; i < array_size; i++) { + json_array_item = cJSON_GetArrayItem(json_item, i); + obj_thresh[i] = json_array_item->valuedouble; + printf("%d: %f\n", i, obj_thresh[i]); + } // nms_thresh json_item = cJSON_GetObjectItem(json_obj, "nms_thresh"); nms_thresh = json_item->valuedouble; @@ -231,7 +248,10 @@ void face_detect() } face_detect_rl.anchor_number = ANCHOR_NUM; face_detect_rl.anchor = anchor; - face_detect_rl.threshold = obj_thresh; + face_detect_rl.threshold = malloc(class_num * sizeof(float)); + for (int idx = 0; idx < class_num; idx++) { + face_detect_rl.threshold[idx] = obj_thresh[idx]; + } face_detect_rl.nms_value = nms_thresh; result = region_layer_init(&face_detect_rl, net_output_shape[0], net_output_shape[1], net_output_shape[2], net_input_size[1], net_input_size[0]); @@ -287,9 +307,9 @@ static void *thread_face_detcet_entry(void *parameter) for (int face_cnt = 0; face_cnt < face_detect_info.obj_number; face_cnt++) { draw_edge((uint32_t *)showbuffer, &face_detect_info, face_cnt, 0xF800, (uint16_t)sensor_output_size[1], (uint16_t)sensor_output_size[0]); - printf("%d: (%d, %d, %d, %d) cls: %d conf: %f\t", face_cnt, face_detect_info.obj[face_cnt].x1, + printf("%d: (%d, %d, %d, %d) cls: %s conf: %f\t", face_cnt, face_detect_info.obj[face_cnt].x1, face_detect_info.obj[face_cnt].y1, face_detect_info.obj[face_cnt].x2, face_detect_info.obj[face_cnt].y2, - face_detect_info.obj[face_cnt].class_id, face_detect_info.obj[face_cnt].prob); + labels[face_detect_info.obj[face_cnt].class_id], face_detect_info.obj[face_cnt].prob); } if (0 != face_detect_info.obj_number) printf("\n"); lcd_draw_picture(0, 0, (uint16_t)sensor_output_size[1], (uint16_t)sensor_output_size[0], (unsigned int *)showbuffer); diff --git a/APP_Framework/Applications/knowing_app/helmet_detect/Kconfig b/APP_Framework/Applications/knowing_app/helmet_detect/Kconfig new file mode 100644 index 00000000..3a090133 --- /dev/null +++ b/APP_Framework/Applications/knowing_app/helmet_detect/Kconfig @@ -0,0 +1,8 @@ +config HELMET_DETECT + bool "enable apps/helmet detect" + depends on BOARD_K210_EVB + depends on DRV_USING_OV2640 + depends on USING_KPU_POSTPROCESSING + depends on USING_YOLOV2 + select LIB_USING_CJSON + default n diff --git a/APP_Framework/Applications/knowing_app/helmet_detect/SConscript b/APP_Framework/Applications/knowing_app/helmet_detect/SConscript new file mode 100644 index 00000000..26ac700a --- /dev/null +++ b/APP_Framework/Applications/knowing_app/helmet_detect/SConscript @@ -0,0 +1,9 @@ +from building import * + +cwd = GetCurrentDir() +src = Glob('*.c') + Glob('*.cpp') +CPPPATH = [cwd] + +group = DefineGroup('Applications', src, depend = ['HELMET_DETECT'], LOCAL_CPPPATH = CPPPATH) + +Return('group') diff --git a/APP_Framework/Applications/knowing_app/helmet_detect/helmet.json b/APP_Framework/Applications/knowing_app/helmet_detect/helmet.json new file mode 100644 index 00000000..856dc4fe --- /dev/null +++ b/APP_Framework/Applications/knowing_app/helmet_detect/helmet.json @@ -0,0 +1,38 @@ +{ + "net_input_size": [ + 256, + 256 + ], + "net_output_shape": [ + 8, + 8, + 35 + ], + "sensor_output_size": [ + 256, + 256 + ], + "anchors": [ + 1.0432, + 1.0920, + 0.8391, + 2.1250, + 1.1085, + 2.7463, + 1.3783, + 3.6706, + 2.0491, + 4.6711 + ], + "kmodel_path": "/kmodel/helmet.kmodel", + "kmodel_size": 2714044, + "obj_thresh": [ + 0.85, + 0.6 + ], + "labels": [ + "helmet", + "head" + ], + "nms_thresh": 0.3 +} \ No newline at end of file diff --git a/APP_Framework/Applications/knowing_app/helmet_detect/helmet_detect.c b/APP_Framework/Applications/knowing_app/helmet_detect/helmet_detect.c new file mode 100644 index 00000000..3ea8e2dd --- /dev/null +++ b/APP_Framework/Applications/knowing_app/helmet_detect/helmet_detect.c @@ -0,0 +1,378 @@ +#include +#ifdef LIB_USING_CJSON +#include +#endif +#include "region_layer.h" +#define ANCHOR_NUM 5 +#define STACK_SIZE (128 * 1024) +#define JSON_FILE_PATH "/kmodel/helmet.json" +#define JSON_BUFFER_SIZE (4 * 1024) + +// params from json +static float anchor[ANCHOR_NUM * 2] = {}; +static int net_output_shape[3] = {}; +static int net_input_size[2] = {}; +static int sensor_output_size[2] = {}; +static char kmodel_path[127] = ""; +static int kmodel_size = 0; +static float obj_thresh[20] = {}; +static float nms_thresh = 0.0; +static char labels[20][32] = {}; +static int class_num = 0; + +#define THREAD_PRIORITY_HELMET_D (11) +static pthread_t helmettid = 0; +static void *thread_helmet_detect_entry(void *parameter); +static int g_fd = 0; +static int kmodel_fd = 0; +static int if_exit = 0; +static unsigned char *showbuffer = NULL; +static unsigned char *kpurgbbuffer = NULL; + +static _ioctl_shoot_para shoot_para_t = {0}; +unsigned char *model_data = NULL; // kpu data load memory +unsigned char *model_data_align = NULL; + +kpu_model_context_t helmet_detect_task; +static region_layer_t helmet_detect_rl; +static obj_info_t helmet_detect_info; +volatile uint32_t g_ai_done_flag; + +static void ai_done(void *ctx) { g_ai_done_flag = 1; } + +static void param_parse() +{ + int fin; + char buffer[JSON_BUFFER_SIZE] = ""; + // char *buffer; + // if (NULL != (buffer = (char*)malloc(JSON_BUFFER_SIZE * sizeof(char)))) { + // memset(buffer, 0, JSON_BUFFER_SIZE * sizeof(char)); + // } else { + // printf("Json buffer malloc failed!"); + // exit(-1); + // } + int array_size; + cJSON *json_obj; + cJSON *json_item; + cJSON *json_array_item; + + fin = open(JSON_FILE_PATH, O_RDONLY); + if (!fin) { + printf("Error open file %s", JSON_FILE_PATH); + exit(-1); + } + read(fin, buffer, sizeof(buffer)); + close(fin); + + // read json string + json_obj = cJSON_Parse(buffer); + // free(buffer); + char *json_print_str = cJSON_Print(json_obj); + printf("Json file content: \n%s\n", json_print_str); + cJSON_free(json_print_str); + // get anchors + json_item = cJSON_GetObjectItem(json_obj, "anchors"); + array_size = cJSON_GetArraySize(json_item); + if (ANCHOR_NUM * 2 != array_size) { + printf("Expect anchor size: %d, got %d in json file", ANCHOR_NUM * 2, array_size); + exit(-1); + } else { + printf("Got %d anchors from json file\n", ANCHOR_NUM); + } + for (int i = 0; i < ANCHOR_NUM * 2; i++) { + json_array_item = cJSON_GetArrayItem(json_item, i); + anchor[i] = json_array_item->valuedouble; + printf("%d: %f\n", i, anchor[i]); + } + // net_input_size + json_item = cJSON_GetObjectItem(json_obj, "net_input_size"); + array_size = cJSON_GetArraySize(json_item); + if (2 != array_size) { + printf("Expect net_input_size: %d, got %d in json file", 2, array_size); + exit(-1); + } else { + printf("Got %d net_input_size from json file\n", 2); + } + for (int i = 0; i < 2; i++) { + json_array_item = cJSON_GetArrayItem(json_item, i); + net_input_size[i] = json_array_item->valueint; + printf("%d: %d\n", i, net_input_size[i]); + } + // net_output_shape + json_item = cJSON_GetObjectItem(json_obj, "net_output_shape"); + array_size = cJSON_GetArraySize(json_item); + if (3 != array_size) { + printf("Expect net_output_shape: %d, got %d in json file", 3, array_size); + exit(-1); + } else { + printf("Got %d net_output_shape from json file\n", 3); + } + for (int i = 0; i < 3; i++) { + json_array_item = cJSON_GetArrayItem(json_item, i); + net_output_shape[i] = json_array_item->valueint; + printf("%d: %d\n", i, net_output_shape[i]); + } + // sensor_output_size + json_item = cJSON_GetObjectItem(json_obj, "sensor_output_size"); + array_size = cJSON_GetArraySize(json_item); + if (2 != array_size) { + printf("Expect sensor_output_size: %d, got %d in json file", 2, array_size); + exit(-1); + } else { + printf("Got %d sensor_output_size from json file\n", 2); + } + for (int i = 0; i < 2; i++) { + json_array_item = cJSON_GetArrayItem(json_item, i); + sensor_output_size[i] = json_array_item->valueint; + printf("%d: %d\n", i, sensor_output_size[i]); + } + // kmodel_path + json_item = cJSON_GetObjectItem(json_obj, "kmodel_path"); + memcpy(kmodel_path, json_item->valuestring, strlen(json_item->valuestring)); + printf("Got kmodel_path: %s\n", kmodel_path); + // kmodel_size + json_item = cJSON_GetObjectItem(json_obj, "kmodel_size"); + kmodel_size = json_item->valueint; + printf("Got kmodel_size: %d\n", kmodel_size); + // labels + json_item = cJSON_GetObjectItem(json_obj, "labels"); + class_num = cJSON_GetArraySize(json_item); + if (0 >= class_num) { + printf("No labels!"); + exit(-1); + } else { + printf("Got %d labels\n", class_num); + } + for (int i = 0; i < class_num; i++) { + json_array_item = cJSON_GetArrayItem(json_item, i); + memcpy(labels[i], json_array_item->valuestring, strlen(json_array_item->valuestring)); + printf("%d: %s\n", i, labels[i]); + } + // obj_thresh + json_item = cJSON_GetObjectItem(json_obj, "obj_thresh"); + array_size = cJSON_GetArraySize(json_item); + if (class_num != array_size) { + printf("label number and thresh number mismatch! label number : %d, obj thresh number %d", class_num, array_size); + exit(-1); + } else { + printf("Got %d obj_thresh\n", array_size); + } + for (int i = 0; i < array_size; i++) { + json_array_item = cJSON_GetArrayItem(json_item, i); + obj_thresh[i] = json_array_item->valuedouble; + printf("%d: %f\n", i, obj_thresh[i]); + } + // nms_thresh + json_item = cJSON_GetObjectItem(json_obj, "nms_thresh"); + nms_thresh = json_item->valuedouble; + printf("Got nms_thresh: %f\n", nms_thresh); + + cJSON_Delete(json_obj); + return; +} + +void helmet_detect() +{ + int ret = 0; + int result = 0; + int size = 0; + param_parse(); + g_fd = open("/dev/ov2640", O_RDONLY); + if (g_fd < 0) { + printf("open ov2640 fail !!"); + return; + } + showbuffer = (unsigned char *)malloc(sensor_output_size[0] * sensor_output_size[1] * 2); + if (NULL == showbuffer) { + close(g_fd); + printf("showbuffer apply memory fail !!"); + return; + } + kpurgbbuffer = (unsigned char *)malloc(net_input_size[0] * net_input_size[1] * 3); + if (NULL == kpurgbbuffer) { + close(g_fd); + free(showbuffer); + printf("kpurgbbuffer apply memory fail !!"); + return; + } + model_data = (unsigned char *)malloc(kmodel_size + 255); + if (NULL == model_data) { + free(showbuffer); + free(kpurgbbuffer); + close(g_fd); + printf("model_data apply memory fail !!"); + return; + } + memset(model_data, 0, kmodel_size + 255); + memset(showbuffer, 0, sensor_output_size[0] * sensor_output_size[1] * 2); + memset(kpurgbbuffer, 127, net_input_size[0] * net_input_size[1] * 3); + shoot_para_t.pdata = (unsigned int *)(showbuffer); + shoot_para_t.length = (size_t)(sensor_output_size[0] * sensor_output_size[1] * 2); + /* + load memory + */ + kmodel_fd = open(kmodel_path, O_RDONLY); + if (kmodel_fd < 0) { + printf("open kmodel fail"); + close(g_fd); + free(showbuffer); + free(kpurgbbuffer); + free(model_data); + return; + } else { + size = read(kmodel_fd, model_data, kmodel_size); + if (size != kmodel_size) { + printf("read kmodel error size %d\n", size); + close(g_fd); + close(kmodel_fd); + free(showbuffer); + free(kpurgbbuffer); + free(model_data); + return; + + } else { + printf("read kmodel success \n"); + } + } + unsigned char *model_data_align = (unsigned char *)(((unsigned int)model_data + 255) & (~255)); + dvp_set_ai_addr((uint32_t)(kpurgbbuffer + net_input_size[1] * (net_input_size[0] - sensor_output_size[0])), + (uint32_t)(kpurgbbuffer + net_input_size[1] * (net_input_size[0] - sensor_output_size[0]) + + net_input_size[0] * net_input_size[1]), + (uint32_t)(kpurgbbuffer + net_input_size[0] * net_input_size[1] * 2 + + net_input_size[1] * (net_input_size[0] - sensor_output_size[0]))); + if (kpu_load_kmodel(&helmet_detect_task, model_data_align) != 0) { + printf("\nmodel init error\n"); + close(g_fd); + close(kmodel_fd); + free(showbuffer); + free(kpurgbbuffer); + free(model_data); + return; + } + helmet_detect_rl.anchor_number = ANCHOR_NUM; + helmet_detect_rl.anchor = anchor; + helmet_detect_rl.threshold = malloc(class_num * sizeof(float)); + for (int idx = 0; idx < class_num; idx++) { + helmet_detect_rl.threshold[idx] = obj_thresh[idx]; + } + helmet_detect_rl.nms_value = nms_thresh; + result = region_layer_init(&helmet_detect_rl, net_output_shape[0], net_output_shape[1], net_output_shape[2], + net_input_size[1], net_input_size[0]); + printf("region_layer_init result %d \n\r", result); + size_t stack_size = STACK_SIZE; + pthread_attr_t attr; /* 线程属性 */ + struct sched_param prio; /* 线程优先级 */ + prio.sched_priority = 8; /* 优先级设置为 8 */ + pthread_attr_init(&attr); /* 先使用默认值初始化属性 */ + pthread_attr_setschedparam(&attr, &prio); /* 修改属性对应的优先级 */ + pthread_attr_setstacksize(&attr, stack_size); + + /* 创建线程 1, 属性为 attr，入口函数是 thread_entry，入口函数参数是 1 */ + result = pthread_create(&helmettid, &attr, thread_helmet_detect_entry, NULL); + if (0 == result) { + printf("thread_helmet_detect_entry successfully!\n"); + } else { + printf("thread_helmet_detect_entry failed! error code is %d\n", result); + close(g_fd); + } +} +#ifdef __RT_THREAD_H__ +MSH_CMD_EXPORT(helmet_detect, helmet detect task); +#endif + +static void *thread_helmet_detect_entry(void *parameter) +{ + extern void lcd_draw_picture(uint16_t x1, uint16_t y1, uint16_t width, uint16_t height, uint32_t * ptr); + printf("thread_helmet_detect_entry start!\n"); + int ret = 0; + // sysctl_enable_irq(); + while (1) { + // memset(showbuffer,0,320*240*2); + g_ai_done_flag = 0; + ret = ioctl(g_fd, IOCTRL_CAMERA_START_SHOT, &shoot_para_t); + if (RT_ERROR == ret) { + printf("ov2640 can't wait event flag"); + rt_free(showbuffer); + close(g_fd); + pthread_exit(NULL); + return NULL; + } + kpu_run_kmodel(&helmet_detect_task, kpurgbbuffer, DMAC_CHANNEL5, ai_done, NULL); + while (!g_ai_done_flag) + ; + float *output; + size_t output_size; + kpu_get_output(&helmet_detect_task, 0, (uint8_t **)&output, &output_size); + helmet_detect_rl.input = output; + region_layer_run(&helmet_detect_rl, &helmet_detect_info); +/* display result */ +#ifdef BSP_USING_LCD + for (int helmet_cnt = 0; helmet_cnt < helmet_detect_info.obj_number; helmet_cnt++) { + // draw_edge((uint32_t *)showbuffer, &helmet_detect_info, helmet_cnt, 0xF800, + // (uint16_t)sensor_output_size[1], + // (uint16_t)sensor_output_size[0]); + printf("%d: (%d, %d, %d, %d) cls: %s conf: %f\t", helmet_cnt, helmet_detect_info.obj[helmet_cnt].x1, + helmet_detect_info.obj[helmet_cnt].y1, helmet_detect_info.obj[helmet_cnt].x2, + helmet_detect_info.obj[helmet_cnt].y2, labels[helmet_detect_info.obj[helmet_cnt].class_id], + helmet_detect_info.obj[helmet_cnt].prob); + } + if (0 != helmet_detect_info.obj_number) { + printf("\n"); + } + lcd_draw_picture(0, 0, (uint16_t)sensor_output_size[1], (uint16_t)sensor_output_size[0], (unsigned int *)showbuffer); +#endif + usleep(1); + if (1 == if_exit) { + if_exit = 0; + printf("thread_helmet_detect_entry exit"); + pthread_exit(NULL); + } + } +} + +void helmet_detect_delete() +{ + if (showbuffer != NULL) { + int ret = 0; + close(g_fd); + close(kmodel_fd); + free(showbuffer); + free(kpurgbbuffer); + free(model_data); + printf("helmet detect task cancel!!! ret %d ", ret); + if_exit = 1; + } +} +#ifdef __RT_THREAD_H__ +MSH_CMD_EXPORT(helmet_detect_delete, helmet detect task delete); +#endif + +void kmodel_load(unsigned char *model_data) +{ + int kmodel_fd = 0; + int size = 0; + kmodel_fd = open(kmodel_path, O_RDONLY); + + model_data = (unsigned char *)malloc(kmodel_size + 255); + if (NULL == model_data) { + printf("model_data apply memory fail !!"); + return; + } + memset(model_data, 0, kmodel_size + 255); + + if (kmodel_fd >= 0) { + size = read(kmodel_fd, model_data, kmodel_size); + if (size != kmodel_size) { + printf("read kmodel error size %d\n", size); + + } else { + printf("read kmodel success"); + } + } else { + free(model_data); + printf("open kmodel fail"); + } +} +#ifdef __RT_THREAD_H__ +MSH_CMD_EXPORT(kmodel_load, kmodel load memory); +#endif diff --git a/APP_Framework/Applications/knowing_app/instrusion_detect/Kconfig b/APP_Framework/Applications/knowing_app/instrusion_detect/Kconfig new file mode 100644 index 00000000..ee90a244 --- /dev/null +++ b/APP_Framework/Applications/knowing_app/instrusion_detect/Kconfig @@ -0,0 +1,8 @@ +config INSTRUSION_DETECT + bool "enable apps/instrusion detect" + depends on BOARD_K210_EVB + depends on DRV_USING_OV2640 + depends on USING_KPU_POSTPROCESSING + depends on USING_YOLOV2 + select LIB_USING_CJSON + default n diff --git a/APP_Framework/Applications/knowing_app/instrusion_detect/SConscript b/APP_Framework/Applications/knowing_app/instrusion_detect/SConscript new file mode 100644 index 00000000..fd445746 --- /dev/null +++ b/APP_Framework/Applications/knowing_app/instrusion_detect/SConscript @@ -0,0 +1,9 @@ +from building import * + +cwd = GetCurrentDir() +src = Glob('*.c') + Glob('*.cpp') +CPPPATH = [cwd] + +group = DefineGroup('Applications', src, depend = ['INSTRUSION_DETECT'], LOCAL_CPPPATH = CPPPATH) + +Return('group') diff --git a/APP_Framework/Applications/knowing_app/instrusion_detect/human.json b/APP_Framework/Applications/knowing_app/instrusion_detect/human.json new file mode 100644 index 00000000..0331ecc2 --- /dev/null +++ b/APP_Framework/Applications/knowing_app/instrusion_detect/human.json @@ -0,0 +1,36 @@ +{ + "net_input_size": [ + 224, + 320 + ], + "net_output_shape": [ + 10, + 7, + 30 + ], + "sensor_output_size": [ + 240, + 320 + ], + "anchors": [ + 1.0432, + 1.0920, + 0.8391, + 2.1250, + 1.1085, + 2.7463, + 1.3783, + 3.6706, + 2.0491, + 4.6711 + ], + "kmodel_path": "/kmodel/human.kmodel", + "kmodel_size": 1903016, + "obj_thresh": [ + 0.35 + ], + "labels": [ + "human" + ], + "nms_thresh": 0.3 +} \ No newline at end of file diff --git a/APP_Framework/Applications/knowing_app/instrusion_detect/instrusion_detect.c b/APP_Framework/Applications/knowing_app/instrusion_detect/instrusion_detect.c new file mode 100644 index 00000000..62929d0a --- /dev/null +++ b/APP_Framework/Applications/knowing_app/instrusion_detect/instrusion_detect.c @@ -0,0 +1,378 @@ +#include +#ifdef LIB_USING_CJSON +#include +#endif +#include "region_layer.h" +#define ANCHOR_NUM 5 +#define STACK_SIZE (128 * 1024) +#define JSON_FILE_PATH "/kmodel/human.json" +#define JSON_BUFFER_SIZE (4 * 1024) + +// params from json +static float anchor[ANCHOR_NUM * 2] = {}; +static int net_output_shape[3] = {}; +static int net_input_size[2] = {}; +static int sensor_output_size[2] = {}; +static char kmodel_path[127] = ""; +static int kmodel_size = 0; +static float obj_thresh[20] = {}; +static float nms_thresh = 0.0; +static char labels[20][32] = {}; +static int class_num = 0; + +#define THREAD_PRIORITY_HUMAN_D (11) +static pthread_t instrusiontid = 0; +static void *thread_instrusion_detect_entry(void *parameter); +static int g_fd = 0; +static int kmodel_fd = 0; +static int if_exit = 0; +static unsigned char *showbuffer = NULL; +static unsigned char *kpurgbbuffer = NULL; + +static _ioctl_shoot_para shoot_para_t = {0}; +unsigned char *model_data = NULL; // kpu data load memory +unsigned char *model_data_align = NULL; + +kpu_model_context_t instrusion_detect_task; +static region_layer_t instrusion_detect_rl; +static obj_info_t instrusion_detect_info; +volatile uint32_t g_ai_done_flag; + +static void ai_done(void *ctx) { g_ai_done_flag = 1; } + +static void param_parse() +{ + int fin; + char buffer[JSON_BUFFER_SIZE] = ""; + // char *buffer; + // if (NULL != (buffer = (char*)malloc(JSON_BUFFER_SIZE * sizeof(char)))) { + // memset(buffer, 0, JSON_BUFFER_SIZE * sizeof(char)); + // } else { + // printf("Json buffer malloc failed!"); + // exit(-1); + // } + int array_size; + cJSON *json_obj; + cJSON *json_item; + cJSON *json_array_item; + + fin = open(JSON_FILE_PATH, O_RDONLY); + if (!fin) { + printf("Error open file %s", JSON_FILE_PATH); + exit(-1); + } + read(fin, buffer, sizeof(buffer)); + close(fin); + + // read json string + json_obj = cJSON_Parse(buffer); + // free(buffer); + char *json_print_str = cJSON_Print(json_obj); + printf("Json file content: \n%s\n", json_print_str); + cJSON_free(json_print_str); + // get anchors + json_item = cJSON_GetObjectItem(json_obj, "anchors"); + array_size = cJSON_GetArraySize(json_item); + if (ANCHOR_NUM * 2 != array_size) { + printf("Expect anchor size: %d, got %d in json file", ANCHOR_NUM * 2, array_size); + exit(-1); + } else { + printf("Got %d anchors from json file\n", ANCHOR_NUM); + } + for (int i = 0; i < ANCHOR_NUM * 2; i++) { + json_array_item = cJSON_GetArrayItem(json_item, i); + anchor[i] = json_array_item->valuedouble; + printf("%d: %f\n", i, anchor[i]); + } + // net_input_size + json_item = cJSON_GetObjectItem(json_obj, "net_input_size"); + array_size = cJSON_GetArraySize(json_item); + if (2 != array_size) { + printf("Expect net_input_size: %d, got %d in json file", 2, array_size); + exit(-1); + } else { + printf("Got %d net_input_size from json file\n", 2); + } + for (int i = 0; i < 2; i++) { + json_array_item = cJSON_GetArrayItem(json_item, i); + net_input_size[i] = json_array_item->valueint; + printf("%d: %d\n", i, net_input_size[i]); + } + // net_output_shape + json_item = cJSON_GetObjectItem(json_obj, "net_output_shape"); + array_size = cJSON_GetArraySize(json_item); + if (3 != array_size) { + printf("Expect net_output_shape: %d, got %d in json file", 3, array_size); + exit(-1); + } else { + printf("Got %d net_output_shape from json file\n", 3); + } + for (int i = 0; i < 3; i++) { + json_array_item = cJSON_GetArrayItem(json_item, i); + net_output_shape[i] = json_array_item->valueint; + printf("%d: %d\n", i, net_output_shape[i]); + } + // sensor_output_size + json_item = cJSON_GetObjectItem(json_obj, "sensor_output_size"); + array_size = cJSON_GetArraySize(json_item); + if (2 != array_size) { + printf("Expect sensor_output_size: %d, got %d in json file", 2, array_size); + exit(-1); + } else { + printf("Got %d sensor_output_size from json file\n", 2); + } + for (int i = 0; i < 2; i++) { + json_array_item = cJSON_GetArrayItem(json_item, i); + sensor_output_size[i] = json_array_item->valueint; + printf("%d: %d\n", i, sensor_output_size[i]); + } + // kmodel_path + json_item = cJSON_GetObjectItem(json_obj, "kmodel_path"); + memcpy(kmodel_path, json_item->valuestring, strlen(json_item->valuestring)); + printf("Got kmodel_path: %s\n", kmodel_path); + // kmodel_size + json_item = cJSON_GetObjectItem(json_obj, "kmodel_size"); + kmodel_size = json_item->valueint; + printf("Got kmodel_size: %d\n", kmodel_size); + // labels + json_item = cJSON_GetObjectItem(json_obj, "labels"); + class_num = cJSON_GetArraySize(json_item); + if (0 >= class_num) { + printf("No labels!"); + exit(-1); + } else { + printf("Got %d labels\n", class_num); + } + for (int i = 0; i < class_num; i++) { + json_array_item = cJSON_GetArrayItem(json_item, i); + memcpy(labels[i], json_array_item->valuestring, strlen(json_array_item->valuestring)); + printf("%d: %s\n", i, labels[i]); + } + // obj_thresh + json_item = cJSON_GetObjectItem(json_obj, "obj_thresh"); + array_size = cJSON_GetArraySize(json_item); + if (class_num != array_size) { + printf("label number and thresh number mismatch! label number : %d, obj thresh number %d", class_num, array_size); + exit(-1); + } else { + printf("Got %d obj_thresh\n", array_size); + } + for (int i = 0; i < array_size; i++) { + json_array_item = cJSON_GetArrayItem(json_item, i); + obj_thresh[i] = json_array_item->valuedouble; + printf("%d: %f\n", i, obj_thresh[i]); + } + // nms_thresh + json_item = cJSON_GetObjectItem(json_obj, "nms_thresh"); + nms_thresh = json_item->valuedouble; + printf("Got nms_thresh: %f\n", nms_thresh); + + cJSON_Delete(json_obj); + return; +} + +void instrusion_detect() +{ + int ret = 0; + int result = 0; + int size = 0; + param_parse(); + g_fd = open("/dev/ov2640", O_RDONLY); + if (g_fd < 0) { + printf("open ov2640 fail !!"); + return; + } + showbuffer = (unsigned char *)malloc(sensor_output_size[0] * sensor_output_size[1] * 2); + if (NULL == showbuffer) { + close(g_fd); + printf("showbuffer apply memory fail !!"); + return; + } + kpurgbbuffer = (unsigned char *)malloc(net_input_size[0] * net_input_size[1] * 3); + if (NULL == kpurgbbuffer) { + close(g_fd); + free(showbuffer); + printf("kpurgbbuffer apply memory fail !!"); + return; + } + model_data = (unsigned char *)malloc(kmodel_size + 255); + if (NULL == model_data) { + free(showbuffer); + free(kpurgbbuffer); + close(g_fd); + printf("model_data apply memory fail !!"); + return; + } + memset(model_data, 0, kmodel_size + 255); + memset(showbuffer, 0, sensor_output_size[0] * sensor_output_size[1] * 2); + memset(kpurgbbuffer, 127, net_input_size[0] * net_input_size[1] * 3); + shoot_para_t.pdata = (unsigned int *)(showbuffer); + shoot_para_t.length = (size_t)(sensor_output_size[0] * sensor_output_size[1] * 2); + /* + load memory + */ + kmodel_fd = open(kmodel_path, O_RDONLY); + if (kmodel_fd < 0) { + printf("open kmodel fail"); + close(g_fd); + free(showbuffer); + free(kpurgbbuffer); + free(model_data); + return; + } else { + size = read(kmodel_fd, model_data, kmodel_size); + if (size != kmodel_size) { + printf("read kmodel error size %d\n", size); + close(g_fd); + close(kmodel_fd); + free(showbuffer); + free(kpurgbbuffer); + free(model_data); + return; + + } else { + printf("read kmodel success \n"); + } + } + unsigned char *model_data_align = (unsigned char *)(((unsigned int)model_data + 255) & (~255)); + dvp_set_ai_addr((uint32_t)(kpurgbbuffer + net_input_size[1] * (net_input_size[0] - sensor_output_size[0])), + (uint32_t)(kpurgbbuffer + net_input_size[1] * (net_input_size[0] - sensor_output_size[0]) + + net_input_size[0] * net_input_size[1]), + (uint32_t)(kpurgbbuffer + net_input_size[0] * net_input_size[1] * 2 + + net_input_size[1] * (net_input_size[0] - sensor_output_size[0]))); + if (kpu_load_kmodel(&instrusion_detect_task, model_data_align) != 0) { + printf("\nmodel init error\n"); + close(g_fd); + close(kmodel_fd); + free(showbuffer); + free(kpurgbbuffer); + free(model_data); + return; + } + instrusion_detect_rl.anchor_number = ANCHOR_NUM; + instrusion_detect_rl.anchor = anchor; + instrusion_detect_rl.threshold = malloc(class_num * sizeof(float)); + for (int idx = 0; idx < class_num; idx++) { + instrusion_detect_rl.threshold[idx] = obj_thresh[idx]; + } + instrusion_detect_rl.nms_value = nms_thresh; + result = region_layer_init(&instrusion_detect_rl, net_output_shape[0], net_output_shape[1], net_output_shape[2], + net_input_size[1], net_input_size[0]); + printf("region_layer_init result %d \n\r", result); + size_t stack_size = STACK_SIZE; + pthread_attr_t attr; /* 线程属性 */ + struct sched_param prio; /* 线程优先级 */ + prio.sched_priority = 8; /* 优先级设置为 8 */ + pthread_attr_init(&attr); /* 先使用默认值初始化属性 */ + pthread_attr_setschedparam(&attr, &prio); /* 修改属性对应的优先级 */ + pthread_attr_setstacksize(&attr, stack_size); + + /* 创建线程 1, 属性为 attr，入口函数是 thread_entry，入口函数参数是 1 */ + result = pthread_create(&instrusiontid, &attr, thread_instrusion_detect_entry, NULL); + if (0 == result) { + printf("thread_instrusion_detect_entry successfully!\n"); + } else { + printf("thread_instrusion_detect_entry failed! error code is %d\n", result); + close(g_fd); + } +} +#ifdef __RT_THREAD_H__ +MSH_CMD_EXPORT(instrusion_detect, instrusion detect task); +#endif + +static void *thread_instrusion_detect_entry(void *parameter) +{ + extern void lcd_draw_picture(uint16_t x1, uint16_t y1, uint16_t width, uint16_t height, uint32_t * ptr); + printf("thread_instrusion_detect_entry start!\n"); + int ret = 0; + // sysctl_enable_irq(); + while (1) { + // memset(showbuffer,0,320*240*2); + g_ai_done_flag = 0; + ret = ioctl(g_fd, IOCTRL_CAMERA_START_SHOT, &shoot_para_t); + if (RT_ERROR == ret) { + printf("ov2640 can't wait event flag"); + rt_free(showbuffer); + close(g_fd); + pthread_exit(NULL); + return NULL; + } + kpu_run_kmodel(&instrusion_detect_task, kpurgbbuffer, DMAC_CHANNEL5, ai_done, NULL); + while (!g_ai_done_flag) + ; + float *output; + size_t output_size; + kpu_get_output(&instrusion_detect_task, 0, (uint8_t **)&output, &output_size); + instrusion_detect_rl.input = output; + region_layer_run(&instrusion_detect_rl, &instrusion_detect_info); +/* display result */ +#ifdef BSP_USING_LCD + for (int instrusion_cnt = 0; instrusion_cnt < instrusion_detect_info.obj_number; instrusion_cnt++) { + // draw_edge((uint32_t *)showbuffer, &instrusion_detect_info, instrusion_cnt, 0xF800, + // (uint16_t)sensor_output_size[1], + // (uint16_t)sensor_output_size[0]); + printf("%d: (%d, %d, %d, %d) cls: %s conf: %f\t", instrusion_cnt, instrusion_detect_info.obj[instrusion_cnt].x1, + instrusion_detect_info.obj[instrusion_cnt].y1, instrusion_detect_info.obj[instrusion_cnt].x2, + instrusion_detect_info.obj[instrusion_cnt].y2, labels[instrusion_detect_info.obj[instrusion_cnt].class_id], + instrusion_detect_info.obj[instrusion_cnt].prob); + } + if (0 != instrusion_detect_info.obj_number) { + printf("\n"); + } + lcd_draw_picture(0, 0, (uint16_t)sensor_output_size[1], (uint16_t)sensor_output_size[0], (unsigned int *)showbuffer); +#endif + usleep(1); + if (1 == if_exit) { + if_exit = 0; + printf("thread_instrusion_detect_entry exit"); + pthread_exit(NULL); + } + } +} + +void instrusion_detect_delete() +{ + if (showbuffer != NULL) { + int ret = 0; + close(g_fd); + close(kmodel_fd); + free(showbuffer); + free(kpurgbbuffer); + free(model_data); + printf("instrusion detect task cancel!!! ret %d ", ret); + if_exit = 1; + } +} +#ifdef __RT_THREAD_H__ +MSH_CMD_EXPORT(instrusion_detect_delete, instrusion detect task delete); +#endif + +void kmodel_load(unsigned char *model_data) +{ + int kmodel_fd = 0; + int size = 0; + kmodel_fd = open(kmodel_path, O_RDONLY); + + model_data = (unsigned char *)malloc(kmodel_size + 255); + if (NULL == model_data) { + printf("model_data apply memory fail !!"); + return; + } + memset(model_data, 0, kmodel_size + 255); + + if (kmodel_fd >= 0) { + size = read(kmodel_fd, model_data, kmodel_size); + if (size != kmodel_size) { + printf("read kmodel error size %d\n", size); + + } else { + printf("read kmodel success"); + } + } else { + free(model_data); + printf("open kmodel fail"); + } +} +#ifdef __RT_THREAD_H__ +MSH_CMD_EXPORT(kmodel_load, kmodel load memory); +#endif diff --git a/APP_Framework/Framework/knowing/kpu-postprocessing/yolov2/region_layer.c b/APP_Framework/Framework/knowing/kpu-postprocessing/yolov2/region_layer.c index dad31268..bb37e6d9 100644 --- a/APP_Framework/Framework/knowing/kpu-postprocessing/yolov2/region_layer.c +++ b/APP_Framework/Framework/knowing/kpu-postprocessing/yolov2/region_layer.c @@ -189,7 +189,7 @@ static void get_region_boxes(region_layer_t *rl, float *predictions, float **pro uint32_t anchor_number = rl->anchor_number; uint32_t classes = rl->classes; uint32_t coords = rl->coords; - float threshold = rl->threshold; + float *threshold = rl->threshold; for (int i = 0; i < layer_width * layer_height; ++i) { int row = i / layer_width; @@ -212,7 +212,7 @@ static void get_region_boxes(region_layer_t *rl, float *predictions, float **pro int class_index = entry_index(rl, n * layer_width * layer_height + i, coords + 1 + j); float prob = scale * predictions[class_index]; - probs[index][j] = (prob > threshold) ? prob : 0; + probs[index][j] = (prob > threshold[j]) ? prob : 0; if (prob > max) max = prob; } probs[index][classes] = max; @@ -315,14 +315,14 @@ static void region_layer_output(region_layer_t *rl, obj_info_t *obj_info) uint32_t image_width = rl->image_width; uint32_t image_height = rl->image_height; uint32_t boxes_number = rl->boxes_number; - float threshold = rl->threshold; + float *threshold = rl->threshold; box_t *boxes = (box_t *)rl->boxes; for (int i = 0; i < rl->boxes_number; ++i) { int class = max_index(rl->probs[i], rl->classes); float prob = rl->probs[i][class]; - if (prob > threshold) { + if (prob > threshold[class]) { box_t *b = boxes + i; obj_info->obj[obj_number].x1 = b->x * image_width - (b->w * image_width / 2); obj_info->obj[obj_number].y1 = b->y * image_height - (b->h * image_height / 2); diff --git a/APP_Framework/Framework/knowing/kpu-postprocessing/yolov2/region_layer.h b/APP_Framework/Framework/knowing/kpu-postprocessing/yolov2/region_layer.h index 70d0ea99..3b9ae4cb 100644 --- a/APP_Framework/Framework/knowing/kpu-postprocessing/yolov2/region_layer.h +++ b/APP_Framework/Framework/knowing/kpu-postprocessing/yolov2/region_layer.h @@ -20,7 +20,7 @@ typedef struct typedef struct { - float threshold; + float *threshold; float nms_value; uint32_t coords; uint32_t anchor_number; diff --git a/Ubiquitous/RT_Thread/bsp/k210/base-drivers/drv_dvp.c b/Ubiquitous/RT_Thread/bsp/k210/base-drivers/drv_dvp.c index c1dbc389..41cd834e 100644 --- a/Ubiquitous/RT_Thread/bsp/k210/base-drivers/drv_dvp.c +++ b/Ubiquitous/RT_Thread/bsp/k210/base-drivers/drv_dvp.c @@ -67,7 +67,8 @@ static rt_err_t rt_dvp_init(rt_device_t dev) dvp_set_output_enable(0, 1); dvp_set_output_enable(1, 1); dvp_set_image_format(DVP_CFG_RGB_FORMAT);//////////////// - dvp_set_image_size(320, 240); + // dvp_set_image_size(320, 240); + dvp_set_image_size(256, 256); dvp_config_interrupt(DVP_CFG_FINISH_INT_ENABLE, 0); dvp_disable_auto(); plic_set_priority(IRQN_DVP_INTERRUPT, 1);