diff --git a/src/xrt/state_trackers/gui/CMakeLists.txt b/src/xrt/state_trackers/gui/CMakeLists.txt index c07ed9c90..610cc458b 100644 --- a/src/xrt/state_trackers/gui/CMakeLists.txt +++ b/src/xrt/state_trackers/gui/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright 2019-2020, Collabora, Ltd. +# Copyright 2019-2021, Collabora, Ltd. # SPDX-License-Identifier: BSL-1.0 # c-imgui doesn't do well with IPO - lots of warnings. @@ -13,6 +13,7 @@ set(GUI_SOURCE_FILES gui_scene_calibrate.c gui_scene_debug.c gui_scene_main_menu.c + gui_scene_record.c gui_scene_remote.c gui_scene_video.c gui_scene_tracking_overrides.c @@ -60,6 +61,12 @@ target_compile_definitions(st_gui PUBLIC CIMGUI_NO_EXPORT ) +if(XRT_HAVE_GST) + target_link_libraries(st_gui PRIVATE + aux_gstreamer + ) +endif() + if(XRT_BUILD_DRIVER_REMOTE) target_link_libraries(st_gui PRIVATE drv_remote diff --git a/src/xrt/state_trackers/gui/gui_common.h b/src/xrt/state_trackers/gui/gui_common.h index 16f8c0158..3405b7ddb 100644 --- a/src/xrt/state_trackers/gui/gui_common.h +++ b/src/xrt/state_trackers/gui/gui_common.h @@ -213,6 +213,14 @@ gui_scene_tracking_overrides(struct gui_program *p); void gui_scene_debug(struct gui_program *p); +/*! + * Create a recording view scene. + * + * @ingroup gui + */ +void +gui_scene_record(struct gui_program *p, const char *camera); + /*! * Remote control debugging UI. * diff --git a/src/xrt/state_trackers/gui/gui_scene_main_menu.c b/src/xrt/state_trackers/gui/gui_scene_main_menu.c index 482048075..602213001 100644 --- a/src/xrt/state_trackers/gui/gui_scene_main_menu.c +++ b/src/xrt/state_trackers/gui/gui_scene_main_menu.c @@ -51,6 +51,17 @@ scene_render(struct gui_scene *scene, struct gui_program *p) gui_scene_debug(p); } + if (igButton("Record (Index)", button_dims)) { + gui_scene_delete_me(p, scene); + gui_scene_record(p, "index"); + } + + + if (igButton("Record (Leap Motion)", button_dims)) { + gui_scene_delete_me(p, scene); + gui_scene_record(p, "leap_motion"); + } + if (igButton("Remote", button_dims)) { gui_scene_delete_me(p, scene); diff --git a/src/xrt/state_trackers/gui/gui_scene_record.c b/src/xrt/state_trackers/gui/gui_scene_record.c new file mode 100644 index 000000000..4523a9c2d --- /dev/null +++ b/src/xrt/state_trackers/gui/gui_scene_record.c @@ -0,0 +1,586 @@ +// Copyright 2019-2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Calibration gui scene. + * @author Jakob Bornecrantz + * @ingroup gui + */ + + +#include "xrt/xrt_config_have.h" +#include "xrt/xrt_config_drivers.h" + +#include "os/os_threading.h" + +#include "util/u_var.h" +#include "util/u_misc.h" +#include "util/u_sink.h" +#include "util/u_file.h" +#include "util/u_json.h" +#include "util/u_frame.h" +#include "util/u_format.h" + +#include "xrt/xrt_frame.h" +#include "xrt/xrt_prober.h" +#include "xrt/xrt_tracking.h" +#include "xrt/xrt_frameserver.h" + +#ifdef XRT_HAVE_GST +#include "gstreamer/gst_sink.h" +#include "gstreamer/gst_pipeline.h" +#endif + +#ifdef XRT_BUILD_DRIVER_VF +#include "vf/vf_interface.h" +#endif + +#include "gui_common.h" +#include "gui_imgui.h" + +#include "stb_image_write.h" + +#include +#include + +enum bitrate +{ + BITRATE_4096, + BITRATE_2048, + BITRATE_1024, +}; + +enum pipeline +{ + PIPELINE_SOFTWARE_FAST, + PIPELINE_SOFTWARE_MEDIUM, + PIPELINE_SOFTWARE_SLOW, + PIPELINE_SOFTWARE_VERYSLOW, + PIPELINE_VAAPI_H246, +}; + +struct record_window +{ + struct xrt_frame_sink sink; + + struct + { + // Use leap_motion. + bool leap_motion; + // Use index. + bool index; + } use; + + struct + { + struct xrt_frame_context xfctx; + + struct xrt_fs *xfs; + + struct xrt_fs_mode mode; + + char name[256]; + } camera; + + struct + { + int scale; + + struct xrt_frame_sink *sink; + struct gui_ogl_texture *ogl; + } texture; + +#ifdef XRT_HAVE_GST + struct + { + enum bitrate bitrate; + + enum pipeline pipeline; + + struct xrt_frame_context xfctx; + + //! When not null we are recording. + struct xrt_frame_sink *sink; + + //! Protects sink + struct os_mutex mutex; + + //! App sink we are pushing frames into. + struct gstreamer_sink *gs; + + //! Recording pipeline. + struct gstreamer_pipeline *gp; + + char filename[512]; + } gst; +#endif +}; + +struct record_scene +{ + struct gui_scene base; + + struct record_window *window; +}; + + +/* + * + * GStreamer functions. + * + */ + +#ifdef XRT_HAVE_GST +static void +create_pipeline(struct record_window *rw) +{ + const char *source_name = "source_name"; + const char *bitrate = NULL; + const char *speed_preset = NULL; + + char pipeline_string[2048]; + + switch (rw->gst.bitrate) { + default: + case BITRATE_4096: bitrate = "4096"; break; + case BITRATE_2048: bitrate = "2048"; break; + case BITRATE_1024: bitrate = "1024"; break; + } + + switch (rw->gst.pipeline) { + case PIPELINE_SOFTWARE_FAST: speed_preset = "fast"; break; + case PIPELINE_SOFTWARE_MEDIUM: speed_preset = "medium"; break; + case PIPELINE_SOFTWARE_SLOW: speed_preset = "slow"; break; + case PIPELINE_SOFTWARE_VERYSLOW: speed_preset = "veryslow"; break; + default: break; + } + + if (speed_preset != NULL) { + snprintf(pipeline_string, // + sizeof(pipeline_string), // + "appsrc name=\"%s\" ! " + "queue ! " + "videoconvert ! " + "queue ! " + "x264enc bitrate=\"%s\" speed-preset=\"%s\" ! " + "h264parse ! " + "queue ! " + "mp4mux ! " + "filesink location=\"%s\"", + source_name, bitrate, speed_preset, rw->gst.filename); + } else { + snprintf(pipeline_string, // + sizeof(pipeline_string), // + "appsrc name=\"%s\" ! " + "queue ! " + "videoconvert ! " + "video/x-raw,format=NV12 ! " + "queue ! " + "vaapih264enc rate-control=cbr bitrate=\"%s\" tune=high-compression ! " + "video/x-h264,profile=main ! " + "h264parse ! " + "queue ! " + "mp4mux ! " + "filesink location=\"%s\"", + source_name, bitrate, rw->gst.filename); + } + + struct xrt_frame_sink *tmp = NULL; + struct gstreamer_pipeline *gp = NULL; + + gstreamer_pipeline_create_from_string(&rw->gst.xfctx, pipeline_string, &gp); + + uint32_t width = rw->camera.mode.width; + uint32_t height = rw->camera.mode.height; + enum xrt_format format = rw->camera.mode.format; + + struct gstreamer_sink *gs = NULL; + + gstreamer_sink_create_with_pipeline(gp, width, height, format, source_name, &gs, &tmp); + u_sink_queue_create(&rw->gst.xfctx, tmp, &tmp); + + os_mutex_lock(&rw->gst.mutex); + rw->gst.gs = gs; + rw->gst.sink = tmp; + rw->gst.gp = gp; + gstreamer_pipeline_play(rw->gst.gp); + os_mutex_unlock(&rw->gst.mutex); +} + +static void +destroy_pipeline(struct record_window *rw) +{ + U_LOG_D("Called"); + + // Make sure we are not streaming any more frames into the pipeline. + os_mutex_lock(&rw->gst.mutex); + rw->gst.gs = NULL; + rw->gst.sink = NULL; + os_mutex_unlock(&rw->gst.mutex); + + // Stop the pipeline. + gstreamer_pipeline_stop(rw->gst.gp); + rw->gst.gp = NULL; + + xrt_frame_context_destroy_nodes(&rw->gst.xfctx); +} + +static void +draw_gst(struct record_window *rw) +{ + static ImVec2 button_dims = {0, 0}; + + if (!igCollapsingHeaderBoolPtr("Record", NULL, ImGuiTreeNodeFlags_DefaultOpen)) { + return; + } + + + os_mutex_lock(&rw->gst.mutex); + bool recording = rw->gst.gp != NULL; + os_mutex_unlock(&rw->gst.mutex); + + igComboStr("Pipeline", (int *)&rw->gst.pipeline, "SW Fast\0SW Medium\0SW Slow\0SW Veryslow\0VAAPI H264\0\0", 5); + + igInputText("Filename", rw->gst.filename, sizeof(rw->gst.filename), 0, NULL, NULL); + + if (!recording && igButton("Start", button_dims)) { + create_pipeline(rw); + } + + if (recording && igButton("Stop", button_dims)) { + destroy_pipeline(rw); + } +} +#endif + + +/* + * + * Record window functions. + * + */ + +static void +window_destroy(struct record_window *rw) +{ + // Stop and remove the recording pipeline first. +#ifdef XRT_HAVE_GST + if (rw->gst.gp != NULL) { + os_mutex_lock(&rw->gst.mutex); + rw->gst.gs = NULL; + rw->gst.sink = NULL; + os_mutex_unlock(&rw->gst.mutex); + + gstreamer_pipeline_stop(rw->gst.gp); + rw->gst.gp = NULL; + xrt_frame_context_destroy_nodes(&rw->gst.xfctx); + } +#endif + + // Stop the camera if we have one. + xrt_frame_context_destroy_nodes(&rw->camera.xfctx); + rw->camera.xfs = NULL; + rw->texture.ogl = NULL; + rw->texture.sink = NULL; + + free(rw); +} + +static bool +window_has_source(struct record_window *rw) +{ + return rw->camera.xfs != NULL; +} + +static void +window_draw_misc(struct record_window *rw) +{ + if (!igCollapsingHeaderBoolPtr("Misc", NULL, ImGuiTreeNodeFlags_DefaultOpen)) { + return; + } + + static ImVec2 button_dims = {0, 0}; + bool plus = igButton("+", button_dims); + igSameLine(0.0f, 4.0f); + bool minus = igButton("-", button_dims); + igSameLine(0.0f, 4.0f); + + if (rw->texture.scale == 1) { + igText("Scale 100%%"); + } else { + igText("Scale 1/%i", rw->texture.scale); + } + + if (plus && rw->texture.scale > 1) { + rw->texture.scale--; + } + if (minus && rw->texture.scale < 6) { + rw->texture.scale++; + } + + igText("Sequence %u", (uint32_t)rw->texture.ogl->seq); +} + +static void +window_render(struct record_window *rw, struct gui_program *p) +{ + igBegin("Preview and Control", NULL, 0); + + gui_ogl_sink_update(rw->texture.ogl); + + struct gui_ogl_texture *tex = rw->texture.ogl; + + int w = tex->w / rw->texture.scale; + int h = tex->h / rw->texture.scale; + + ImVec2 size = {(float)w, (float)h}; + ImVec2 uv0 = {0, 0}; + ImVec2 uv1 = {1, 1}; + ImVec4 white = {1, 1, 1, 1}; + ImTextureID id = (ImTextureID)(intptr_t)tex->id; + igImage(id, size, uv0, uv1, white, white); + +#ifdef XRT_HAVE_GST + draw_gst(rw); +#endif + + window_draw_misc(rw); + + igEnd(); +} + +static void +window_frame(struct xrt_frame_sink *xfs, struct xrt_frame *xf) +{ + struct record_window *rw = container_of(xfs, struct record_window, sink); + +#ifdef XRT_HAVE_GST + os_mutex_lock(&rw->gst.mutex); + if (rw->gst.sink != NULL) { + xrt_sink_push_frame(rw->gst.sink, xf); + } + os_mutex_unlock(&rw->gst.mutex); +#endif + + xrt_sink_push_frame(rw->texture.sink, xf); +} + +static struct record_window * +window_create(struct gui_program *p, const char *camera) +{ + struct record_window *rw = U_TYPED_CALLOC(struct record_window); + rw->sink.push_frame = window_frame; + rw->use.index = camera == NULL ? false : strcmp(camera, "index") == 0; + rw->use.leap_motion = camera == NULL ? false : strcmp(camera, "leap_motion") == 0; + + if (!rw->use.index && !rw->use.leap_motion) { + U_LOG_W("Can't recongnize camera name '%s', options are 'index' & 'leap_motion'", camera); + rw->use.index = true; + } + + // Setup the preview texture. + rw->texture.scale = 1; + struct xrt_frame_sink *tmp = NULL; + rw->texture.ogl = gui_ogl_sink_create("View", &rw->camera.xfctx, &tmp); + u_sink_create_to_r8g8b8_or_l8(&rw->camera.xfctx, tmp, &tmp); + u_sink_queue_create(&rw->camera.xfctx, tmp, &rw->texture.sink); + +#ifdef XRT_HAVE_GST + int ret = os_mutex_init(&rw->gst.mutex); + if (ret < 0) { + free(rw); + return NULL; + } + + snprintf(rw->gst.filename, sizeof(rw->gst.filename), "/tmp/capture.mp4"); +#endif + + return rw; +} + + +/* + * + * Video frame functions + * + */ + +#ifdef XRT_BUILD_DRIVER_VF +static void +create_videotestsrc(struct record_window *rw) +{ + uint32_t width = 1920; + uint32_t height = 960; + rw->camera.xfs = vf_fs_videotestsource(&rw->camera.xfctx, width, height); + + // Just after the camera create a quirk stream. + struct u_sink_quirk_params qp; + U_ZERO(&qp); + qp.stereo_sbs = false; + qp.ps4_cam = false; + qp.leap_motion = false; + + struct xrt_frame_sink *tmp = NULL; + u_sink_quirk_create(&rw->camera.xfctx, &rw->sink, &qp, &tmp); + + // Now that we have setup a node graph, start it. + xrt_fs_stream_start(rw->camera.xfs, tmp, 0, XRT_FS_CAPTURE_TYPE_TRACKING); + + rw->camera.mode.width = width; + rw->camera.mode.height = height; + rw->camera.mode.format = XRT_FORMAT_R8G8B8; + + // If it's a large mode, scale to 50% + if (rw->camera.mode.width > 640) { + rw->texture.scale = 2; + } +} +#endif /* XRT_BUILD_DRIVER_VF */ + + +/* + * + * Prober functions. + * + */ + +static bool +is_camera_index(const char *product, const char *manufacturer) +{ + return strcmp(product, "3D Camera") == 0 && strcmp(manufacturer, "Etron Technology, Inc.") == 0; +} + +static bool +is_camera_leap_motion(const char *product, const char *manufacturer) +{ + return strcmp(product, "Leap Motion Controller") == 0 && strcmp(manufacturer, "Leap Motion") == 0; +} + +static void +on_video_device(struct xrt_prober *xp, + struct xrt_prober_device *pdev, + const char *product, + const char *manufacturer, + const char *serial, + void *ptr) +{ + struct record_window *rw = (struct record_window *)ptr; + + if (rw->camera.xfs != NULL) { + return; + } + + // Hardcoded for the Index. + if (rw->use.index && !is_camera_index(product, manufacturer)) { + return; + } + + // Hardcoded for the Leap Motion. + if (rw->use.leap_motion && !is_camera_leap_motion(product, manufacturer)) { + return; + } + + snprintf(rw->camera.name, sizeof(rw->camera.name), "%s-%s", product, serial); + + xrt_prober_open_video_device(xp, pdev, &rw->camera.xfctx, &rw->camera.xfs); + + struct xrt_frame_sink *tmp = &rw->sink; + + if (rw->use.leap_motion) { + // De-interleaving. + u_sink_deinterleaver_create(&rw->camera.xfctx, tmp, &tmp); + } + + // Just after the camera create a quirk stream. + struct u_sink_quirk_params qp; + U_ZERO(&qp); + qp.stereo_sbs = false; + qp.ps4_cam = false; + qp.leap_motion = rw->use.leap_motion; + + u_sink_quirk_create(&rw->camera.xfctx, tmp, &qp, &tmp); + + struct xrt_fs_mode *modes = NULL; + uint32_t mode_count = 0; + xrt_fs_enumerate_modes(rw->camera.xfs, &modes, &mode_count); + assert(mode_count > 0); + + // Just use the first one. + uint32_t mode_index = 0; + + rw->camera.mode = modes[mode_index]; + free(modes); + modes = NULL; + + // Touch up. + if (rw->use.leap_motion) { + rw->camera.mode.width = rw->camera.mode.width * 2; + rw->camera.mode.format = XRT_FORMAT_L8; + } + + // If it's a large mode, scale to 50% + if (rw->camera.mode.width > 640) { + rw->texture.scale = 2; + } + + // Now that we have setup a node graph, start it. + xrt_fs_stream_start(rw->camera.xfs, tmp, mode_index, XRT_FS_CAPTURE_TYPE_TRACKING); +} + + +/* + * + * Scene functions. + * + */ + +static void +scene_render(struct gui_scene *scene, struct gui_program *p) +{ + static ImVec2 button_dims = {0, 0}; + struct record_scene *rs = (struct record_scene *)scene; + + window_render(rs->window, p); + + igBegin("Record-a-tron!", NULL, 0); + if (igButton("Exit", button_dims)) { + gui_scene_delete_me(p, &rs->base); + } + igEnd(); +} + +static void +scene_destroy(struct gui_scene *scene, struct gui_program *p) +{ + struct record_scene *rs = (struct record_scene *)scene; + + if (rs->window != NULL) { + window_destroy(rs->window); + rs->window = NULL; + } + + free(rs); +} + +void +gui_scene_record(struct gui_program *p, const char *camera) +{ + struct record_scene *rs = U_TYPED_CALLOC(struct record_scene); + + rs->base.render = scene_render; + rs->base.destroy = scene_destroy; + + rs->window = window_create(p, camera); + + if (!window_has_source(rs->window)) { + xrt_prober_list_video_devices(p->xp, on_video_device, rs->window); + } + +#ifdef XRT_BUILD_DRIVER_VF + if (!window_has_source(rs->window)) { + create_videotestsrc(rs->window); + } +#endif + + gui_scene_push_front(p, &rs->base); +} diff --git a/src/xrt/state_trackers/gui/meson.build b/src/xrt/state_trackers/gui/meson.build index 5de806557..75fd44a11 100644 --- a/src/xrt/state_trackers/gui/meson.build +++ b/src/xrt/state_trackers/gui/meson.build @@ -1,4 +1,4 @@ -# Copyright 2019-2020, Collabora, Ltd. +# Copyright 2019-2021, Collabora, Ltd. # SPDX-License-Identifier: BSL-1.0 gui_sources = [ @@ -10,6 +10,7 @@ gui_sources = [ 'gui_scene_calibrate.c', 'gui_scene_debug.c', 'gui_scene_main_menu.c', + 'gui_scene_record.c', 'gui_scene_remote.c', 'gui_scene_video.c', 'gui_scene_tracking_overrides.c', @@ -41,6 +42,10 @@ gui_sources = [ gui_deps = [aux, xrt_config_have] +if 'vf' in drivers + gui_deps += [drv_vf] +endif + lib_st_gui = static_library( 'st_gui', files(gui_sources), diff --git a/src/xrt/targets/gui/gui_sdl2_main.c b/src/xrt/targets/gui/gui_sdl2_main.c index 9ba17e2d6..1aa0fb281 100644 --- a/src/xrt/targets/gui/gui_sdl2_main.c +++ b/src/xrt/targets/gui/gui_sdl2_main.c @@ -41,6 +41,8 @@ main(int argc, char **argv) gui_scene_select_video_calibrate(&p.base); } else if (argc >= 2 && strcmp("tracking_overrides", argv[1]) == 0) { gui_scene_tracking_overrides(&p.base); + } else if (argc >= 2 && strcmp("record", argv[1]) == 0) { + gui_scene_record(&p.base, argc >= 3 ? argv[2] : NULL); } else if (argc >= 2 && strcmp("remote", argv[1]) == 0) { gui_scene_remote(&p.base); } else {