From ec4482f963005101eb2c157aaf40d80660272d3d Mon Sep 17 00:00:00 2001 From: DavidVentura Date: Sun, 30 Apr 2023 16:40:49 +0200 Subject: [PATCH 1/8] crappy but working vnc impl --- app/meson.build | 4 ++ app/src/cli.c | 9 ++++ app/src/options.c | 1 + app/src/options.h | 1 + app/src/scrcpy.c | 15 +++++- app/src/vnc_sink.c | 113 +++++++++++++++++++++++++++++++++++++++++++++ app/src/vnc_sink.h | 48 +++++++++++++++++++ 7 files changed, 190 insertions(+), 1 deletion(-) create mode 100644 app/src/vnc_sink.c create mode 100644 app/src/vnc_sink.h diff --git a/app/meson.build b/app/meson.build index 061fdcab68..8f0993d58d 100644 --- a/app/meson.build +++ b/app/meson.build @@ -31,6 +31,7 @@ src = [ 'src/screen.c', 'src/server.c', 'src/version.c', + 'src/vnc_sink.c', 'src/trait/frame_source.c', 'src/trait/packet_source.c', 'src/util/acksync.c', @@ -108,6 +109,9 @@ if not crossbuild_windows dependency('libavutil'), dependency('libswresample'), dependency('sdl2', version: '>= 2.0.5'), + #cc.find_library('swscale-5', dirs: ffmpeg_bin_dir), + dependency('libswscale'), + dependency('libvncserver'), ] if v4l2_support diff --git a/app/src/cli.c b/app/src/cli.c index d6d9f41dd9..343c1436d1 100644 --- a/app/src/cli.c +++ b/app/src/cli.c @@ -72,6 +72,7 @@ enum { OPT_REQUIRE_AUDIO, OPT_AUDIO_BUFFER, OPT_AUDIO_OUTPUT_BUFFER, + OPT_VNC_SERVER, }; struct sc_option { @@ -669,6 +670,11 @@ static const struct sc_option options[] = { .text = "Set the initial window height.\n" "Default is 0 (automatic).", }, + { + .longopt_id = OPT_VNC_SERVER, + .longopt = "vnc-server", + .text = "Enable VNC server.", + }, }; static const struct sc_shortcut shortcuts[] = { @@ -1861,6 +1867,9 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[], return false; } break; + case OPT_VNC_SERVER: + opts->vnc_server = true; + break; default: // getopt prints the error message on stderr return false; diff --git a/app/src/options.c b/app/src/options.c index 8b99f6f3ad..7768198fe9 100644 --- a/app/src/options.c +++ b/app/src/options.c @@ -53,6 +53,7 @@ const struct scrcpy_options scrcpy_options_default = { .always_on_top = false, .control = true, .display = true, + .vnc_server = false, .turn_screen_off = false, .key_inject_mode = SC_KEY_INJECT_MODE_MIXED, .window_borderless = false, diff --git a/app/src/options.h b/app/src/options.h index c41e275703..63e8eda9a7 100644 --- a/app/src/options.h +++ b/app/src/options.h @@ -136,6 +136,7 @@ struct scrcpy_options { bool always_on_top; bool control; bool display; + bool vnc_server; bool turn_screen_off; enum sc_key_inject_mode key_inject_mode; bool window_borderless; diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index efa69d31aa..daa0aabc6c 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -25,6 +25,7 @@ #include "recorder.h" #include "screen.h" #include "server.h" +#include "vnc_sink.h" #ifdef HAVE_USB # include "usb/aoa_hid.h" # include "usb/hid_keyboard.h" @@ -48,6 +49,8 @@ struct scrcpy { struct sc_decoder video_decoder; struct sc_decoder audio_decoder; struct sc_recorder recorder; + struct sc_vnc_sink vnc_sink; + struct sc_delay_buffer vnc_buffer; struct sc_delay_buffer display_buffer; #ifdef HAVE_V4L2 struct sc_v4l2_sink v4l2_sink; @@ -311,6 +314,7 @@ scrcpy(struct scrcpy_options *options) { #ifdef HAVE_V4L2 bool v4l2_sink_initialized = false; #endif + bool vnc_sink_initialized = false; bool video_demuxer_started = false; bool audio_demuxer_started = false; #ifdef HAVE_USB @@ -451,7 +455,7 @@ scrcpy(struct scrcpy_options *options) { &audio_demuxer_cbs, options); } - bool needs_video_decoder = options->display; + bool needs_video_decoder = options->display || options->vnc_server; bool needs_audio_decoder = options->audio && options->display; #ifdef HAVE_V4L2 needs_video_decoder |= !!options->v4l2_device; @@ -694,6 +698,15 @@ scrcpy(struct scrcpy_options *options) { &s->audio_player.frame_sink); } } + if (options->vnc_server) { + if (!sc_vnc_sink_init(&s->vnc_sink, "my vnc server")) { + printf("bad vnc init \n"); + goto end; + } + vnc_sink_initialized = true; + struct sc_frame_source *src = &s->video_decoder.frame_source; + sc_frame_source_add_sink(src, &s->vnc_sink.frame_sink); + } #ifdef HAVE_V4L2 if (options->v4l2_device) { diff --git a/app/src/vnc_sink.c b/app/src/vnc_sink.c new file mode 100644 index 0000000000..0ffec82baf --- /dev/null +++ b/app/src/vnc_sink.c @@ -0,0 +1,113 @@ +#include "vnc_sink.h" + +#include + +#include "util/log.h" +#include "util/str.h" + +/** Downcast frame_sink to sc_vnc_sink */ +#define DOWNCAST(SINK) container_of(SINK, struct sc_vnc_sink, frame_sink) + + +/* +static int +run_vnc_sink(void *data) { +} + +static bool +sc_vnc_sink_open(struct sc_vnc_sink *vs, const AVCodecContext *ctx) { +} + +static void +sc_vnc_sink_close(struct sc_vnc_sink *vs) { +} + +static bool +sc_vnc_sink_push(struct sc_vnc_sink *vs, const AVFrame *frame) { +} +*/ + +static bool +sc_vnc_frame_sink_open(struct sc_frame_sink *sink, const AVCodecContext *ctx) { + assert(ctx->pix_fmt == AV_PIX_FMT_YUV420P); + struct sc_vnc_sink *vnc = DOWNCAST(sink); +} + +static void +sc_vnc_frame_sink_close(struct sc_frame_sink *sink) { +} + +void consume_frame(struct sc_vnc_sink* vnc, const AVFrame *frame) { + // XXX: ideally this would get "damage" regions from the decoder + // to prevent marking the entire screen as modified if only a small + // part changed + if(frame->width != vnc->scrWidth || frame->height != vnc->scrHeight) { + printf("RESIZE EVENT\n"); + if(vnc->ctx) { + sws_freeContext(vnc->ctx); + vnc->ctx = NULL; + } + } + if(vnc->ctx == NULL) { + vnc->scrWidth = frame->width; + vnc->scrHeight = frame->height; + vnc->ctx = sws_getContext(frame->width, frame->height, AV_PIX_FMT_YUV420P, + frame->width, frame->height, AV_PIX_FMT_RGBA, + 0, 0, 0, 0); + if(vnc->ctx == NULL) { + printf("could not make context\n"); + } + printf("good ctx\n"); + uint8_t *currentFrameBuffer = vnc->screen->frameBuffer; + uint8_t *newFrameBuffer = (uint8_t *)malloc(vnc->scrWidth*vnc->scrHeight*vnc->bpp); + rfbNewFramebuffer(vnc->screen, newFrameBuffer, vnc->scrWidth, vnc->scrHeight, 8, 3, vnc->bpp); + free(currentFrameBuffer); + } + assert(vnc->ctx != NULL); + + int linesize[1] = {frame->width*vnc->bpp}; + int* data[1] = {vnc->screen->frameBuffer}; + sws_scale(vnc->ctx, (const uint8_t * const *)frame->data, frame->linesize, 0, frame->height, data, linesize); + + rfbMarkRectAsModified(vnc->screen,0,0,frame->width,frame->height); +} + +static bool +sc_vnc_frame_sink_push(struct sc_frame_sink *sink, const AVFrame *frame) { + struct sc_vnc_sink *vnc = DOWNCAST(sink); + bool previous_skipped; + consume_frame(vnc, frame); + return true; +} + +bool +sc_vnc_sink_init(struct sc_vnc_sink *vs, const char *device_name) { + bool ok = sc_frame_buffer_init(&vs->fb); + if (!ok) { + return false; + } + static const struct sc_frame_sink_ops ops = { + .open = sc_vnc_frame_sink_open, + .close = sc_vnc_frame_sink_close, + .push = sc_vnc_frame_sink_push, + }; + + vs->frame_sink.ops = &ops; + vs->bpp = 4; + vs->screen = rfbGetScreen(0,NULL,32,32,8,3,vs->bpp); + vs->screen->frameBuffer = (uint8_t*)malloc(32*32*vs->bpp); + rfbInitServer(vs->screen); + rfbRunEventLoop(vs->screen,-1,TRUE); + return true; +} + +void +sc_vnc_sink_destroy(struct sc_vnc_sink *vs) { + if(vs->screen) { + free(vs->screen->frameBuffer); + rfbScreenCleanup(vs->screen); + } + if(vs->ctx) { + sws_freeContext(vs->ctx); + } +} diff --git a/app/src/vnc_sink.h b/app/src/vnc_sink.h new file mode 100644 index 0000000000..eddd3abe2b --- /dev/null +++ b/app/src/vnc_sink.h @@ -0,0 +1,48 @@ +#ifndef SC_VNC_SINK_H +#define SC_VNC_SINK_H + +#include "common.h" + +#include +#include +#include +#include + +#include "coords.h" +#include "trait/frame_sink.h" +#include "frame_buffer.h" +#include "util/tick.h" + +struct sc_vnc_sink { + struct sc_frame_sink frame_sink; // frame sink trait + + struct sc_frame_buffer fb; + AVFormatContext *format_ctx; + AVCodecContext *encoder_ctx; + struct SwsContext * ctx; + rfbScreenInfoPtr screen; + uint8_t frameBuffer; + uint16_t scrWidth; + uint16_t scrHeight; + uint8_t bpp; + + char *device_name; + + sc_thread thread; + sc_mutex mutex; + sc_cond cond; + bool has_frame; + bool stopped; + bool header_written; + + AVFrame *frame; + AVPacket *packet; +}; + +bool +sc_vnc_sink_init(struct sc_vnc_sink *vs, const char *device_name); + +void +sc_vnc_sink_destroy(struct sc_vnc_sink *vs); + +#endif From 0de04bcb7c1683f2dc441e7cde41a417068cc088 Mon Sep 17 00:00:00 2001 From: DavidVentura Date: Sun, 30 Apr 2023 17:39:59 +0200 Subject: [PATCH 2/8] allow not having a display if using a vnc server --- app/src/cli.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/src/cli.c b/app/src/cli.c index 343c1436d1..ad066e65ef 100644 --- a/app/src/cli.c +++ b/app/src/cli.c @@ -1899,9 +1899,10 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[], } #ifdef HAVE_V4L2 - if (!opts->display && !opts->record_filename && !opts->v4l2_device) { + if (!opts->display && !opts->record_filename && !opts->v4l2_device && !opts->vnc_server) { LOGE("-N/--no-display requires either screen recording (-r/--record)" - " or sink to v4l2loopback device (--v4l2-sink)"); + " or sink to v4l2loopback device (--v4l2-sink) or setting up a VNC server" + " (--vnc-server)"); return false; } From 3147c8d395231bdfc1c271f08df0a548e9fd92a5 Mon Sep 17 00:00:00 2001 From: DavidVentura Date: Sun, 30 Apr 2023 19:31:49 +0200 Subject: [PATCH 3/8] clean up comments & warnings; implement basic mouse support --- app/src/scrcpy.c | 5 ++- app/src/vnc_sink.c | 93 ++++++++++++++++++++++++++++++---------------- app/src/vnc_sink.h | 22 ++++------- 3 files changed, 72 insertions(+), 48 deletions(-) diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index daa0aabc6c..bc1f7bb63a 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -699,7 +699,7 @@ scrcpy(struct scrcpy_options *options) { } } if (options->vnc_server) { - if (!sc_vnc_sink_init(&s->vnc_sink, "my vnc server")) { + if (!sc_vnc_sink_init(&s->vnc_sink, "my vnc server", controller)) { printf("bad vnc init \n"); goto end; } @@ -799,6 +799,9 @@ scrcpy(struct scrcpy_options *options) { sc_v4l2_sink_destroy(&s->v4l2_sink); } #endif + if (vnc_sink_initialized) { + sc_vnc_sink_destroy(&s->vnc_sink); + } #ifdef HAVE_USB if (aoa_hid_initialized) { diff --git a/app/src/vnc_sink.c b/app/src/vnc_sink.c index 0ffec82baf..a8c8d11ec2 100644 --- a/app/src/vnc_sink.c +++ b/app/src/vnc_sink.c @@ -9,32 +9,17 @@ #define DOWNCAST(SINK) container_of(SINK, struct sc_vnc_sink, frame_sink) -/* -static int -run_vnc_sink(void *data) { -} - -static bool -sc_vnc_sink_open(struct sc_vnc_sink *vs, const AVCodecContext *ctx) { -} - -static void -sc_vnc_sink_close(struct sc_vnc_sink *vs) { -} - -static bool -sc_vnc_sink_push(struct sc_vnc_sink *vs, const AVFrame *frame) { -} -*/ - static bool sc_vnc_frame_sink_open(struct sc_frame_sink *sink, const AVCodecContext *ctx) { assert(ctx->pix_fmt == AV_PIX_FMT_YUV420P); - struct sc_vnc_sink *vnc = DOWNCAST(sink); + (void) sink; + (void) ctx; + return true; } static void sc_vnc_frame_sink_close(struct sc_frame_sink *sink) { + (void) sink; } void consume_frame(struct sc_vnc_sink* vnc, const AVFrame *frame) { @@ -58,34 +43,31 @@ void consume_frame(struct sc_vnc_sink* vnc, const AVFrame *frame) { printf("could not make context\n"); } printf("good ctx\n"); - uint8_t *currentFrameBuffer = vnc->screen->frameBuffer; - uint8_t *newFrameBuffer = (uint8_t *)malloc(vnc->scrWidth*vnc->scrHeight*vnc->bpp); + char *currentFrameBuffer = vnc->screen->frameBuffer; + char *newFrameBuffer = (char *)malloc(vnc->scrWidth*vnc->scrHeight*vnc->bpp); rfbNewFramebuffer(vnc->screen, newFrameBuffer, vnc->scrWidth, vnc->scrHeight, 8, 3, vnc->bpp); free(currentFrameBuffer); } assert(vnc->ctx != NULL); int linesize[1] = {frame->width*vnc->bpp}; - int* data[1] = {vnc->screen->frameBuffer}; + uint8_t *const data[1] = {(uint8_t*)vnc->screen->frameBuffer}; sws_scale(vnc->ctx, (const uint8_t * const *)frame->data, frame->linesize, 0, frame->height, data, linesize); - rfbMarkRectAsModified(vnc->screen,0,0,frame->width,frame->height); + rfbMarkRectAsModified(vnc->screen, 0, 0, frame->width, frame->height); } static bool sc_vnc_frame_sink_push(struct sc_frame_sink *sink, const AVFrame *frame) { struct sc_vnc_sink *vnc = DOWNCAST(sink); - bool previous_skipped; consume_frame(vnc, frame); return true; } bool -sc_vnc_sink_init(struct sc_vnc_sink *vs, const char *device_name) { - bool ok = sc_frame_buffer_init(&vs->fb); - if (!ok) { - return false; - } +sc_vnc_sink_init(struct sc_vnc_sink *vs, const char *device_name, struct sc_controller *controller) { + uint8_t placeholder_width = 32; + uint8_t placeholder_height = 32; static const struct sc_frame_sink_ops ops = { .open = sc_vnc_frame_sink_open, .close = sc_vnc_frame_sink_close, @@ -94,10 +76,16 @@ sc_vnc_sink_init(struct sc_vnc_sink *vs, const char *device_name) { vs->frame_sink.ops = &ops; vs->bpp = 4; - vs->screen = rfbGetScreen(0,NULL,32,32,8,3,vs->bpp); - vs->screen->frameBuffer = (uint8_t*)malloc(32*32*vs->bpp); + vs->screen = rfbGetScreen(0, NULL, placeholder_width, placeholder_height, 8, 3, vs->bpp); + vs->screen->desktopName = device_name; + vs->screen->alwaysShared = TRUE; + vs->screen->frameBuffer = (char *)malloc(placeholder_width * placeholder_height * vs->bpp); + vs->screen->ptrAddEvent = ptr_add_event; + vs->screen->screenData = vs; // XXX: any other way to pass a reference back? + vs->was_down = FALSE; + vs->controller = controller; rfbInitServer(vs->screen); - rfbRunEventLoop(vs->screen,-1,TRUE); + rfbRunEventLoop(vs->screen, -1, TRUE); // TODO: integrate into proper lifecycle return true; } @@ -111,3 +99,44 @@ sc_vnc_sink_destroy(struct sc_vnc_sink *vs) { sws_freeContext(vs->ctx); } } + +void +ptr_add_event(int buttonMask, int x, int y, rfbClientPtr cl) { + struct sc_vnc_sink *vnc = cl->screen->screenData; + // buttonMask is 3 bits: MOUSE_RIGHT | MOUSE_MIDDLE | MOUSE_LEFT + // value of 1 in that bit indicates it's being pressed; 0 indicates released + + // XXX: only doing left click + bool up = (buttonMask & 0x1) == 0; + /* TODO: this needs a screen + struct sc_point mouse = sc_screen_convert_window_to_frame_coords(im->screen, x, y); + printf("transformed to! x %d y %d \n", sc_point.x, sc_point.y); + */ + + + struct sc_control_msg msg; + struct sc_size screen_size = {vnc->scrWidth, vnc->scrHeight}; + struct sc_point point = {x, y}; + + msg.type = SC_CONTROL_MSG_TYPE_INJECT_TOUCH_EVENT; + if(vnc->was_down && !up) { + msg.inject_touch_event.action = AMOTION_EVENT_ACTION_MOVE; + } else { + msg.inject_touch_event.action = up ? AMOTION_EVENT_ACTION_UP : AMOTION_EVENT_ACTION_DOWN; + } + msg.inject_touch_event.position.screen_size = screen_size; + msg.inject_touch_event.position.point = point; + msg.inject_touch_event.pointer_id = POINTER_ID_VIRTUAL_FINGER; + // TODO: how to decide vs POINTER_ID_VIRTUAL_MOUSE? + + msg.inject_touch_event.pressure = up ? 0.0f : 1.0f; + msg.inject_touch_event.action_button = 0; + msg.inject_touch_event.buttons = 0; + + if (!sc_controller_push_msg(vnc->controller, &msg)) { + LOGW("Could not request 'inject virtual finger event'"); + } + + rfbDefaultPtrAddEvent(buttonMask, x, y, cl); + vnc->was_down = !up; +} diff --git a/app/src/vnc_sink.h b/app/src/vnc_sink.h index eddd3abe2b..f73d8a0fa9 100644 --- a/app/src/vnc_sink.h +++ b/app/src/vnc_sink.h @@ -9,40 +9,32 @@ #include #include "coords.h" +#include "control_msg.h" +#include "controller.h" #include "trait/frame_sink.h" #include "frame_buffer.h" #include "util/tick.h" struct sc_vnc_sink { struct sc_frame_sink frame_sink; // frame sink trait + struct sc_controller *controller; - struct sc_frame_buffer fb; - AVFormatContext *format_ctx; - AVCodecContext *encoder_ctx; struct SwsContext * ctx; rfbScreenInfoPtr screen; - uint8_t frameBuffer; uint16_t scrWidth; uint16_t scrHeight; uint8_t bpp; + bool was_down; char *device_name; - - sc_thread thread; - sc_mutex mutex; - sc_cond cond; - bool has_frame; - bool stopped; - bool header_written; - - AVFrame *frame; - AVPacket *packet; }; bool -sc_vnc_sink_init(struct sc_vnc_sink *vs, const char *device_name); +sc_vnc_sink_init(struct sc_vnc_sink *vs, const char *device_name, struct sc_controller *controller); void sc_vnc_sink_destroy(struct sc_vnc_sink *vs); +void +ptr_add_event(int buttonMask, int x, int y, rfbClientPtr cl); #endif From c01c61b72aa49bee33ee8e41f165d7e7af70f316 Mon Sep 17 00:00:00 2001 From: DavidVentura Date: Sun, 30 Apr 2023 19:36:18 +0200 Subject: [PATCH 4/8] whitespace --- app/src/cli.c | 4 +- app/src/scrcpy.c | 21 +++---- app/src/vnc_sink.c | 152 ++++++++++++++++++++++----------------------- app/src/vnc_sink.h | 14 ++--- 4 files changed, 95 insertions(+), 96 deletions(-) diff --git a/app/src/cli.c b/app/src/cli.c index ad066e65ef..02baa32b0f 100644 --- a/app/src/cli.c +++ b/app/src/cli.c @@ -1869,7 +1869,7 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[], break; case OPT_VNC_SERVER: opts->vnc_server = true; - break; + break; default: // getopt prints the error message on stderr return false; @@ -1902,7 +1902,7 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[], if (!opts->display && !opts->record_filename && !opts->v4l2_device && !opts->vnc_server) { LOGE("-N/--no-display requires either screen recording (-r/--record)" " or sink to v4l2loopback device (--v4l2-sink) or setting up a VNC server" - " (--vnc-server)"); + " (--vnc-server)"); return false; } diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index bc1f7bb63a..fa50e899b3 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -50,7 +50,6 @@ struct scrcpy { struct sc_decoder audio_decoder; struct sc_recorder recorder; struct sc_vnc_sink vnc_sink; - struct sc_delay_buffer vnc_buffer; struct sc_delay_buffer display_buffer; #ifdef HAVE_V4L2 struct sc_v4l2_sink v4l2_sink; @@ -698,15 +697,15 @@ scrcpy(struct scrcpy_options *options) { &s->audio_player.frame_sink); } } - if (options->vnc_server) { - if (!sc_vnc_sink_init(&s->vnc_sink, "my vnc server", controller)) { - printf("bad vnc init \n"); - goto end; - } - vnc_sink_initialized = true; + if (options->vnc_server) { + if (!sc_vnc_sink_init(&s->vnc_sink, "my vnc server", controller)) { + printf("bad vnc init \n"); + goto end; + } + vnc_sink_initialized = true; struct sc_frame_source *src = &s->video_decoder.frame_source; sc_frame_source_add_sink(src, &s->vnc_sink.frame_sink); - } + } #ifdef HAVE_V4L2 if (options->v4l2_device) { @@ -799,9 +798,9 @@ scrcpy(struct scrcpy_options *options) { sc_v4l2_sink_destroy(&s->v4l2_sink); } #endif - if (vnc_sink_initialized) { - sc_vnc_sink_destroy(&s->vnc_sink); - } + if (vnc_sink_initialized) { + sc_vnc_sink_destroy(&s->vnc_sink); + } #ifdef HAVE_USB if (aoa_hid_initialized) { diff --git a/app/src/vnc_sink.c b/app/src/vnc_sink.c index a8c8d11ec2..a764298cbc 100644 --- a/app/src/vnc_sink.c +++ b/app/src/vnc_sink.c @@ -12,62 +12,62 @@ static bool sc_vnc_frame_sink_open(struct sc_frame_sink *sink, const AVCodecContext *ctx) { assert(ctx->pix_fmt == AV_PIX_FMT_YUV420P); - (void) sink; - (void) ctx; - return true; + (void) sink; + (void) ctx; + return true; } static void sc_vnc_frame_sink_close(struct sc_frame_sink *sink) { - (void) sink; + (void) sink; } void consume_frame(struct sc_vnc_sink* vnc, const AVFrame *frame) { - // XXX: ideally this would get "damage" regions from the decoder - // to prevent marking the entire screen as modified if only a small - // part changed - if(frame->width != vnc->scrWidth || frame->height != vnc->scrHeight) { - printf("RESIZE EVENT\n"); - if(vnc->ctx) { - sws_freeContext(vnc->ctx); - vnc->ctx = NULL; - } - } - if(vnc->ctx == NULL) { - vnc->scrWidth = frame->width; - vnc->scrHeight = frame->height; - vnc->ctx = sws_getContext(frame->width, frame->height, AV_PIX_FMT_YUV420P, - frame->width, frame->height, AV_PIX_FMT_RGBA, - 0, 0, 0, 0); - if(vnc->ctx == NULL) { - printf("could not make context\n"); - } - printf("good ctx\n"); - char *currentFrameBuffer = vnc->screen->frameBuffer; - char *newFrameBuffer = (char *)malloc(vnc->scrWidth*vnc->scrHeight*vnc->bpp); - rfbNewFramebuffer(vnc->screen, newFrameBuffer, vnc->scrWidth, vnc->scrHeight, 8, 3, vnc->bpp); - free(currentFrameBuffer); - } - assert(vnc->ctx != NULL); - - int linesize[1] = {frame->width*vnc->bpp}; - uint8_t *const data[1] = {(uint8_t*)vnc->screen->frameBuffer}; - sws_scale(vnc->ctx, (const uint8_t * const *)frame->data, frame->linesize, 0, frame->height, data, linesize); - - rfbMarkRectAsModified(vnc->screen, 0, 0, frame->width, frame->height); + // XXX: ideally this would get "damage" regions from the decoder + // to prevent marking the entire screen as modified if only a small + // part changed + if(frame->width != vnc->scrWidth || frame->height != vnc->scrHeight) { + printf("RESIZE EVENT\n"); + if(vnc->ctx) { + sws_freeContext(vnc->ctx); + vnc->ctx = NULL; + } + } + if(vnc->ctx == NULL) { + vnc->scrWidth = frame->width; + vnc->scrHeight = frame->height; + vnc->ctx = sws_getContext(frame->width, frame->height, AV_PIX_FMT_YUV420P, + frame->width, frame->height, AV_PIX_FMT_RGBA, + 0, 0, 0, 0); + if(vnc->ctx == NULL) { + printf("could not make context\n"); + } + printf("good ctx\n"); + char *currentFrameBuffer = vnc->screen->frameBuffer; + char *newFrameBuffer = (char *)malloc(vnc->scrWidth*vnc->scrHeight*vnc->bpp); + rfbNewFramebuffer(vnc->screen, newFrameBuffer, vnc->scrWidth, vnc->scrHeight, 8, 3, vnc->bpp); + free(currentFrameBuffer); + } + assert(vnc->ctx != NULL); + + int linesize[1] = {frame->width*vnc->bpp}; + uint8_t *const data[1] = {(uint8_t*)vnc->screen->frameBuffer}; + sws_scale(vnc->ctx, (const uint8_t * const *)frame->data, frame->linesize, 0, frame->height, data, linesize); + + rfbMarkRectAsModified(vnc->screen, 0, 0, frame->width, frame->height); } static bool sc_vnc_frame_sink_push(struct sc_frame_sink *sink, const AVFrame *frame) { struct sc_vnc_sink *vnc = DOWNCAST(sink); - consume_frame(vnc, frame); - return true; + consume_frame(vnc, frame); + return true; } bool sc_vnc_sink_init(struct sc_vnc_sink *vs, const char *device_name, struct sc_controller *controller) { - uint8_t placeholder_width = 32; - uint8_t placeholder_height = 32; + uint8_t placeholder_width = 32; + uint8_t placeholder_height = 32; static const struct sc_frame_sink_ops ops = { .open = sc_vnc_frame_sink_open, .close = sc_vnc_frame_sink_close, @@ -75,59 +75,59 @@ sc_vnc_sink_init(struct sc_vnc_sink *vs, const char *device_name, struct sc_cont }; vs->frame_sink.ops = &ops; - vs->bpp = 4; - vs->screen = rfbGetScreen(0, NULL, placeholder_width, placeholder_height, 8, 3, vs->bpp); - vs->screen->desktopName = device_name; - vs->screen->alwaysShared = TRUE; - vs->screen->frameBuffer = (char *)malloc(placeholder_width * placeholder_height * vs->bpp); - vs->screen->ptrAddEvent = ptr_add_event; - vs->screen->screenData = vs; // XXX: any other way to pass a reference back? - vs->was_down = FALSE; - vs->controller = controller; - rfbInitServer(vs->screen); - rfbRunEventLoop(vs->screen, -1, TRUE); // TODO: integrate into proper lifecycle - return true; + vs->bpp = 4; + vs->screen = rfbGetScreen(0, NULL, placeholder_width, placeholder_height, 8, 3, vs->bpp); + vs->screen->desktopName = device_name; + vs->screen->alwaysShared = TRUE; + vs->screen->frameBuffer = (char *)malloc(placeholder_width * placeholder_height * vs->bpp); + vs->screen->ptrAddEvent = ptr_add_event; + vs->screen->screenData = vs; // XXX: any other way to pass a reference back? + vs->was_down = FALSE; + vs->controller = controller; + rfbInitServer(vs->screen); + rfbRunEventLoop(vs->screen, -1, TRUE); // TODO: integrate into proper lifecycle + return true; } void sc_vnc_sink_destroy(struct sc_vnc_sink *vs) { - if(vs->screen) { - free(vs->screen->frameBuffer); - rfbScreenCleanup(vs->screen); - } - if(vs->ctx) { - sws_freeContext(vs->ctx); - } + if(vs->screen) { + free(vs->screen->frameBuffer); + rfbScreenCleanup(vs->screen); + } + if(vs->ctx) { + sws_freeContext(vs->ctx); + } } void ptr_add_event(int buttonMask, int x, int y, rfbClientPtr cl) { - struct sc_vnc_sink *vnc = cl->screen->screenData; - // buttonMask is 3 bits: MOUSE_RIGHT | MOUSE_MIDDLE | MOUSE_LEFT - // value of 1 in that bit indicates it's being pressed; 0 indicates released + struct sc_vnc_sink *vnc = cl->screen->screenData; + // buttonMask is 3 bits: MOUSE_RIGHT | MOUSE_MIDDLE | MOUSE_LEFT + // value of 1 in that bit indicates it's being pressed; 0 indicates released - // XXX: only doing left click + // TODO: only doing left click bool up = (buttonMask & 0x1) == 0; - /* TODO: this needs a screen + /* TODO: this needs a screen struct sc_point mouse = sc_screen_convert_window_to_frame_coords(im->screen, x, y); - printf("transformed to! x %d y %d \n", sc_point.x, sc_point.y); - */ + printf("transformed to! x %d y %d \n", sc_point.x, sc_point.y); + */ struct sc_control_msg msg; - struct sc_size screen_size = {vnc->scrWidth, vnc->scrHeight}; - struct sc_point point = {x, y}; + struct sc_size screen_size = {vnc->scrWidth, vnc->scrHeight}; + struct sc_point point = {x, y}; msg.type = SC_CONTROL_MSG_TYPE_INJECT_TOUCH_EVENT; - if(vnc->was_down && !up) { - msg.inject_touch_event.action = AMOTION_EVENT_ACTION_MOVE; - } else { - msg.inject_touch_event.action = up ? AMOTION_EVENT_ACTION_UP : AMOTION_EVENT_ACTION_DOWN; - } + if(vnc->was_down && !up) { + msg.inject_touch_event.action = AMOTION_EVENT_ACTION_MOVE; + } else { + msg.inject_touch_event.action = up ? AMOTION_EVENT_ACTION_UP : AMOTION_EVENT_ACTION_DOWN; + } msg.inject_touch_event.position.screen_size = screen_size; msg.inject_touch_event.position.point = point; msg.inject_touch_event.pointer_id = POINTER_ID_VIRTUAL_FINGER; - // TODO: how to decide vs POINTER_ID_VIRTUAL_MOUSE? + // TODO: how to decide vs POINTER_ID_VIRTUAL_MOUSE? msg.inject_touch_event.pressure = up ? 0.0f : 1.0f; msg.inject_touch_event.action_button = 0; @@ -137,6 +137,6 @@ ptr_add_event(int buttonMask, int x, int y, rfbClientPtr cl) { LOGW("Could not request 'inject virtual finger event'"); } - rfbDefaultPtrAddEvent(buttonMask, x, y, cl); - vnc->was_down = !up; + rfbDefaultPtrAddEvent(buttonMask, x, y, cl); + vnc->was_down = !up; } diff --git a/app/src/vnc_sink.h b/app/src/vnc_sink.h index f73d8a0fa9..3bd837a86f 100644 --- a/app/src/vnc_sink.h +++ b/app/src/vnc_sink.h @@ -17,15 +17,15 @@ struct sc_vnc_sink { struct sc_frame_sink frame_sink; // frame sink trait - struct sc_controller *controller; + struct sc_controller *controller; - struct SwsContext * ctx; - rfbScreenInfoPtr screen; - uint16_t scrWidth; - uint16_t scrHeight; - uint8_t bpp; + struct SwsContext * ctx; + rfbScreenInfoPtr screen; + uint16_t scrWidth; + uint16_t scrHeight; + uint8_t bpp; - bool was_down; + bool was_down; char *device_name; }; From d43e9c0839ee07ceb6fce6ac6a0bd295ce6f4b3a Mon Sep 17 00:00:00 2001 From: DavidVentura Date: Sun, 30 Apr 2023 19:38:35 +0200 Subject: [PATCH 5/8] remove comment --- app/meson.build | 1 - 1 file changed, 1 deletion(-) diff --git a/app/meson.build b/app/meson.build index 8f0993d58d..8efcea2240 100644 --- a/app/meson.build +++ b/app/meson.build @@ -109,7 +109,6 @@ if not crossbuild_windows dependency('libavutil'), dependency('libswresample'), dependency('sdl2', version: '>= 2.0.5'), - #cc.find_library('swscale-5', dirs: ffmpeg_bin_dir), dependency('libswscale'), dependency('libvncserver'), ] From 378624c24bb1ac2a36efc59e498725aa9051986e Mon Sep 17 00:00:00 2001 From: DavidVentura Date: Mon, 1 May 2023 16:56:49 +0200 Subject: [PATCH 6/8] remove debug --- app/src/vnc_sink.c | 1 - 1 file changed, 1 deletion(-) diff --git a/app/src/vnc_sink.c b/app/src/vnc_sink.c index a764298cbc..c2b45975bb 100644 --- a/app/src/vnc_sink.c +++ b/app/src/vnc_sink.c @@ -27,7 +27,6 @@ void consume_frame(struct sc_vnc_sink* vnc, const AVFrame *frame) { // to prevent marking the entire screen as modified if only a small // part changed if(frame->width != vnc->scrWidth || frame->height != vnc->scrHeight) { - printf("RESIZE EVENT\n"); if(vnc->ctx) { sws_freeContext(vnc->ctx); vnc->ctx = NULL; From 6d2682eca58d44eafad1d89e67fe0943807f7cb4 Mon Sep 17 00:00:00 2001 From: DavidVentura Date: Mon, 1 May 2023 17:24:56 +0200 Subject: [PATCH 7/8] make VNC optional --- app/meson.build | 15 ++++++++++++--- app/src/cli.c | 30 ++++++++++++++++++++---------- app/src/scrcpy.c | 14 ++++++++++++-- meson_options.txt | 1 + 4 files changed, 45 insertions(+), 15 deletions(-) diff --git a/app/meson.build b/app/meson.build index 8efcea2240..ac4f6e0fdc 100644 --- a/app/meson.build +++ b/app/meson.build @@ -31,7 +31,6 @@ src = [ 'src/screen.c', 'src/server.c', 'src/version.c', - 'src/vnc_sink.c', 'src/trait/frame_source.c', 'src/trait/packet_source.c', 'src/util/acksync.c', @@ -84,6 +83,11 @@ if v4l2_support src += [ 'src/v4l2_sink.c' ] endif +vnc_support = get_option('vnc') and host_machine.system() == 'linux' +if vnc_support + src += [ 'src/vnc_sink.c' ] +endif + usb_support = get_option('usb') if usb_support src += [ @@ -109,13 +113,15 @@ if not crossbuild_windows dependency('libavutil'), dependency('libswresample'), dependency('sdl2', version: '>= 2.0.5'), - dependency('libswscale'), - dependency('libvncserver'), ] if v4l2_support dependencies += dependency('libavdevice') endif + if vnc_support + dependencies += dependency('libswscale') + dependencies += dependency('libvncserver') + endif if usb_support dependencies += dependency('libusb-1.0') @@ -217,6 +223,9 @@ conf.set('SERVER_DEBUGGER_METHOD_NEW', get_option('server_debugger_method') == ' # enable V4L2 support (linux only) conf.set('HAVE_V4L2', v4l2_support) +# enable libvnc support +conf.set('HAVE_VNC', vnc_support) + # enable HID over AOA support (linux only) conf.set('HAVE_USB', usb_support) diff --git a/app/src/cli.c b/app/src/cli.c index 02baa32b0f..fbe7b97eb2 100644 --- a/app/src/cli.c +++ b/app/src/cli.c @@ -1868,7 +1868,12 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[], } break; case OPT_VNC_SERVER: +#ifdef HAVE_VNC opts->vnc_server = true; +#else + LOGE("VNC (--vnc-server) is disabled."); + return false; +#endif break; default: // getopt prints the error message on stderr @@ -1898,11 +1903,22 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[], return false; } + bool has_sink = opts->record_filename; +#ifdef HAVE_V4L2 + has_sink |= (opts->v4l2_device != NULL); +#endif +#ifdef HAVE_VNC + has_sink |= opts->vnc_server; +#endif + if (!opts->display && !has_sink) { + LOGE("-N/--no-display requires at least one of the following to be set: "); + LOGE("* screen recording (-r/--record)"); #ifdef HAVE_V4L2 - if (!opts->display && !opts->record_filename && !opts->v4l2_device && !opts->vnc_server) { - LOGE("-N/--no-display requires either screen recording (-r/--record)" - " or sink to v4l2loopback device (--v4l2-sink) or setting up a VNC server" - " (--vnc-server)"); + LOGE("* sink to v4l2loopback device (--v4l2-sink)"); +#endif +#ifdef HAVE_VNC + LOGE("* setting up a VNC server (--vnc-server)"); +#endif return false; } @@ -1924,12 +1940,6 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[], LOGE("V4L2 buffer value without V4L2 sink\n"); return false; } -#else - if (!opts->display && !opts->record_filename) { - LOGE("-N/--no-display requires screen recording (-r/--record)"); - return false; - } -#endif if (opts->audio && !opts->display && !opts->record_filename) { LOGI("No display and no recording: audio disabled"); diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index fa50e899b3..b4d95d01df 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -25,7 +25,6 @@ #include "recorder.h" #include "screen.h" #include "server.h" -#include "vnc_sink.h" #ifdef HAVE_USB # include "usb/aoa_hid.h" # include "usb/hid_keyboard.h" @@ -39,6 +38,9 @@ #ifdef HAVE_V4L2 # include "v4l2_sink.h" #endif +#ifdef HAVE_VNC +#include "vnc_sink.h" +#endif struct scrcpy { struct sc_server server; @@ -49,8 +51,10 @@ struct scrcpy { struct sc_decoder video_decoder; struct sc_decoder audio_decoder; struct sc_recorder recorder; - struct sc_vnc_sink vnc_sink; struct sc_delay_buffer display_buffer; +#ifdef HAVE_VNC + struct sc_vnc_sink vnc_sink; +#endif #ifdef HAVE_V4L2 struct sc_v4l2_sink v4l2_sink; struct sc_delay_buffer v4l2_buffer; @@ -313,7 +317,9 @@ scrcpy(struct scrcpy_options *options) { #ifdef HAVE_V4L2 bool v4l2_sink_initialized = false; #endif +#ifdef HAVE_VNC bool vnc_sink_initialized = false; +#endif bool video_demuxer_started = false; bool audio_demuxer_started = false; #ifdef HAVE_USB @@ -697,6 +703,7 @@ scrcpy(struct scrcpy_options *options) { &s->audio_player.frame_sink); } } +#ifdef HAVE_VNC if (options->vnc_server) { if (!sc_vnc_sink_init(&s->vnc_sink, "my vnc server", controller)) { printf("bad vnc init \n"); @@ -706,6 +713,7 @@ scrcpy(struct scrcpy_options *options) { struct sc_frame_source *src = &s->video_decoder.frame_source; sc_frame_source_add_sink(src, &s->vnc_sink.frame_sink); } +#endif #ifdef HAVE_V4L2 if (options->v4l2_device) { @@ -798,9 +806,11 @@ scrcpy(struct scrcpy_options *options) { sc_v4l2_sink_destroy(&s->v4l2_sink); } #endif +#ifdef HAVE_VNC if (vnc_sink_initialized) { sc_vnc_sink_destroy(&s->vnc_sink); } +#endif #ifdef HAVE_USB if (aoa_hid_initialized) { diff --git a/meson_options.txt b/meson_options.txt index d103069460..61dbdcadee 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -5,4 +5,5 @@ option('portable', type: 'boolean', value: false, description: 'Use scrcpy-serve option('server_debugger', type: 'boolean', value: false, description: 'Run a server debugger and wait for a client to be attached') option('server_debugger_method', type: 'combo', choices: ['old', 'new'], value: 'new', description: 'Select the debugger method (Android < 9: "old", Android >= 9: "new")') option('v4l2', type: 'boolean', value: true, description: 'Enable V4L2 feature when supported') +option('vnc', type: 'boolean', value: true, description: 'Enable VNC server feature when supported') option('usb', type: 'boolean', value: true, description: 'Enable HID/OTG features when supported') From ad435fec747f2d3aae77ac650fad0b91181c411b Mon Sep 17 00:00:00 2001 From: DavidVentura Date: Mon, 1 May 2023 17:35:56 +0200 Subject: [PATCH 8/8] small cleanups --- app/src/vnc_sink.c | 29 ++++++++++------------------- 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/app/src/vnc_sink.c b/app/src/vnc_sink.c index c2b45975bb..bbd03ef91b 100644 --- a/app/src/vnc_sink.c +++ b/app/src/vnc_sink.c @@ -22,7 +22,9 @@ sc_vnc_frame_sink_close(struct sc_frame_sink *sink) { (void) sink; } -void consume_frame(struct sc_vnc_sink* vnc, const AVFrame *frame) { +static bool +sc_vnc_frame_sink_push(struct sc_frame_sink *sink, const AVFrame *frame) { + struct sc_vnc_sink *vnc = DOWNCAST(sink); // XXX: ideally this would get "damage" regions from the decoder // to prevent marking the entire screen as modified if only a small // part changed @@ -39,9 +41,9 @@ void consume_frame(struct sc_vnc_sink* vnc, const AVFrame *frame) { frame->width, frame->height, AV_PIX_FMT_RGBA, 0, 0, 0, 0); if(vnc->ctx == NULL) { - printf("could not make context\n"); + LOGE("could not make context"); + return false; } - printf("good ctx\n"); char *currentFrameBuffer = vnc->screen->frameBuffer; char *newFrameBuffer = (char *)malloc(vnc->scrWidth*vnc->scrHeight*vnc->bpp); rfbNewFramebuffer(vnc->screen, newFrameBuffer, vnc->scrWidth, vnc->scrHeight, 8, 3, vnc->bpp); @@ -54,12 +56,6 @@ void consume_frame(struct sc_vnc_sink* vnc, const AVFrame *frame) { sws_scale(vnc->ctx, (const uint8_t * const *)frame->data, frame->linesize, 0, frame->height, data, linesize); rfbMarkRectAsModified(vnc->screen, 0, 0, frame->width, frame->height); -} - -static bool -sc_vnc_frame_sink_push(struct sc_frame_sink *sink, const AVFrame *frame) { - struct sc_vnc_sink *vnc = DOWNCAST(sink); - consume_frame(vnc, frame); return true; } @@ -77,20 +73,21 @@ sc_vnc_sink_init(struct sc_vnc_sink *vs, const char *device_name, struct sc_cont vs->bpp = 4; vs->screen = rfbGetScreen(0, NULL, placeholder_width, placeholder_height, 8, 3, vs->bpp); vs->screen->desktopName = device_name; - vs->screen->alwaysShared = TRUE; + vs->screen->alwaysShared = true; vs->screen->frameBuffer = (char *)malloc(placeholder_width * placeholder_height * vs->bpp); vs->screen->ptrAddEvent = ptr_add_event; - vs->screen->screenData = vs; // XXX: any other way to pass a reference back? - vs->was_down = FALSE; + vs->screen->screenData = vs; + vs->was_down = false; vs->controller = controller; rfbInitServer(vs->screen); - rfbRunEventLoop(vs->screen, -1, TRUE); // TODO: integrate into proper lifecycle + rfbRunEventLoop(vs->screen, -1, true); // TODO: integrate into proper lifecycle return true; } void sc_vnc_sink_destroy(struct sc_vnc_sink *vs) { if(vs->screen) { + rfbShutdownServer(vs->screen, true); free(vs->screen->frameBuffer); rfbScreenCleanup(vs->screen); } @@ -107,12 +104,6 @@ ptr_add_event(int buttonMask, int x, int y, rfbClientPtr cl) { // TODO: only doing left click bool up = (buttonMask & 0x1) == 0; - /* TODO: this needs a screen - struct sc_point mouse = sc_screen_convert_window_to_frame_coords(im->screen, x, y); - printf("transformed to! x %d y %d \n", sc_point.x, sc_point.y); - */ - - struct sc_control_msg msg; struct sc_size screen_size = {vnc->scrWidth, vnc->scrHeight}; struct sc_point point = {x, y};