1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "remoting/host/input_injector.h"
7 #include <X11/extensions/XInput.h>
8 #include <X11/extensions/XTest.h>
10 #include <X11/XKBlib.h>
11 #undef Status // Xlib.h #defines this, which breaks protobuf headers.
15 #include "base/basictypes.h"
16 #include "base/bind.h"
17 #include "base/compiler_specific.h"
18 #include "base/location.h"
19 #include "base/single_thread_task_runner.h"
20 #include "base/strings/utf_string_conversion_utils.h"
21 #include "remoting/base/logging.h"
22 #if defined(OS_CHROMEOS)
23 #include "remoting/host/chromeos/point_transformer.h"
25 #include "remoting/host/clipboard.h"
26 #include "remoting/host/linux/unicode_to_keysym.h"
27 #include "remoting/proto/internal.pb.h"
28 #include "remoting/protocol/usb_key_codes.h"
29 #include "third_party/webrtc/modules/desktop_capture/desktop_geometry.h"
30 #include "ui/events/keycodes/dom/keycode_converter.h"
36 using protocol::ClipboardEvent
;
37 using protocol::KeyEvent
;
38 using protocol::TextEvent
;
39 using protocol::MouseEvent
;
40 using protocol::TouchEvent
;
42 bool FindKeycodeForKeySym(Display
* display
,
45 uint32_t* modifiers
) {
46 *keycode
= XKeysymToKeycode(display
, key_sym
);
48 const uint32_t kModifiersToTry
[] = {
59 // TODO(sergeyu): Is there a better way to find modifiers state?
60 for (size_t i
= 0; i
< arraysize(kModifiersToTry
); ++i
) {
61 unsigned long key_sym_with_mods
;
62 if (XkbLookupKeySym(display
, *keycode
, kModifiersToTry
[i
], nullptr,
63 &key_sym_with_mods
) &&
64 key_sym_with_mods
== key_sym
) {
65 *modifiers
= kModifiersToTry
[i
];
73 // Finds a keycode and set of modifiers that generate character with the
74 // specified |code_point|.
75 bool FindKeycodeForUnicode(Display
* display
,
78 uint32_t* modifiers
) {
79 std::vector
<uint32_t> keysyms
;
80 GetKeySymsForUnicode(code_point
, &keysyms
);
82 for (std::vector
<uint32_t>::iterator it
= keysyms
.begin();
83 it
!= keysyms
.end(); ++it
) {
84 if (FindKeycodeForKeySym(display
, *it
, keycode
, modifiers
)) {
92 bool IsModifierKey(int usb_keycode
) {
93 return usb_keycode
== kUsbLeftControl
||
94 usb_keycode
== kUsbLeftShift
||
95 usb_keycode
== kUsbLeftAlt
||
96 usb_keycode
== kUsbLeftOs
||
97 usb_keycode
== kUsbRightControl
||
98 usb_keycode
== kUsbRightShift
||
99 usb_keycode
== kUsbRightAlt
||
100 usb_keycode
== kUsbRightOs
;
103 // Pixel-to-wheel-ticks conversion ratio used by GTK.
104 // From third_party/WebKit/Source/web/gtk/WebInputEventFactory.cpp .
105 const float kWheelTicksPerPixel
= 3.0f
/ 160.0f
;
107 // A class to generate events on X11.
108 class InputInjectorX11
: public InputInjector
{
110 explicit InputInjectorX11(
111 scoped_refptr
<base::SingleThreadTaskRunner
> task_runner
);
112 ~InputInjectorX11() override
;
116 // Clipboard stub interface.
117 void InjectClipboardEvent(const ClipboardEvent
& event
) override
;
119 // InputStub interface.
120 void InjectKeyEvent(const KeyEvent
& event
) override
;
121 void InjectTextEvent(const TextEvent
& event
) override
;
122 void InjectMouseEvent(const MouseEvent
& event
) override
;
123 void InjectTouchEvent(const TouchEvent
& event
) override
;
125 // InputInjector interface.
126 void Start(scoped_ptr
<protocol::ClipboardStub
> client_clipboard
) override
;
129 // The actual implementation resides in InputInjectorX11::Core class.
130 class Core
: public base::RefCountedThreadSafe
<Core
> {
132 explicit Core(scoped_refptr
<base::SingleThreadTaskRunner
> task_runner
);
136 // Mirrors the ClipboardStub interface.
137 void InjectClipboardEvent(const ClipboardEvent
& event
);
139 // Mirrors the InputStub interface.
140 void InjectKeyEvent(const KeyEvent
& event
);
141 void InjectTextEvent(const TextEvent
& event
);
142 void InjectMouseEvent(const MouseEvent
& event
);
144 // Mirrors the InputInjector interface.
145 void Start(scoped_ptr
<protocol::ClipboardStub
> client_clipboard
);
150 friend class base::RefCountedThreadSafe
<Core
>;
153 void InitClipboard();
155 // Queries whether keyboard auto-repeat is globally enabled. This is used
156 // to decide whether to temporarily disable then restore this setting. If
157 // auto-repeat has already been disabled, this class should leave it
159 bool IsAutoRepeatEnabled();
161 // Enables or disables keyboard auto-repeat globally.
162 void SetAutoRepeatEnabled(bool enabled
);
164 void InjectScrollWheelClicks(int button
, int count
);
165 // Compensates for global button mappings and resets the XTest device
167 void InitMouseButtonMap();
168 int MouseButtonToX11ButtonNumber(MouseEvent::MouseButton button
);
169 int HorizontalScrollWheelToX11ButtonNumber(int dx
);
170 int VerticalScrollWheelToX11ButtonNumber(int dy
);
172 scoped_refptr
<base::SingleThreadTaskRunner
> task_runner_
;
174 std::set
<int> pressed_keys_
;
175 webrtc::DesktopVector latest_mouse_position_
;
176 float wheel_ticks_x_
;
177 float wheel_ticks_y_
;
179 // X11 graphics context.
183 int test_event_base_
;
184 int test_error_base_
;
186 // Number of buttons we support.
187 // Left, Right, Middle, VScroll Up/Down, HScroll Left/Right.
188 static const int kNumPointerButtons
= 7;
190 int pointer_button_map_
[kNumPointerButtons
];
192 #if defined(OS_CHROMEOS)
193 PointTransformer point_transformer_
;
196 scoped_ptr
<Clipboard
> clipboard_
;
198 bool saved_auto_repeat_enabled_
;
200 DISALLOW_COPY_AND_ASSIGN(Core
);
203 scoped_refptr
<Core
> core_
;
205 DISALLOW_COPY_AND_ASSIGN(InputInjectorX11
);
208 InputInjectorX11::InputInjectorX11(
209 scoped_refptr
<base::SingleThreadTaskRunner
> task_runner
) {
210 core_
= new Core(task_runner
);
213 InputInjectorX11::~InputInjectorX11() {
217 bool InputInjectorX11::Init() {
218 return core_
->Init();
221 void InputInjectorX11::InjectClipboardEvent(const ClipboardEvent
& event
) {
222 core_
->InjectClipboardEvent(event
);
225 void InputInjectorX11::InjectKeyEvent(const KeyEvent
& event
) {
226 core_
->InjectKeyEvent(event
);
229 void InputInjectorX11::InjectTextEvent(const TextEvent
& event
) {
230 core_
->InjectTextEvent(event
);
233 void InputInjectorX11::InjectMouseEvent(const MouseEvent
& event
) {
234 core_
->InjectMouseEvent(event
);
237 void InputInjectorX11::InjectTouchEvent(const TouchEvent
& event
) {
238 NOTIMPLEMENTED() << "Raw touch event injection not implemented for X11.";
241 void InputInjectorX11::Start(
242 scoped_ptr
<protocol::ClipboardStub
> client_clipboard
) {
243 core_
->Start(client_clipboard
.Pass());
246 InputInjectorX11::Core::Core(
247 scoped_refptr
<base::SingleThreadTaskRunner
> task_runner
)
248 : task_runner_(task_runner
),
249 latest_mouse_position_(-1, -1),
250 wheel_ticks_x_(0.0f
),
251 wheel_ticks_y_(0.0f
),
252 display_(XOpenDisplay(nullptr)),
253 root_window_(BadValue
),
254 saved_auto_repeat_enabled_(false) {
257 bool InputInjectorX11::Core::Init() {
260 if (!task_runner_
->BelongsToCurrentThread())
261 task_runner_
->PostTask(FROM_HERE
, base::Bind(&Core::InitClipboard
, this));
263 root_window_
= RootWindow(display_
, DefaultScreen(display_
));
264 if (root_window_
== BadValue
) {
265 LOG(ERROR
) << "Unable to get the root window";
269 // TODO(ajwong): Do we want to check the major/minor version at all for XTest?
272 if (!XTestQueryExtension(display_
, &test_event_base_
, &test_error_base_
,
274 LOG(ERROR
) << "Server does not support XTest.";
277 InitMouseButtonMap();
281 void InputInjectorX11::Core::InjectClipboardEvent(
282 const ClipboardEvent
& event
) {
283 if (!task_runner_
->BelongsToCurrentThread()) {
284 task_runner_
->PostTask(
285 FROM_HERE
, base::Bind(&Core::InjectClipboardEvent
, this, event
));
289 // |clipboard_| will ignore unknown MIME-types, and verify the data's format.
290 clipboard_
->InjectClipboardEvent(event
);
293 void InputInjectorX11::Core::InjectKeyEvent(const KeyEvent
& event
) {
294 // HostEventDispatcher should filter events missing the pressed field.
295 if (!event
.has_pressed() || !event
.has_usb_keycode())
298 if (!task_runner_
->BelongsToCurrentThread()) {
299 task_runner_
->PostTask(FROM_HERE
,
300 base::Bind(&Core::InjectKeyEvent
, this, event
));
305 ui::KeycodeConverter::UsbKeycodeToNativeKeycode(event
.usb_keycode());
307 VLOG(3) << "Converting USB keycode: " << std::hex
<< event
.usb_keycode()
308 << " to keycode: " << keycode
<< std::dec
;
310 // Ignore events which can't be mapped.
311 if (keycode
== ui::KeycodeConverter::InvalidNativeKeycode())
314 if (event
.pressed()) {
315 if (pressed_keys_
.find(keycode
) != pressed_keys_
.end()) {
316 // Ignore repeats for modifier keys.
317 if (IsModifierKey(event
.usb_keycode()))
319 // Key is already held down, so lift the key up to ensure this repeated
320 // press takes effect.
321 XTestFakeKeyEvent(display_
, keycode
, False
, CurrentTime
);
324 if (pressed_keys_
.empty()) {
325 // Disable auto-repeat, if necessary, to avoid triggering auto-repeat
326 // if network congestion delays the key-up event from the client.
327 saved_auto_repeat_enabled_
= IsAutoRepeatEnabled();
328 if (saved_auto_repeat_enabled_
)
329 SetAutoRepeatEnabled(false);
331 pressed_keys_
.insert(keycode
);
333 pressed_keys_
.erase(keycode
);
334 if (pressed_keys_
.empty()) {
335 // Re-enable auto-repeat, if necessary, when all keys are released.
336 if (saved_auto_repeat_enabled_
)
337 SetAutoRepeatEnabled(true);
341 XTestFakeKeyEvent(display_
, keycode
, event
.pressed(), CurrentTime
);
345 void InputInjectorX11::Core::InjectTextEvent(const TextEvent
& event
) {
346 if (!task_runner_
->BelongsToCurrentThread()) {
347 task_runner_
->PostTask(FROM_HERE
,
348 base::Bind(&Core::InjectTextEvent
, this, event
));
352 // Release all keys before injecting text event. This is necessary to avoid
353 // any interference with the currently pressed keys. E.g. if Shift is pressed
354 // when TextEvent is received.
355 for (int key
: pressed_keys_
) {
356 XTestFakeKeyEvent(display_
, key
, False
, CurrentTime
);
358 pressed_keys_
.clear();
360 const std::string text
= event
.text();
361 for (int32 index
= 0; index
< static_cast<int32
>(text
.size()); ++index
) {
363 if (!base::ReadUnicodeCharacter(
364 text
.c_str(), text
.size(), &index
, &code_point
)) {
370 if (!FindKeycodeForUnicode(display_
, code_point
, &keycode
, &modifiers
))
373 XkbLockModifiers(display_
, XkbUseCoreKbd
, modifiers
, modifiers
);
375 XTestFakeKeyEvent(display_
, keycode
, True
, CurrentTime
);
376 XTestFakeKeyEvent(display_
, keycode
, False
, CurrentTime
);
378 XkbLockModifiers(display_
, XkbUseCoreKbd
, modifiers
, 0);
384 InputInjectorX11::Core::~Core() {
385 CHECK(pressed_keys_
.empty());
388 void InputInjectorX11::Core::InitClipboard() {
389 DCHECK(task_runner_
->BelongsToCurrentThread());
390 clipboard_
= Clipboard::Create();
393 bool InputInjectorX11::Core::IsAutoRepeatEnabled() {
394 XKeyboardState state
;
395 if (!XGetKeyboardControl(display_
, &state
)) {
396 LOG(ERROR
) << "Failed to get keyboard auto-repeat status, assuming ON.";
399 return state
.global_auto_repeat
== AutoRepeatModeOn
;
402 void InputInjectorX11::Core::SetAutoRepeatEnabled(bool mode
) {
403 XKeyboardControl control
;
404 control
.auto_repeat_mode
= mode
? AutoRepeatModeOn
: AutoRepeatModeOff
;
405 XChangeKeyboardControl(display_
, KBAutoRepeatMode
, &control
);
408 void InputInjectorX11::Core::InjectScrollWheelClicks(int button
, int count
) {
410 LOG(WARNING
) << "Ignoring unmapped scroll wheel button";
413 for (int i
= 0; i
< count
; i
++) {
414 // Generate a button-down and a button-up to simulate a wheel click.
415 XTestFakeButtonEvent(display_
, button
, true, CurrentTime
);
416 XTestFakeButtonEvent(display_
, button
, false, CurrentTime
);
420 void InputInjectorX11::Core::InjectMouseEvent(const MouseEvent
& event
) {
421 if (!task_runner_
->BelongsToCurrentThread()) {
422 task_runner_
->PostTask(FROM_HERE
,
423 base::Bind(&Core::InjectMouseEvent
, this, event
));
427 if (event
.has_delta_x() &&
428 event
.has_delta_y() &&
429 (event
.delta_x() != 0 || event
.delta_y() != 0)) {
430 latest_mouse_position_
.set(-1, -1);
431 VLOG(3) << "Moving mouse by " << event
.delta_x() << "," << event
.delta_y();
432 XTestFakeRelativeMotionEvent(display_
,
433 event
.delta_x(), event
.delta_y(),
436 } else if (event
.has_x() && event
.has_y()) {
437 // Injecting a motion event immediately before a button release results in
438 // a MotionNotify even if the mouse position hasn't changed, which confuses
439 // apps which assume MotionNotify implies movement. See crbug.com/138075.
440 bool inject_motion
= true;
441 webrtc::DesktopVector
new_mouse_position(event
.x(), event
.y());
442 #if defined(OS_CHROMEOS)
443 // Interim hack to handle display rotation on Chrome OS.
444 // TODO(kelvin): Remove this when Chrome OS has completely migrated to
445 // Ozone (crbug.com/439287).
446 gfx::PointF screen_location
= point_transformer_
.ToScreenCoordinates(
447 gfx::PointF(event
.x(), event
.y()));
448 new_mouse_position
.set(screen_location
.x(), screen_location
.y());
450 if (event
.has_button() && event
.has_button_down() && !event
.button_down()) {
451 if (new_mouse_position
.equals(latest_mouse_position_
))
452 inject_motion
= false;
456 latest_mouse_position_
.set(std::max(0, new_mouse_position
.x()),
457 std::max(0, new_mouse_position
.y()));
459 VLOG(3) << "Moving mouse to " << latest_mouse_position_
.x()
460 << "," << latest_mouse_position_
.y();
461 XTestFakeMotionEvent(display_
, DefaultScreen(display_
),
462 latest_mouse_position_
.x(),
463 latest_mouse_position_
.y(),
468 if (event
.has_button() && event
.has_button_down()) {
469 int button_number
= MouseButtonToX11ButtonNumber(event
.button());
471 if (button_number
< 0) {
472 LOG(WARNING
) << "Ignoring unknown button type: " << event
.button();
476 VLOG(3) << "Button " << event
.button()
477 << " received, sending "
478 << (event
.button_down() ? "down " : "up ")
480 XTestFakeButtonEvent(display_
, button_number
, event
.button_down(),
484 // Older client plugins always send scroll events in pixels, which
485 // must be accumulated host-side. Recent client plugins send both
486 // pixels and ticks with every scroll event, allowing the host to
487 // choose the best model on a per-platform basis. Since we can only
488 // inject ticks on Linux, use them if available.
490 if (event
.has_wheel_ticks_y()) {
491 ticks_y
= event
.wheel_ticks_y();
492 } else if (event
.has_wheel_delta_y()) {
493 wheel_ticks_y_
+= event
.wheel_delta_y() * kWheelTicksPerPixel
;
494 ticks_y
= static_cast<int>(wheel_ticks_y_
);
495 wheel_ticks_y_
-= ticks_y
;
498 InjectScrollWheelClicks(VerticalScrollWheelToX11ButtonNumber(ticks_y
),
503 if (event
.has_wheel_ticks_x()) {
504 ticks_x
= event
.wheel_ticks_x();
505 } else if (event
.has_wheel_delta_x()) {
506 wheel_ticks_x_
+= event
.wheel_delta_x() * kWheelTicksPerPixel
;
507 ticks_x
= static_cast<int>(wheel_ticks_x_
);
508 wheel_ticks_x_
-= ticks_x
;
511 InjectScrollWheelClicks(HorizontalScrollWheelToX11ButtonNumber(ticks_x
),
518 void InputInjectorX11::Core::InitMouseButtonMap() {
519 // TODO(rmsousa): Run this on global/device mapping change events.
521 // Do not touch global pointer mapping, since this may affect the local user.
522 // Instead, try to work around it by reversing the mapping.
523 // Note that if a user has a global mapping that completely disables a button
524 // (by assigning 0 to it), we won't be able to inject it.
525 int num_buttons
= XGetPointerMapping(display_
, nullptr, 0);
526 scoped_ptr
<unsigned char[]> pointer_mapping(new unsigned char[num_buttons
]);
527 num_buttons
= XGetPointerMapping(display_
, pointer_mapping
.get(),
529 for (int i
= 0; i
< kNumPointerButtons
; i
++) {
530 pointer_button_map_
[i
] = -1;
532 for (int i
= 0; i
< num_buttons
; i
++) {
533 // Reverse the mapping.
534 if (pointer_mapping
[i
] > 0 && pointer_mapping
[i
] <= kNumPointerButtons
)
535 pointer_button_map_
[pointer_mapping
[i
] - 1] = i
+ 1;
537 for (int i
= 0; i
< kNumPointerButtons
; i
++) {
538 if (pointer_button_map_
[i
] == -1)
539 LOG(ERROR
) << "Global pointer mapping does not support button " << i
+ 1;
542 int opcode
, event
, error
;
543 if (!XQueryExtension(display_
, "XInputExtension", &opcode
, &event
, &error
)) {
544 // If XInput is not available, we're done. But it would be very unusual to
545 // have a server that supports XTest but not XInput, so log it as an error.
546 LOG(ERROR
) << "X Input extension not available: " << error
;
550 // Make sure the XTEST XInput pointer device mapping is trivial. It should be
551 // safe to reset this mapping, as it won't affect the user's local devices.
552 // In fact, the reason why we do this is because an old gnome-settings-daemon
553 // may have mistakenly applied left-handed preferences to the XTEST device.
555 bool device_found
= false;
557 XDeviceInfo
* devices
;
558 devices
= XListInputDevices(display_
, &num_devices
);
559 for (int i
= 0; i
< num_devices
; i
++) {
560 XDeviceInfo
* device_info
= &devices
[i
];
561 if (device_info
->use
== IsXExtensionPointer
&&
562 strcmp(device_info
->name
, "Virtual core XTEST pointer") == 0) {
563 device_id
= device_info
->id
;
568 XFreeDeviceList(devices
);
571 HOST_LOG
<< "Cannot find XTest device.";
575 XDevice
* device
= XOpenDevice(display_
, device_id
);
577 LOG(ERROR
) << "Cannot open XTest device.";
581 int num_device_buttons
=
582 XGetDeviceButtonMapping(display_
, device
, nullptr, 0);
583 scoped_ptr
<unsigned char[]> button_mapping(new unsigned char[num_buttons
]);
584 for (int i
= 0; i
< num_device_buttons
; i
++) {
585 button_mapping
[i
] = i
+ 1;
587 error
= XSetDeviceButtonMapping(display_
, device
, button_mapping
.get(),
589 if (error
!= Success
)
590 LOG(ERROR
) << "Failed to set XTest device button mapping: " << error
;
592 XCloseDevice(display_
, device
);
595 int InputInjectorX11::Core::MouseButtonToX11ButtonNumber(
596 MouseEvent::MouseButton button
) {
598 case MouseEvent::BUTTON_LEFT
:
599 return pointer_button_map_
[0];
601 case MouseEvent::BUTTON_RIGHT
:
602 return pointer_button_map_
[2];
604 case MouseEvent::BUTTON_MIDDLE
:
605 return pointer_button_map_
[1];
607 case MouseEvent::BUTTON_UNDEFINED
:
613 int InputInjectorX11::Core::HorizontalScrollWheelToX11ButtonNumber(int dx
) {
614 return (dx
> 0 ? pointer_button_map_
[5] : pointer_button_map_
[6]);
617 int InputInjectorX11::Core::VerticalScrollWheelToX11ButtonNumber(int dy
) {
618 // Positive y-values are wheel scroll-up events (button 4), negative y-values
619 // are wheel scroll-down events (button 5).
620 return (dy
> 0 ? pointer_button_map_
[3] : pointer_button_map_
[4]);
623 void InputInjectorX11::Core::Start(
624 scoped_ptr
<protocol::ClipboardStub
> client_clipboard
) {
625 if (!task_runner_
->BelongsToCurrentThread()) {
626 task_runner_
->PostTask(
628 base::Bind(&Core::Start
, this, base::Passed(&client_clipboard
)));
632 InitMouseButtonMap();
634 clipboard_
->Start(client_clipboard
.Pass());
637 void InputInjectorX11::Core::Stop() {
638 if (!task_runner_
->BelongsToCurrentThread()) {
639 task_runner_
->PostTask(FROM_HERE
, base::Bind(&Core::Stop
, this));
649 scoped_ptr
<InputInjector
> InputInjector::Create(
650 scoped_refptr
<base::SingleThreadTaskRunner
> main_task_runner
,
651 scoped_refptr
<base::SingleThreadTaskRunner
> ui_task_runner
) {
652 scoped_ptr
<InputInjectorX11
> injector(
653 new InputInjectorX11(main_task_runner
));
654 if (!injector
->Init())
656 return injector
.Pass();
660 bool InputInjector::SupportsTouchEvents() {
664 } // namespace remoting