remoting/host/input_injector_x11.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "remoting/host/input_injector.h"
   6
   7 #include <X11/extensions/XInput.h>
   8 #include <X11/extensions/XTest.h>
   9 #include <X11/Xlib.h>
  10 #include <X11/XKBlib.h>
  11
  12 #include <set>
  13
  14 #include "base/basictypes.h"
  15 #include "base/bind.h"
  16 #include "base/compiler_specific.h"
  17 #include "base/location.h"
  18 #include "base/single_thread_task_runner.h"
  19 #include "base/strings/utf_string_conversion_utils.h"
  20 #include "remoting/base/logging.h"
  21 #include "remoting/host/clipboard.h"
  22 #include "remoting/host/linux/unicode_to_keysym.h"
  23 #include "remoting/proto/internal.pb.h"
  24 #include "third_party/webrtc/modules/desktop_capture/desktop_geometry.h"
  25 #include "ui/events/keycodes/dom4/keycode_converter.h"
  26
  27 namespace remoting {
  28
  29 namespace {
  30
  31 using protocol::ClipboardEvent;
  32 using protocol::KeyEvent;
  33 using protocol::TextEvent;
  34 using protocol::MouseEvent;
  35
  36 bool FindKeycodeForKeySym(Display* display,
  37                           KeySym key_sym,
  38                           uint32_t* keycode,
  39                           uint32_t* modifiers) {
  40   *keycode = XKeysymToKeycode(display, key_sym);
  41
  42   const uint32_t kModifiersToTry[] = {
  43     0,
  44     ShiftMask,
  45     Mod2Mask,
  46     Mod3Mask,
  47     Mod4Mask,
  48     ShiftMask | Mod2Mask,
  49     ShiftMask | Mod3Mask,
  50     ShiftMask | Mod4Mask,
  51   };
  52
  53   // TODO(sergeyu): Is there a better way to find modifiers state?
  54   for (size_t i = 0; i < arraysize(kModifiersToTry); ++i) {
  55     unsigned long key_sym_with_mods;
  56     if (XkbLookupKeySym(
  57             display, *keycode, kModifiersToTry[i], NULL, &key_sym_with_mods) &&
  58         key_sym_with_mods == key_sym) {
  59       *modifiers = kModifiersToTry[i];
  60       return true;
  61     }
  62   }
  63
  64   return false;
  65 }
  66
  67 // Finds a keycode and set of modifiers that generate character with the
  68 // specified |code_point|.
  69 bool FindKeycodeForUnicode(Display* display,
  70                           uint32_t code_point,
  71                           uint32_t* keycode,
  72                           uint32_t* modifiers) {
  73   std::vector<uint32_t> keysyms;
  74   GetKeySymsForUnicode(code_point, &keysyms);
  75
  76   for (std::vector<uint32_t>::iterator it = keysyms.begin();
  77        it != keysyms.end(); ++it) {
  78     if (FindKeycodeForKeySym(display, *it, keycode, modifiers)) {
  79       return true;
  80     }
  81   }
  82
  83   return false;
  84 }
  85
  86 // Pixel-to-wheel-ticks conversion ratio used by GTK.
  87 // From third_party/WebKit/Source/web/gtk/WebInputEventFactory.cpp .
  88 const float kWheelTicksPerPixel = 3.0f / 160.0f;
  89
  90 // A class to generate events on X11.
  91 class InputInjectorX11 : public InputInjector {
  92  public:
  93   explicit InputInjectorX11(
  94       scoped_refptr<base::SingleThreadTaskRunner> task_runner);
  95   ~InputInjectorX11() override;
  96
  97   bool Init();
  98
  99   // Clipboard stub interface.
 100   void InjectClipboardEvent(const ClipboardEvent& event) override;
 101
 102   // InputStub interface.
 103   void InjectKeyEvent(const KeyEvent& event) override;
 104   void InjectTextEvent(const TextEvent& event) override;
 105   void InjectMouseEvent(const MouseEvent& event) override;
 106
 107   // InputInjector interface.
 108   void Start(scoped_ptr<protocol::ClipboardStub> client_clipboard) override;
 109
 110  private:
 111   // The actual implementation resides in InputInjectorX11::Core class.
 112   class Core : public base::RefCountedThreadSafe<Core> {
 113    public:
 114     explicit Core(scoped_refptr<base::SingleThreadTaskRunner> task_runner);
 115
 116     bool Init();
 117
 118     // Mirrors the ClipboardStub interface.
 119     void InjectClipboardEvent(const ClipboardEvent& event);
 120
 121     // Mirrors the InputStub interface.
 122     void InjectKeyEvent(const KeyEvent& event);
 123     void InjectTextEvent(const TextEvent& event);
 124     void InjectMouseEvent(const MouseEvent& event);
 125
 126     // Mirrors the InputInjector interface.
 127     void Start(scoped_ptr<protocol::ClipboardStub> client_clipboard);
 128
 129     void Stop();
 130
 131    private:
 132     friend class base::RefCountedThreadSafe<Core>;
 133     virtual ~Core();
 134
 135     void InitClipboard();
 136
 137     // Queries whether keyboard auto-repeat is globally enabled. This is used
 138     // to decide whether to temporarily disable then restore this setting. If
 139     // auto-repeat has already been disabled, this class should leave it
 140     // untouched.
 141     bool IsAutoRepeatEnabled();
 142
 143     // Enables or disables keyboard auto-repeat globally.
 144     void SetAutoRepeatEnabled(bool enabled);
 145
 146     void InjectScrollWheelClicks(int button, int count);
 147     // Compensates for global button mappings and resets the XTest device
 148     // mapping.
 149     void InitMouseButtonMap();
 150     int MouseButtonToX11ButtonNumber(MouseEvent::MouseButton button);
 151     int HorizontalScrollWheelToX11ButtonNumber(int dx);
 152     int VerticalScrollWheelToX11ButtonNumber(int dy);
 153
 154     scoped_refptr<base::SingleThreadTaskRunner> task_runner_;
 155
 156     std::set<int> pressed_keys_;
 157     webrtc::DesktopVector latest_mouse_position_;
 158     float wheel_ticks_x_;
 159     float wheel_ticks_y_;
 160
 161     // X11 graphics context.
 162     Display* display_;
 163     Window root_window_;
 164
 165     int test_event_base_;
 166     int test_error_base_;
 167
 168     // Number of buttons we support.
 169     // Left, Right, Middle, VScroll Up/Down, HScroll Left/Right.
 170     static const int kNumPointerButtons = 7;
 171
 172     int pointer_button_map_[kNumPointerButtons];
 173
 174     scoped_ptr<Clipboard> clipboard_;
 175
 176     bool saved_auto_repeat_enabled_;
 177
 178     DISALLOW_COPY_AND_ASSIGN(Core);
 179   };
 180
 181   scoped_refptr<Core> core_;
 182
 183   DISALLOW_COPY_AND_ASSIGN(InputInjectorX11);
 184 };
 185
 186 InputInjectorX11::InputInjectorX11(
 187     scoped_refptr<base::SingleThreadTaskRunner> task_runner) {
 188   core_ = new Core(task_runner);
 189 }
 190
 191 InputInjectorX11::~InputInjectorX11() {
 192   core_->Stop();
 193 }
 194
 195 bool InputInjectorX11::Init() {
 196   return core_->Init();
 197 }
 198
 199 void InputInjectorX11::InjectClipboardEvent(const ClipboardEvent& event) {
 200   core_->InjectClipboardEvent(event);
 201 }
 202
 203 void InputInjectorX11::InjectKeyEvent(const KeyEvent& event) {
 204   core_->InjectKeyEvent(event);
 205 }
 206
 207 void InputInjectorX11::InjectTextEvent(const TextEvent& event) {
 208   core_->InjectTextEvent(event);
 209 }
 210
 211 void InputInjectorX11::InjectMouseEvent(const MouseEvent& event) {
 212   core_->InjectMouseEvent(event);
 213 }
 214
 215 void InputInjectorX11::Start(
 216     scoped_ptr<protocol::ClipboardStub> client_clipboard) {
 217   core_->Start(client_clipboard.Pass());
 218 }
 219
 220 InputInjectorX11::Core::Core(
 221     scoped_refptr<base::SingleThreadTaskRunner> task_runner)
 222     : task_runner_(task_runner),
 223       latest_mouse_position_(-1, -1),
 224       wheel_ticks_x_(0.0f),
 225       wheel_ticks_y_(0.0f),
 226       display_(XOpenDisplay(NULL)),
 227       root_window_(BadValue),
 228       saved_auto_repeat_enabled_(false) {
 229 }
 230
 231 bool InputInjectorX11::Core::Init() {
 232   CHECK(display_);
 233
 234   if (!task_runner_->BelongsToCurrentThread())
 235     task_runner_->PostTask(FROM_HERE, base::Bind(&Core::InitClipboard, this));
 236
 237   root_window_ = RootWindow(display_, DefaultScreen(display_));
 238   if (root_window_ == BadValue) {
 239     LOG(ERROR) << "Unable to get the root window";
 240     return false;
 241   }
 242
 243   // TODO(ajwong): Do we want to check the major/minor version at all for XTest?
 244   int major = 0;
 245   int minor = 0;
 246   if (!XTestQueryExtension(display_, &test_event_base_, &test_error_base_,
 247                            &major, &minor)) {
 248     LOG(ERROR) << "Server does not support XTest.";
 249     return false;
 250   }
 251   InitMouseButtonMap();
 252   return true;
 253 }
 254
 255 void InputInjectorX11::Core::InjectClipboardEvent(
 256     const ClipboardEvent& event) {
 257   if (!task_runner_->BelongsToCurrentThread()) {
 258     task_runner_->PostTask(
 259         FROM_HERE, base::Bind(&Core::InjectClipboardEvent, this, event));
 260     return;
 261   }
 262
 263   // |clipboard_| will ignore unknown MIME-types, and verify the data's format.
 264   clipboard_->InjectClipboardEvent(event);
 265 }
 266
 267 void InputInjectorX11::Core::InjectKeyEvent(const KeyEvent& event) {
 268   // HostEventDispatcher should filter events missing the pressed field.
 269   if (!event.has_pressed() || !event.has_usb_keycode())
 270     return;
 271
 272   if (!task_runner_->BelongsToCurrentThread()) {
 273     task_runner_->PostTask(FROM_HERE,
 274                            base::Bind(&Core::InjectKeyEvent, this, event));
 275     return;
 276   }
 277
 278   int keycode =
 279       ui::KeycodeConverter::UsbKeycodeToNativeKeycode(event.usb_keycode());
 280
 281   VLOG(3) << "Converting USB keycode: " << std::hex << event.usb_keycode()
 282           << " to keycode: " << keycode << std::dec;
 283
 284   // Ignore events which can't be mapped.
 285   if (keycode == ui::KeycodeConverter::InvalidNativeKeycode())
 286     return;
 287
 288   if (event.pressed()) {
 289     if (pressed_keys_.find(keycode) != pressed_keys_.end()) {
 290       // Key is already held down, so lift the key up to ensure this repeated
 291       // press takes effect.
 292       XTestFakeKeyEvent(display_, keycode, False, CurrentTime);
 293     }
 294
 295     if (pressed_keys_.empty()) {
 296       // Disable auto-repeat, if necessary, to avoid triggering auto-repeat
 297       // if network congestion delays the key-up event from the client.
 298       saved_auto_repeat_enabled_ = IsAutoRepeatEnabled();
 299       if (saved_auto_repeat_enabled_)
 300         SetAutoRepeatEnabled(false);
 301     }
 302     pressed_keys_.insert(keycode);
 303   } else {
 304     pressed_keys_.erase(keycode);
 305     if (pressed_keys_.empty()) {
 306       // Re-enable auto-repeat, if necessary, when all keys are released.
 307       if (saved_auto_repeat_enabled_)
 308         SetAutoRepeatEnabled(true);
 309     }
 310   }
 311
 312   XTestFakeKeyEvent(display_, keycode, event.pressed(), CurrentTime);
 313   XFlush(display_);
 314 }
 315
 316 void InputInjectorX11::Core::InjectTextEvent(const TextEvent& event) {
 317   if (!task_runner_->BelongsToCurrentThread()) {
 318     task_runner_->PostTask(FROM_HERE,
 319                            base::Bind(&Core::InjectTextEvent, this, event));
 320     return;
 321   }
 322
 323   const std::string text = event.text();
 324   for (int32 index = 0; index < static_cast<int32>(text.size()); ++index) {
 325     uint32_t code_point;
 326     if (!base::ReadUnicodeCharacter(
 327             text.c_str(), text.size(), &index, &code_point)) {
 328       continue;
 329     }
 330
 331     uint32_t keycode;
 332     uint32_t modifiers;
 333     if (!FindKeycodeForUnicode(display_, code_point, &keycode, &modifiers))
 334       continue;
 335
 336     XkbLockModifiers(display_, XkbUseCoreKbd,  modifiers, modifiers);
 337
 338     XTestFakeKeyEvent(display_, keycode, True, CurrentTime);
 339     XTestFakeKeyEvent(display_, keycode, False, CurrentTime);
 340
 341     XkbLockModifiers(display_, XkbUseCoreKbd, modifiers, 0);
 342   }
 343
 344   XFlush(display_);
 345 }
 346
 347 InputInjectorX11::Core::~Core() {
 348   CHECK(pressed_keys_.empty());
 349 }
 350
 351 void InputInjectorX11::Core::InitClipboard() {
 352   DCHECK(task_runner_->BelongsToCurrentThread());
 353   clipboard_ = Clipboard::Create();
 354 }
 355
 356 bool InputInjectorX11::Core::IsAutoRepeatEnabled() {
 357   XKeyboardState state;
 358   if (!XGetKeyboardControl(display_, &state)) {
 359     LOG(ERROR) << "Failed to get keyboard auto-repeat status, assuming ON.";
 360     return true;
 361   }
 362   return state.global_auto_repeat == AutoRepeatModeOn;
 363 }
 364
 365 void InputInjectorX11::Core::SetAutoRepeatEnabled(bool mode) {
 366   XKeyboardControl control;
 367   control.auto_repeat_mode = mode ? AutoRepeatModeOn : AutoRepeatModeOff;
 368   XChangeKeyboardControl(display_, KBAutoRepeatMode, &control);
 369 }
 370
 371 void InputInjectorX11::Core::InjectScrollWheelClicks(int button, int count) {
 372   if (button < 0) {
 373     LOG(WARNING) << "Ignoring unmapped scroll wheel button";
 374     return;
 375   }
 376   for (int i = 0; i < count; i++) {
 377     // Generate a button-down and a button-up to simulate a wheel click.
 378     XTestFakeButtonEvent(display_, button, true, CurrentTime);
 379     XTestFakeButtonEvent(display_, button, false, CurrentTime);
 380   }
 381 }
 382
 383 void InputInjectorX11::Core::InjectMouseEvent(const MouseEvent& event) {
 384   if (!task_runner_->BelongsToCurrentThread()) {
 385     task_runner_->PostTask(FROM_HERE,
 386                            base::Bind(&Core::InjectMouseEvent, this, event));
 387     return;
 388   }
 389
 390   if (event.has_delta_x() &&
 391       event.has_delta_y() &&
 392       (event.delta_x() != 0 || event.delta_y() != 0)) {
 393     latest_mouse_position_.set(-1, -1);
 394     VLOG(3) << "Moving mouse by " << event.delta_x() << "," << event.delta_y();
 395     XTestFakeRelativeMotionEvent(display_,
 396                                  event.delta_x(), event.delta_y(),
 397                                  CurrentTime);
 398
 399   } else if (event.has_x() && event.has_y()) {
 400     // Injecting a motion event immediately before a button release results in
 401     // a MotionNotify even if the mouse position hasn't changed, which confuses
 402     // apps which assume MotionNotify implies movement. See crbug.com/138075.
 403     bool inject_motion = true;
 404     webrtc::DesktopVector new_mouse_position(
 405         webrtc::DesktopVector(event.x(), event.y()));
 406     if (event.has_button() && event.has_button_down() && !event.button_down()) {
 407       if (new_mouse_position.equals(latest_mouse_position_))
 408         inject_motion = false;
 409     }
 410
 411     if (inject_motion) {
 412       latest_mouse_position_.set(std::max(0, new_mouse_position.x()),
 413                                  std::max(0, new_mouse_position.y()));
 414
 415       VLOG(3) << "Moving mouse to " << latest_mouse_position_.x()
 416               << "," << latest_mouse_position_.y();
 417       XTestFakeMotionEvent(display_, DefaultScreen(display_),
 418                            latest_mouse_position_.x(),
 419                            latest_mouse_position_.y(),
 420                            CurrentTime);
 421     }
 422   }
 423
 424   if (event.has_button() && event.has_button_down()) {
 425     int button_number = MouseButtonToX11ButtonNumber(event.button());
 426
 427     if (button_number < 0) {
 428       LOG(WARNING) << "Ignoring unknown button type: " << event.button();
 429       return;
 430     }
 431
 432     VLOG(3) << "Button " << event.button()
 433             << " received, sending "
 434             << (event.button_down() ? "down " : "up ")
 435             << button_number;
 436     XTestFakeButtonEvent(display_, button_number, event.button_down(),
 437                          CurrentTime);
 438   }
 439
 440   // Older client plugins always send scroll events in pixels, which
 441   // must be accumulated host-side. Recent client plugins send both
 442   // pixels and ticks with every scroll event, allowing the host to
 443   // choose the best model on a per-platform basis. Since we can only
 444   // inject ticks on Linux, use them if available.
 445   int ticks_y = 0;
 446   if (event.has_wheel_ticks_y()) {
 447     ticks_y = event.wheel_ticks_y();
 448   } else if (event.has_wheel_delta_y()) {
 449     wheel_ticks_y_ += event.wheel_delta_y() * kWheelTicksPerPixel;
 450     ticks_y = static_cast<int>(wheel_ticks_y_);
 451     wheel_ticks_y_ -= ticks_y;
 452   }
 453   if (ticks_y != 0) {
 454     InjectScrollWheelClicks(VerticalScrollWheelToX11ButtonNumber(ticks_y),
 455                             abs(ticks_y));
 456   }
 457
 458   int ticks_x = 0;
 459   if (event.has_wheel_ticks_x()) {
 460     ticks_x = event.wheel_ticks_x();
 461   } else if (event.has_wheel_delta_x()) {
 462     wheel_ticks_x_ += event.wheel_delta_x() * kWheelTicksPerPixel;
 463     ticks_x = static_cast<int>(wheel_ticks_x_);
 464     wheel_ticks_x_ -= ticks_x;
 465   }
 466   if (ticks_x != 0) {
 467     InjectScrollWheelClicks(HorizontalScrollWheelToX11ButtonNumber(ticks_x),
 468                             abs(ticks_x));
 469   }
 470
 471   XFlush(display_);
 472 }
 473
 474 void InputInjectorX11::Core::InitMouseButtonMap() {
 475   // TODO(rmsousa): Run this on global/device mapping change events.
 476
 477   // Do not touch global pointer mapping, since this may affect the local user.
 478   // Instead, try to work around it by reversing the mapping.
 479   // Note that if a user has a global mapping that completely disables a button
 480   // (by assigning 0 to it), we won't be able to inject it.
 481   int num_buttons = XGetPointerMapping(display_, NULL, 0);
 482   scoped_ptr<unsigned char[]> pointer_mapping(new unsigned char[num_buttons]);
 483   num_buttons = XGetPointerMapping(display_, pointer_mapping.get(),
 484                                    num_buttons);
 485   for (int i = 0; i < kNumPointerButtons; i++) {
 486     pointer_button_map_[i] = -1;
 487   }
 488   for (int i = 0; i < num_buttons; i++) {
 489     // Reverse the mapping.
 490     if (pointer_mapping[i] > 0 && pointer_mapping[i] <= kNumPointerButtons)
 491       pointer_button_map_[pointer_mapping[i] - 1] = i + 1;
 492   }
 493   for (int i = 0; i < kNumPointerButtons; i++) {
 494     if (pointer_button_map_[i] == -1)
 495       LOG(ERROR) << "Global pointer mapping does not support button " << i + 1;
 496   }
 497
 498   int opcode, event, error;
 499   if (!XQueryExtension(display_, "XInputExtension", &opcode, &event, &error)) {
 500     // If XInput is not available, we're done. But it would be very unusual to
 501     // have a server that supports XTest but not XInput, so log it as an error.
 502     LOG(ERROR) << "X Input extension not available: " << error;
 503     return;
 504   }
 505
 506   // Make sure the XTEST XInput pointer device mapping is trivial. It should be
 507   // safe to reset this mapping, as it won't affect the user's local devices.
 508   // In fact, the reason why we do this is because an old gnome-settings-daemon
 509   // may have mistakenly applied left-handed preferences to the XTEST device.
 510   XID device_id = 0;
 511   bool device_found = false;
 512   int num_devices;
 513   XDeviceInfo* devices;
 514   devices = XListInputDevices(display_, &num_devices);
 515   for (int i = 0; i < num_devices; i++) {
 516     XDeviceInfo* device_info = &devices[i];
 517     if (device_info->use == IsXExtensionPointer &&
 518         strcmp(device_info->name, "Virtual core XTEST pointer") == 0) {
 519       device_id = device_info->id;
 520       device_found = true;
 521       break;
 522     }
 523   }
 524   XFreeDeviceList(devices);
 525
 526   if (!device_found) {
 527     HOST_LOG << "Cannot find XTest device.";
 528     return;
 529   }
 530
 531   XDevice* device = XOpenDevice(display_, device_id);
 532   if (!device) {
 533     LOG(ERROR) << "Cannot open XTest device.";
 534     return;
 535   }
 536
 537   int num_device_buttons = XGetDeviceButtonMapping(display_, device, NULL, 0);
 538   scoped_ptr<unsigned char[]> button_mapping(new unsigned char[num_buttons]);
 539   for (int i = 0; i < num_device_buttons; i++) {
 540     button_mapping[i] = i + 1;
 541   }
 542   error = XSetDeviceButtonMapping(display_, device, button_mapping.get(),
 543                                   num_device_buttons);
 544   if (error != Success)
 545     LOG(ERROR) << "Failed to set XTest device button mapping: " << error;
 546
 547   XCloseDevice(display_, device);
 548 }
 549
 550 int InputInjectorX11::Core::MouseButtonToX11ButtonNumber(
 551     MouseEvent::MouseButton button) {
 552   switch (button) {
 553     case MouseEvent::BUTTON_LEFT:
 554       return pointer_button_map_[0];
 555
 556     case MouseEvent::BUTTON_RIGHT:
 557       return pointer_button_map_[2];
 558
 559     case MouseEvent::BUTTON_MIDDLE:
 560       return pointer_button_map_[1];
 561
 562     case MouseEvent::BUTTON_UNDEFINED:
 563     default:
 564       return -1;
 565   }
 566 }
 567
 568 int InputInjectorX11::Core::HorizontalScrollWheelToX11ButtonNumber(int dx) {
 569   return (dx > 0 ? pointer_button_map_[5] : pointer_button_map_[6]);
 570 }
 571
 572 int InputInjectorX11::Core::VerticalScrollWheelToX11ButtonNumber(int dy) {
 573   // Positive y-values are wheel scroll-up events (button 4), negative y-values
 574   // are wheel scroll-down events (button 5).
 575   return (dy > 0 ? pointer_button_map_[3] : pointer_button_map_[4]);
 576 }
 577
 578 void InputInjectorX11::Core::Start(
 579     scoped_ptr<protocol::ClipboardStub> client_clipboard) {
 580   if (!task_runner_->BelongsToCurrentThread()) {
 581     task_runner_->PostTask(
 582         FROM_HERE,
 583         base::Bind(&Core::Start, this, base::Passed(&client_clipboard)));
 584     return;
 585   }
 586
 587   InitMouseButtonMap();
 588
 589   clipboard_->Start(client_clipboard.Pass());
 590 }
 591
 592 void InputInjectorX11::Core::Stop() {
 593   if (!task_runner_->BelongsToCurrentThread()) {
 594     task_runner_->PostTask(FROM_HERE, base::Bind(&Core::Stop, this));
 595     return;
 596   }
 597
 598   clipboard_->Stop();
 599 }
 600
 601 }  // namespace
 602
 603 scoped_ptr<InputInjector> InputInjector::Create(
 604     scoped_refptr<base::SingleThreadTaskRunner> main_task_runner,
 605     scoped_refptr<base::SingleThreadTaskRunner> ui_task_runner) {
 606   scoped_ptr<InputInjectorX11> injector(
 607       new InputInjectorX11(main_task_runner));
 608   if (!injector->Init())
 609     return nullptr;
 610   return injector.Pass();
 611 }
 612
 613 }  // namespace remoting