win8/test/ui_automation_client.cc

   1 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "win8/test/ui_automation_client.h"
   6
   7 #include <atlbase.h>
   8 #include <atlcom.h>
   9 #include <oleauto.h>
  10 #include <uiautomation.h>
  11
  12 #include <algorithm>
  13
  14 #include "base/bind.h"
  15 #include "base/callback.h"
  16 #include "base/memory/ref_counted.h"
  17 #include "base/strings/string_number_conversions.h"
  18 #include "base/strings/string_util.h"
  19 #include "base/thread_task_runner_handle.h"
  20 #include "base/win/scoped_comptr.h"
  21 #include "base/win/scoped_variant.h"
  22
  23 namespace win8 {
  24 namespace internal {
  25
  26 // The guts of the UI automation client which runs on a dedicated thread in the
  27 // multi-threaded COM apartment. An instance may be constructed on any thread,
  28 // but Initialize() must be invoked on a thread in the MTA.
  29 class UIAutomationClient::Context {
  30  public:
  31   // Returns a new instance ready for initialization and use on another thread.
  32   static base::WeakPtr<Context> Create();
  33
  34   // Deletes the instance.
  35   void DeleteOnAutomationThread();
  36
  37   // Initializes the context, invoking |init_callback| via |client_runner| when
  38   // done. On success, |result_callback| will eventually be called after the
  39   // window has been processed. On failure, this instance self-destructs after
  40   // posting |init_callback|.
  41   void Initialize(
  42       scoped_refptr<base::SingleThreadTaskRunner> client_runner,
  43       base::string16 class_name,
  44       base::string16 item_name,
  45       UIAutomationClient::InitializedCallback init_callback,
  46       UIAutomationClient::ResultCallback result_callback);
  47
  48   // Methods invoked by event handlers via weak pointers.
  49   void HandleAutomationEvent(
  50       base::win::ScopedComPtr<IUIAutomationElement> sender,
  51       EVENTID eventId);
  52
  53  private:
  54   class EventHandler;
  55
  56   // The only and only method that may be called from outside of the automation
  57   // thread.
  58   Context();
  59   ~Context();
  60
  61   HRESULT InstallWindowObserver();
  62   HRESULT RemoveWindowObserver();
  63
  64   void HandleWindowOpen(
  65       const base::win::ScopedComPtr<IUIAutomationElement>& window);
  66   void ProcessWindow(
  67       const base::win::ScopedComPtr<IUIAutomationElement>& window);
  68   HRESULT InvokeDesiredItem(
  69       const base::win::ScopedComPtr<IUIAutomationElement>& element);
  70   HRESULT GetInvokableItems(
  71       const base::win::ScopedComPtr<IUIAutomationElement>& element,
  72       std::vector<base::string16>* choices);
  73   void CloseWindow(const base::win::ScopedComPtr<IUIAutomationElement>& window);
  74
  75   base::ThreadChecker thread_checker_;
  76
  77   // The loop on which the client itself lives.
  78   scoped_refptr<base::SingleThreadTaskRunner> client_runner_;
  79
  80   // The class name of the window for which the client waits.
  81   base::string16 class_name_;
  82
  83   // The name of the item to invoke.
  84   base::string16 item_name_;
  85
  86   // The consumer's result callback.
  87   ResultCallback result_callback_;
  88
  89   // The automation client.
  90   base::win::ScopedComPtr<IUIAutomation> automation_;
  91
  92   // A handler of Window open events.
  93   base::win::ScopedComPtr<IUIAutomationEventHandler> event_handler_;
  94
  95   // Weak pointers to the context are given to event handlers.
  96   base::WeakPtrFactory<UIAutomationClient::Context> weak_ptr_factory_;
  97
  98   DISALLOW_COPY_AND_ASSIGN(Context);
  99 };
 100
 101 class UIAutomationClient::Context::EventHandler
 102     : public CComObjectRootEx<CComMultiThreadModel>,
 103       public IUIAutomationEventHandler {
 104  public:
 105   BEGIN_COM_MAP(UIAutomationClient::Context::EventHandler)
 106     COM_INTERFACE_ENTRY(IUIAutomationEventHandler)
 107   END_COM_MAP()
 108
 109   EventHandler();
 110   virtual ~EventHandler();
 111
 112   // Initializes the object with its parent UI automation client context's
 113   // message loop and pointer. Events are dispatched back to the context on
 114   // the given loop.
 115   void Initialize(
 116       const scoped_refptr<base::SingleThreadTaskRunner>& context_runner,
 117       const base::WeakPtr<UIAutomationClient::Context>& context);
 118
 119   // IUIAutomationEventHandler methods.
 120   STDMETHOD(HandleAutomationEvent)(IUIAutomationElement* sender,
 121                                    EVENTID eventId);
 122
 123  private:
 124   // The task runner for the UI automation client context.
 125   scoped_refptr<base::SingleThreadTaskRunner> context_runner_;
 126
 127   // The parent UI automation client context.
 128   base::WeakPtr<UIAutomationClient::Context> context_;
 129
 130   DISALLOW_COPY_AND_ASSIGN(EventHandler);
 131 };
 132
 133 UIAutomationClient::Context::EventHandler::EventHandler() {}
 134
 135 UIAutomationClient::Context::EventHandler::~EventHandler() {}
 136
 137 void UIAutomationClient::Context::EventHandler::Initialize(
 138     const scoped_refptr<base::SingleThreadTaskRunner>& context_runner,
 139     const base::WeakPtr<UIAutomationClient::Context>& context) {
 140   context_runner_ = context_runner;
 141   context_ = context;
 142 }
 143
 144 HRESULT UIAutomationClient::Context::EventHandler::HandleAutomationEvent(
 145     IUIAutomationElement* sender,
 146     EVENTID eventId) {
 147   // Event handlers are invoked on an arbitrary thread in the MTA. Send the
 148   // event back to the main UI automation thread for processing.
 149   context_runner_->PostTask(
 150       FROM_HERE,
 151       base::Bind(&UIAutomationClient::Context::HandleAutomationEvent, context_,
 152                  base::win::ScopedComPtr<IUIAutomationElement>(sender),
 153                  eventId));
 154
 155   return S_OK;
 156 }
 157
 158 base::WeakPtr<UIAutomationClient::Context>
 159     UIAutomationClient::Context::Create() {
 160   Context* context = new Context();
 161   return context->weak_ptr_factory_.GetWeakPtr();
 162 }
 163
 164 void UIAutomationClient::Context::DeleteOnAutomationThread() {
 165   DCHECK(thread_checker_.CalledOnValidThread());
 166   delete this;
 167 }
 168
 169 UIAutomationClient::Context::Context() : weak_ptr_factory_(this) {}
 170
 171 UIAutomationClient::Context::~Context() {
 172   DCHECK(thread_checker_.CalledOnValidThread());
 173
 174   if (event_handler_.get()) {
 175     event_handler_ = NULL;
 176     HRESULT result = automation_->RemoveAllEventHandlers();
 177     LOG_IF(ERROR, FAILED(result)) << std::hex << result;
 178   }
 179 }
 180
 181 void UIAutomationClient::Context::Initialize(
 182     scoped_refptr<base::SingleThreadTaskRunner> client_runner,
 183     base::string16 class_name,
 184     base::string16 item_name,
 185     UIAutomationClient::InitializedCallback init_callback,
 186     UIAutomationClient::ResultCallback result_callback) {
 187   // This and all other methods must be called on the automation thread.
 188   DCHECK(!client_runner->BelongsToCurrentThread());
 189   // Bind the checker to this thread.
 190   thread_checker_.DetachFromThread();
 191   DCHECK(thread_checker_.CalledOnValidThread());
 192
 193   client_runner_ = client_runner;
 194   class_name_ = class_name;
 195   item_name_ = item_name;
 196   result_callback_ = result_callback;
 197
 198   HRESULT result = automation_.CreateInstance(CLSID_CUIAutomation, NULL,
 199                                               CLSCTX_INPROC_SERVER);
 200   if (FAILED(result) || !automation_.get())
 201     LOG(ERROR) << std::hex << result;
 202   else
 203     result = InstallWindowObserver();
 204
 205   // Tell the client that initialization is complete.
 206   client_runner_->PostTask(FROM_HERE, base::Bind(init_callback, result));
 207
 208   // Self-destruct if the overall operation failed.
 209   if (FAILED(result))
 210     delete this;
 211 }
 212
 213 // Installs the window observer.
 214 HRESULT UIAutomationClient::Context::InstallWindowObserver() {
 215   DCHECK(thread_checker_.CalledOnValidThread());
 216   DCHECK(automation_.get());
 217   DCHECK(!event_handler_.get());
 218
 219   HRESULT result = S_OK;
 220   base::win::ScopedComPtr<IUIAutomationElement> root_element;
 221   base::win::ScopedComPtr<IUIAutomationCacheRequest> cache_request;
 222
 223   // Observe the opening of all windows.
 224   result = automation_->GetRootElement(root_element.Receive());
 225   if (FAILED(result)) {
 226     LOG(ERROR) << std::hex << result;
 227     return result;
 228   }
 229
 230   // Cache Window class, HWND, and window pattern for opened windows.
 231   result = automation_->CreateCacheRequest(cache_request.Receive());
 232   if (FAILED(result)) {
 233     LOG(ERROR) << std::hex << result;
 234     return result;
 235   }
 236   cache_request->AddProperty(UIA_ClassNamePropertyId);
 237   cache_request->AddProperty(UIA_NativeWindowHandlePropertyId);
 238
 239   // Create the observer.
 240   CComObject<EventHandler>* event_handler_obj = NULL;
 241   result = CComObject<EventHandler>::CreateInstance(&event_handler_obj);
 242   if (FAILED(result)) {
 243     LOG(ERROR) << std::hex << result;
 244     return result;
 245   }
 246   event_handler_obj->Initialize(base::ThreadTaskRunnerHandle::Get(),
 247                                 weak_ptr_factory_.GetWeakPtr());
 248   base::win::ScopedComPtr<IUIAutomationEventHandler> event_handler(
 249       event_handler_obj);
 250
 251   result = automation_->AddAutomationEventHandler(
 252       UIA_Window_WindowOpenedEventId, root_element.get(), TreeScope_Descendants,
 253       cache_request.get(), event_handler.get());
 254
 255   if (FAILED(result)) {
 256     LOG(ERROR) << std::hex << result;
 257     return result;
 258   }
 259
 260   event_handler_ = event_handler;
 261   return S_OK;
 262 }
 263
 264 // Removes this instance's window observer.
 265 HRESULT UIAutomationClient::Context::RemoveWindowObserver() {
 266   DCHECK(thread_checker_.CalledOnValidThread());
 267   DCHECK(automation_.get());
 268   DCHECK(event_handler_.get());
 269
 270   HRESULT result = S_OK;
 271   base::win::ScopedComPtr<IUIAutomationElement> root_element;
 272
 273   // The opening of all windows are observed.
 274   result = automation_->GetRootElement(root_element.Receive());
 275   if (FAILED(result)) {
 276     LOG(ERROR) << std::hex << result;
 277     return result;
 278   }
 279
 280   result = automation_->RemoveAutomationEventHandler(
 281       UIA_Window_WindowOpenedEventId, root_element.get(), event_handler_.get());
 282   if (FAILED(result)) {
 283     LOG(ERROR) << std::hex << result;
 284     return result;
 285   }
 286
 287   event_handler_ = NULL;
 288   return S_OK;
 289 }
 290
 291 // Handles an automation event. If the event results in the processing for which
 292 // this context was created, the context self-destructs after posting the
 293 // results to the client.
 294 void UIAutomationClient::Context::HandleAutomationEvent(
 295     base::win::ScopedComPtr<IUIAutomationElement> sender,
 296     EVENTID eventId) {
 297   DCHECK(thread_checker_.CalledOnValidThread());
 298   if (eventId == UIA_Window_WindowOpenedEventId)
 299     HandleWindowOpen(sender);
 300 }
 301
 302 // Handles a WindowOpen event. If |window| is the one for which this instance is
 303 // waiting, it is processed and this instance self-destructs after posting the
 304 // results to the client.
 305 void UIAutomationClient::Context::HandleWindowOpen(
 306     const base::win::ScopedComPtr<IUIAutomationElement>& window) {
 307   DCHECK(thread_checker_.CalledOnValidThread());
 308   HRESULT hr = S_OK;
 309   base::win::ScopedVariant var;
 310
 311   hr = window->GetCachedPropertyValueEx(UIA_ClassNamePropertyId, TRUE,
 312                                         var.Receive());
 313   if (FAILED(hr)) {
 314     LOG(ERROR) << std::hex << hr;
 315     return;
 316   }
 317
 318   if (V_VT(var.ptr()) != VT_BSTR) {
 319     LOG(ERROR) << __FUNCTION__
 320                << " class name is not a BSTR: " << V_VT(var.ptr());
 321     return;
 322   }
 323
 324   base::string16 class_name(V_BSTR(var.ptr()));
 325
 326   // Window class names are atoms, which are case-insensitive.
 327   if (class_name.size() == class_name_.size() &&
 328       std::equal(class_name.begin(), class_name.end(), class_name_.begin(),
 329                  base::CaseInsensitiveCompare<wchar_t>())) {
 330     RemoveWindowObserver();
 331     ProcessWindow(window);
 332   }
 333 }
 334
 335 // Processes |window| by invoking the desired child item. If the item cannot be
 336 // found or invoked, an attempt is made to get a list of all invokable children.
 337 // The results are posted back to the client on |client_runner_|, and this
 338 // instance self-destructs.
 339 void UIAutomationClient::Context::ProcessWindow(
 340     const base::win::ScopedComPtr<IUIAutomationElement>& window) {
 341   DCHECK(thread_checker_.CalledOnValidThread());
 342
 343   HRESULT result = S_OK;
 344   std::vector<base::string16> choices;
 345   result = InvokeDesiredItem(window);
 346   if (FAILED(result)) {
 347     GetInvokableItems(window, &choices);
 348     CloseWindow(window);
 349   }
 350
 351   client_runner_->PostTask(FROM_HERE,
 352                            base::Bind(result_callback_, result, choices));
 353
 354   // Self-destruct since there's nothing more to be done here.
 355   delete this;
 356 }
 357
 358 // Invokes the desired child of |element|.
 359 HRESULT UIAutomationClient::Context::InvokeDesiredItem(
 360     const base::win::ScopedComPtr<IUIAutomationElement>& element) {
 361   DCHECK(thread_checker_.CalledOnValidThread());
 362
 363   HRESULT result = S_OK;
 364   base::win::ScopedVariant var;
 365   base::win::ScopedComPtr<IUIAutomationCondition> invokable_condition;
 366   base::win::ScopedComPtr<IUIAutomationCondition> item_name_condition;
 367   base::win::ScopedComPtr<IUIAutomationCondition> control_view_condition;
 368   base::win::ScopedComPtr<IUIAutomationCondition> condition;
 369   base::win::ScopedComPtr<IUIAutomationCacheRequest> cache_request;
 370   base::win::ScopedComPtr<IUIAutomationElement> target;
 371
 372   // Search for an invokable element named item_name.
 373   var.Set(true);
 374   result = automation_->CreatePropertyCondition(
 375       UIA_IsInvokePatternAvailablePropertyId,
 376       var,
 377       invokable_condition.Receive());
 378   var.Reset();
 379   if (FAILED(result)) {
 380     LOG(ERROR) << std::hex << result;
 381     return result;
 382   }
 383
 384   var.Set(item_name_.c_str());
 385   result = automation_->CreatePropertyCondition(UIA_NamePropertyId,
 386                                                 var,
 387                                                 item_name_condition.Receive());
 388   var.Reset();
 389   if (FAILED(result)) {
 390     LOG(ERROR) << std::hex << result;
 391     return result;
 392   }
 393
 394   result = automation_->get_ControlViewCondition(
 395       control_view_condition.Receive());
 396   if (FAILED(result)) {
 397     LOG(ERROR) << std::hex << result;
 398     return result;
 399   }
 400
 401   std::vector<IUIAutomationCondition*> conditions;
 402   conditions.push_back(invokable_condition.get());
 403   conditions.push_back(item_name_condition.get());
 404   conditions.push_back(control_view_condition.get());
 405   result = automation_->CreateAndConditionFromNativeArray(
 406       &conditions[0], conditions.size(), condition.Receive());
 407   if (FAILED(result)) {
 408     LOG(ERROR) << std::hex << result;
 409     return result;
 410   }
 411
 412   // Cache invokable pattern for the item.
 413   result = automation_->CreateCacheRequest(cache_request.Receive());
 414   if (FAILED(result)) {
 415     LOG(ERROR) << std::hex << result;
 416     return result;
 417   }
 418   cache_request->AddPattern(UIA_InvokePatternId);
 419
 420   result = element->FindFirstBuildCache(
 421       static_cast<TreeScope>(TreeScope_Children | TreeScope_Descendants),
 422       condition.get(), cache_request.get(), target.Receive());
 423   if (FAILED(result)) {
 424     LOG(ERROR) << std::hex << result;
 425     return result;
 426   }
 427
 428   // If the item was found, invoke it.
 429   if (!target.get()) {
 430     LOG(WARNING) << "Failed to find desired item to invoke.";
 431     return E_FAIL;
 432   }
 433
 434   base::win::ScopedComPtr<IUIAutomationInvokePattern> invoker;
 435   result = target->GetCachedPatternAs(UIA_InvokePatternId, invoker.iid(),
 436                                       invoker.ReceiveVoid());
 437   if (FAILED(result)) {
 438     LOG(ERROR) << std::hex << result;
 439     return result;
 440   }
 441
 442   result = invoker->Invoke();
 443   if (FAILED(result)) {
 444     LOG(ERROR) << std::hex << result;
 445     return result;
 446   }
 447
 448   return S_OK;
 449 }
 450
 451 // Populates |choices| with the names of all invokable children of |element|.
 452 HRESULT UIAutomationClient::Context::GetInvokableItems(
 453     const base::win::ScopedComPtr<IUIAutomationElement>& element,
 454     std::vector<base::string16>* choices) {
 455   DCHECK(choices);
 456   DCHECK(thread_checker_.CalledOnValidThread());
 457
 458   HRESULT result = S_OK;
 459   base::win::ScopedVariant var;
 460   base::win::ScopedComPtr<IUIAutomationCondition> invokable_condition;
 461   base::win::ScopedComPtr<IUIAutomationCondition> control_view_condition;
 462   base::win::ScopedComPtr<IUIAutomationCondition> condition;
 463   base::win::ScopedComPtr<IUIAutomationCacheRequest> cache_request;
 464   base::win::ScopedComPtr<IUIAutomationElementArray> element_array;
 465   base::win::ScopedComPtr<IUIAutomationElement> child_element;
 466
 467   // Search for all invokable elements.
 468   var.Set(true);
 469   result = automation_->CreatePropertyCondition(
 470       UIA_IsInvokePatternAvailablePropertyId,
 471       var,
 472       invokable_condition.Receive());
 473   var.Reset();
 474   if (FAILED(result)) {
 475     LOG(ERROR) << std::hex << result;
 476     return result;
 477   }
 478
 479   result = automation_->get_ControlViewCondition(
 480       control_view_condition.Receive());
 481   if (FAILED(result)) {
 482     LOG(ERROR) << std::hex << result;
 483     return result;
 484   }
 485
 486   result = automation_->CreateAndCondition(invokable_condition.get(),
 487                                            control_view_condition.get(),
 488                                            condition.Receive());
 489   if (FAILED(result)) {
 490     LOG(ERROR) << std::hex << result;
 491     return result;
 492   }
 493
 494   // Cache item names.
 495   result = automation_->CreateCacheRequest(cache_request.Receive());
 496   if (FAILED(result)) {
 497     LOG(ERROR) << std::hex << result;
 498     return result;
 499   }
 500   cache_request->AddProperty(UIA_NamePropertyId);
 501
 502   result = element->FindAllBuildCache(
 503       static_cast<TreeScope>(TreeScope_Children | TreeScope_Descendants),
 504       condition.get(), cache_request.get(), element_array.Receive());
 505   if (FAILED(result)) {
 506     LOG(ERROR) << std::hex << result;
 507     return result;
 508   }
 509
 510   if (!element_array.get()) {
 511     LOG(ERROR) << "The window may have vanished.";
 512     return S_OK;
 513   }
 514
 515   int num_elements = 0;
 516   result = element_array->get_Length(&num_elements);
 517   if (FAILED(result)) {
 518     LOG(ERROR) << std::hex << result;
 519     return result;
 520   }
 521
 522   choices->clear();
 523   choices->reserve(num_elements);
 524   for (int i = 0; i < num_elements; ++i) {
 525     child_element.Release();
 526     result = element_array->GetElement(i, child_element.Receive());
 527     if (FAILED(result)) {
 528       LOG(ERROR) << std::hex << result;
 529       continue;
 530     }
 531     result = child_element->GetCachedPropertyValueEx(UIA_NamePropertyId, TRUE,
 532                                                      var.Receive());
 533     if (FAILED(result)) {
 534       LOG(ERROR) << std::hex << result;
 535       continue;
 536     }
 537     if (V_VT(var.ptr()) != VT_BSTR) {
 538       LOG(ERROR) << __FUNCTION__ << " name is not a BSTR: " << V_VT(var.ptr());
 539       continue;
 540     }
 541     choices->push_back(base::string16(V_BSTR(var.ptr())));
 542     var.Reset();
 543   }
 544
 545   return result;
 546 }
 547
 548 // Closes the element |window| by sending it an escape key.
 549 void UIAutomationClient::Context::CloseWindow(
 550     const base::win::ScopedComPtr<IUIAutomationElement>& window) {
 551   DCHECK(thread_checker_.CalledOnValidThread());
 552
 553   // It's tempting to get the Window pattern from |window| and invoke its Close
 554   // method. Unfortunately, this doesn't work. Sending an escape key does the
 555   // trick, though.
 556   HRESULT result = S_OK;
 557   base::win::ScopedVariant var;
 558
 559   result = window->GetCachedPropertyValueEx(
 560       UIA_NativeWindowHandlePropertyId,
 561       TRUE,
 562       var.Receive());
 563   if (FAILED(result)) {
 564     LOG(ERROR) << std::hex << result;
 565     return;
 566   }
 567
 568   if (V_VT(var.ptr()) != VT_I4) {
 569     LOG(ERROR) << __FUNCTION__
 570                << " window handle is not an int: " << V_VT(var.ptr());
 571     return;
 572   }
 573
 574   HWND handle = reinterpret_cast<HWND>(V_I4(var.ptr()));
 575
 576   uint32 scan_code = MapVirtualKey(VK_ESCAPE, MAPVK_VK_TO_VSC);
 577   PostMessage(handle, WM_KEYDOWN, VK_ESCAPE,
 578               MAKELPARAM(1, scan_code));
 579   PostMessage(handle, WM_KEYUP, VK_ESCAPE,
 580               MAKELPARAM(1, scan_code | KF_REPEAT | KF_UP));
 581 }
 582
 583 UIAutomationClient::UIAutomationClient()
 584     : automation_thread_("UIAutomation") {}
 585
 586 UIAutomationClient::~UIAutomationClient() {
 587   DCHECK(thread_checker_.CalledOnValidThread());
 588
 589   // context_ is still valid when the caller destroys the instance before the
 590   // callback(s) have fired. In this case, delete the context on the automation
 591   // thread before joining with it.
 592   automation_thread_.message_loop()->PostTask(
 593       FROM_HERE,
 594       base::Bind(&UIAutomationClient::Context::DeleteOnAutomationThread,
 595                  context_));
 596 }
 597
 598 void UIAutomationClient::Begin(const wchar_t* class_name,
 599                                const base::string16& item_name,
 600                                const InitializedCallback& init_callback,
 601                                const ResultCallback& result_callback) {
 602   DCHECK(thread_checker_.CalledOnValidThread());
 603   DCHECK_EQ(context_.get(), static_cast<Context*>(NULL));
 604
 605   // Start the automation thread and initialize our automation client on it.
 606   context_ = Context::Create();
 607   automation_thread_.init_com_with_mta(true);
 608   automation_thread_.Start();
 609   automation_thread_.message_loop()->PostTask(
 610       FROM_HERE,
 611       base::Bind(&UIAutomationClient::Context::Initialize,
 612                  context_,
 613                  base::ThreadTaskRunnerHandle::Get(),
 614                  base::string16(class_name),
 615                  item_name,
 616                  init_callback,
 617                  result_callback));
 618 }
 619
 620 }  // namespace internal
 621 }  // namespace win8