win8/test/ui_automation_client.cc

   1 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "win8/test/ui_automation_client.h"
   6
   7 #include <atlbase.h>
   8 #include <atlcom.h>
   9 #include <oleauto.h>
  10 #include <uiautomation.h>
  11
  12 #include <algorithm>
  13
  14 #include "base/bind.h"
  15 #include "base/callback.h"
  16 #include "base/memory/ref_counted.h"
  17 #include "base/strings/string_number_conversions.h"
  18 #include "base/strings/string_util.h"
  19 #include "base/thread_task_runner_handle.h"
  20 #include "base/win/scoped_comptr.h"
  21 #include "base/win/scoped_variant.h"
  22
  23 namespace win8 {
  24 namespace internal {
  25
  26 // The guts of the UI automation client which runs on a dedicated thread in the
  27 // multi-threaded COM apartment. An instance may be constructed on any thread,
  28 // but Initialize() must be invoked on a thread in the MTA.
  29 class UIAutomationClient::Context {
  30  public:
  31   // Returns a new instance ready for initialization and use on another thread.
  32   static base::WeakPtr<Context> Create();
  33
  34   // Deletes the instance.
  35   void DeleteOnAutomationThread();
  36
  37   // Initializes the context, invoking |init_callback| via |client_runner| when
  38   // done. On success, |result_callback| will eventually be called after the
  39   // window has been processed. On failure, this instance self-destructs after
  40   // posting |init_callback|.
  41   void Initialize(
  42       scoped_refptr<base::SingleThreadTaskRunner> client_runner,
  43       base::string16 class_name,
  44       base::string16 item_name,
  45       UIAutomationClient::InitializedCallback init_callback,
  46       UIAutomationClient::ResultCallback result_callback);
  47
  48   // Methods invoked by event handlers via weak pointers.
  49   void HandleAutomationEvent(
  50       base::win::ScopedComPtr<IUIAutomationElement> sender,
  51       EVENTID eventId);
  52
  53  private:
  54   class EventHandler;
  55
  56   // The only and only method that may be called from outside of the automation
  57   // thread.
  58   Context();
  59   ~Context();
  60
  61   HRESULT InstallWindowObserver();
  62   HRESULT RemoveWindowObserver();
  63
  64   void HandleWindowOpen(
  65       const base::win::ScopedComPtr<IUIAutomationElement>& window);
  66   void ProcessWindow(
  67       const base::win::ScopedComPtr<IUIAutomationElement>& window);
  68   HRESULT InvokeDesiredItem(
  69       const base::win::ScopedComPtr<IUIAutomationElement>& element);
  70   HRESULT GetInvokableItems(
  71       const base::win::ScopedComPtr<IUIAutomationElement>& element,
  72       std::vector<base::string16>* choices);
  73   void CloseWindow(const base::win::ScopedComPtr<IUIAutomationElement>& window);
  74
  75   base::ThreadChecker thread_checker_;
  76
  77   // The loop on which the client itself lives.
  78   scoped_refptr<base::SingleThreadTaskRunner> client_runner_;
  79
  80   // The class name of the window for which the client waits.
  81   base::string16 class_name_;
  82
  83   // The name of the item to invoke.
  84   base::string16 item_name_;
  85
  86   // The consumer's result callback.
  87   ResultCallback result_callback_;
  88
  89   // The automation client.
  90   base::win::ScopedComPtr<IUIAutomation> automation_;
  91
  92   // A handler of Window open events.
  93   base::win::ScopedComPtr<IUIAutomationEventHandler> event_handler_;
  94
  95   // Weak pointers to the context are given to event handlers.
  96   base::WeakPtrFactory<UIAutomationClient::Context> weak_ptr_factory_;
  97
  98   DISALLOW_COPY_AND_ASSIGN(Context);
  99 };
 100
 101 class UIAutomationClient::Context::EventHandler
 102     : public CComObjectRootEx<CComMultiThreadModel>,
 103       public IUIAutomationEventHandler {
 104  public:
 105   BEGIN_COM_MAP(UIAutomationClient::Context::EventHandler)
 106     COM_INTERFACE_ENTRY(IUIAutomationEventHandler)
 107   END_COM_MAP()
 108
 109   EventHandler();
 110   virtual ~EventHandler();
 111
 112   // Initializes the object with its parent UI automation client context's
 113   // message loop and pointer. Events are dispatched back to the context on
 114   // the given loop.
 115   void Initialize(
 116       const scoped_refptr<base::SingleThreadTaskRunner>& context_runner,
 117       const base::WeakPtr<UIAutomationClient::Context>& context);
 118
 119   // IUIAutomationEventHandler methods.
 120   STDMETHOD(HandleAutomationEvent)(IUIAutomationElement* sender,
 121                                    EVENTID eventId) override;
 122
 123  private:
 124   // The task runner for the UI automation client context.
 125   scoped_refptr<base::SingleThreadTaskRunner> context_runner_;
 126
 127   // The parent UI automation client context.
 128   base::WeakPtr<UIAutomationClient::Context> context_;
 129
 130   DISALLOW_COPY_AND_ASSIGN(EventHandler);
 131 };
 132
 133 UIAutomationClient::Context::EventHandler::EventHandler() {}
 134
 135 UIAutomationClient::Context::EventHandler::~EventHandler() {}
 136
 137 void UIAutomationClient::Context::EventHandler::Initialize(
 138     const scoped_refptr<base::SingleThreadTaskRunner>& context_runner,
 139     const base::WeakPtr<UIAutomationClient::Context>& context) {
 140   context_runner_ = context_runner;
 141   context_ = context;
 142 }
 143
 144 HRESULT UIAutomationClient::Context::EventHandler::HandleAutomationEvent(
 145     IUIAutomationElement* sender,
 146     EVENTID eventId) {
 147   // Event handlers are invoked on an arbitrary thread in the MTA. Send the
 148   // event back to the main UI automation thread for processing.
 149   context_runner_->PostTask(
 150       FROM_HERE,
 151       base::Bind(&UIAutomationClient::Context::HandleAutomationEvent, context_,
 152                  base::win::ScopedComPtr<IUIAutomationElement>(sender),
 153                  eventId));
 154
 155   return S_OK;
 156 }
 157
 158 base::WeakPtr<UIAutomationClient::Context>
 159     UIAutomationClient::Context::Create() {
 160   Context* context = new Context();
 161   return context->weak_ptr_factory_.GetWeakPtr();
 162 }
 163
 164 void UIAutomationClient::Context::DeleteOnAutomationThread() {
 165   DCHECK(thread_checker_.CalledOnValidThread());
 166   delete this;
 167 }
 168
 169 UIAutomationClient::Context::Context() : weak_ptr_factory_(this) {}
 170
 171 UIAutomationClient::Context::~Context() {
 172   DCHECK(thread_checker_.CalledOnValidThread());
 173
 174   if (event_handler_.get()) {
 175     event_handler_ = NULL;
 176     HRESULT result = automation_->RemoveAllEventHandlers();
 177     LOG_IF(ERROR, FAILED(result)) << std::hex << result;
 178   }
 179 }
 180
 181 void UIAutomationClient::Context::Initialize(
 182     scoped_refptr<base::SingleThreadTaskRunner> client_runner,
 183     base::string16 class_name,
 184     base::string16 item_name,
 185     UIAutomationClient::InitializedCallback init_callback,
 186     UIAutomationClient::ResultCallback result_callback) {
 187   // This and all other methods must be called on the automation thread.
 188   DCHECK(!client_runner->BelongsToCurrentThread());
 189   // Bind the checker to this thread.
 190   thread_checker_.DetachFromThread();
 191   DCHECK(thread_checker_.CalledOnValidThread());
 192
 193   client_runner_ = client_runner;
 194   class_name_ = class_name;
 195   item_name_ = item_name;
 196   result_callback_ = result_callback;
 197
 198   HRESULT result = automation_.CreateInstance(CLSID_CUIAutomation, NULL,
 199                                               CLSCTX_INPROC_SERVER);
 200   if (FAILED(result) || !automation_.get())
 201     LOG(ERROR) << std::hex << result;
 202   else
 203     result = InstallWindowObserver();
 204
 205   // Tell the client that initialization is complete.
 206   client_runner_->PostTask(FROM_HERE, base::Bind(init_callback, result));
 207
 208   // Self-destruct if the overall operation failed.
 209   if (FAILED(result))
 210     delete this;
 211 }
 212
 213 // Installs the window observer.
 214 HRESULT UIAutomationClient::Context::InstallWindowObserver() {
 215   DCHECK(thread_checker_.CalledOnValidThread());
 216   DCHECK(automation_.get());
 217   DCHECK(!event_handler_.get());
 218
 219   HRESULT result = S_OK;
 220   base::win::ScopedComPtr<IUIAutomationElement> root_element;
 221   base::win::ScopedComPtr<IUIAutomationCacheRequest> cache_request;
 222
 223   // Observe the opening of all windows.
 224   result = automation_->GetRootElement(root_element.Receive());
 225   if (FAILED(result)) {
 226     LOG(ERROR) << std::hex << result;
 227     return result;
 228   }
 229
 230   // Cache Window class, HWND, and window pattern for opened windows.
 231   result = automation_->CreateCacheRequest(cache_request.Receive());
 232   if (FAILED(result)) {
 233     LOG(ERROR) << std::hex << result;
 234     return result;
 235   }
 236   cache_request->AddProperty(UIA_ClassNamePropertyId);
 237   cache_request->AddProperty(UIA_NativeWindowHandlePropertyId);
 238
 239   // Create the observer.
 240   CComObject<EventHandler>* event_handler_obj = NULL;
 241   result = CComObject<EventHandler>::CreateInstance(&event_handler_obj);
 242   if (FAILED(result)) {
 243     LOG(ERROR) << std::hex << result;
 244     return result;
 245   }
 246   event_handler_obj->Initialize(base::ThreadTaskRunnerHandle::Get(),
 247                                 weak_ptr_factory_.GetWeakPtr());
 248   base::win::ScopedComPtr<IUIAutomationEventHandler> event_handler(
 249       event_handler_obj);
 250
 251   result = automation_->AddAutomationEventHandler(
 252       UIA_Window_WindowOpenedEventId, root_element.get(), TreeScope_Descendants,
 253       cache_request.get(), event_handler.get());
 254
 255   if (FAILED(result)) {
 256     LOG(ERROR) << std::hex << result;
 257     return result;
 258   }
 259
 260   event_handler_ = event_handler;
 261   return S_OK;
 262 }
 263
 264 // Removes this instance's window observer.
 265 HRESULT UIAutomationClient::Context::RemoveWindowObserver() {
 266   DCHECK(thread_checker_.CalledOnValidThread());
 267   DCHECK(automation_.get());
 268   DCHECK(event_handler_.get());
 269
 270   HRESULT result = S_OK;
 271   base::win::ScopedComPtr<IUIAutomationElement> root_element;
 272
 273   // The opening of all windows are observed.
 274   result = automation_->GetRootElement(root_element.Receive());
 275   if (FAILED(result)) {
 276     LOG(ERROR) << std::hex << result;
 277     return result;
 278   }
 279
 280   result = automation_->RemoveAutomationEventHandler(
 281       UIA_Window_WindowOpenedEventId, root_element.get(), event_handler_.get());
 282   if (FAILED(result)) {
 283     LOG(ERROR) << std::hex << result;
 284     return result;
 285   }
 286
 287   event_handler_ = NULL;
 288   return S_OK;
 289 }
 290
 291 // Handles an automation event. If the event results in the processing for which
 292 // this context was created, the context self-destructs after posting the
 293 // results to the client.
 294 void UIAutomationClient::Context::HandleAutomationEvent(
 295     base::win::ScopedComPtr<IUIAutomationElement> sender,
 296     EVENTID eventId) {
 297   DCHECK(thread_checker_.CalledOnValidThread());
 298   if (eventId == UIA_Window_WindowOpenedEventId)
 299     HandleWindowOpen(sender);
 300 }
 301
 302 // Handles a WindowOpen event. If |window| is the one for which this instance is
 303 // waiting, it is processed and this instance self-destructs after posting the
 304 // results to the client.
 305 void UIAutomationClient::Context::HandleWindowOpen(
 306     const base::win::ScopedComPtr<IUIAutomationElement>& window) {
 307   DCHECK(thread_checker_.CalledOnValidThread());
 308   HRESULT hr = S_OK;
 309   base::win::ScopedVariant var;
 310
 311   hr = window->GetCachedPropertyValueEx(UIA_ClassNamePropertyId, TRUE,
 312                                         var.Receive());
 313   if (FAILED(hr)) {
 314     LOG(ERROR) << std::hex << hr;
 315     return;
 316   }
 317
 318   if (V_VT(var.ptr()) != VT_BSTR) {
 319     LOG(ERROR) << __FUNCTION__
 320                << " class name is not a BSTR: " << V_VT(var.ptr());
 321     return;
 322   }
 323
 324   base::string16 class_name(V_BSTR(var.ptr()));
 325
 326   // Window class names are atoms, which are case-insensitive. Assume that
 327   // the window in question only needs ASCII case-insensitivity.
 328   if (class_name.size() == class_name_.size() &&
 329       std::equal(class_name.begin(), class_name.end(), class_name_.begin(),
 330                  base::CaseInsensitiveCompareASCII<wchar_t>())) {
 331     RemoveWindowObserver();
 332     ProcessWindow(window);
 333   }
 334 }
 335
 336 // Processes |window| by invoking the desired child item. If the item cannot be
 337 // found or invoked, an attempt is made to get a list of all invokable children.
 338 // The results are posted back to the client on |client_runner_|, and this
 339 // instance self-destructs.
 340 void UIAutomationClient::Context::ProcessWindow(
 341     const base::win::ScopedComPtr<IUIAutomationElement>& window) {
 342   DCHECK(thread_checker_.CalledOnValidThread());
 343
 344   HRESULT result = S_OK;
 345   std::vector<base::string16> choices;
 346   result = InvokeDesiredItem(window);
 347   if (FAILED(result)) {
 348     GetInvokableItems(window, &choices);
 349     CloseWindow(window);
 350   }
 351
 352   client_runner_->PostTask(FROM_HERE,
 353                            base::Bind(result_callback_, result, choices));
 354
 355   // Self-destruct since there's nothing more to be done here.
 356   delete this;
 357 }
 358
 359 // Invokes the desired child of |element|.
 360 HRESULT UIAutomationClient::Context::InvokeDesiredItem(
 361     const base::win::ScopedComPtr<IUIAutomationElement>& element) {
 362   DCHECK(thread_checker_.CalledOnValidThread());
 363
 364   HRESULT result = S_OK;
 365   base::win::ScopedVariant var;
 366   base::win::ScopedComPtr<IUIAutomationCondition> invokable_condition;
 367   base::win::ScopedComPtr<IUIAutomationCondition> item_name_condition;
 368   base::win::ScopedComPtr<IUIAutomationCondition> control_view_condition;
 369   base::win::ScopedComPtr<IUIAutomationCondition> condition;
 370   base::win::ScopedComPtr<IUIAutomationCacheRequest> cache_request;
 371   base::win::ScopedComPtr<IUIAutomationElement> target;
 372
 373   // Search for an invokable element named item_name.
 374   var.Set(true);
 375   result = automation_->CreatePropertyCondition(
 376       UIA_IsInvokePatternAvailablePropertyId,
 377       var,
 378       invokable_condition.Receive());
 379   var.Reset();
 380   if (FAILED(result)) {
 381     LOG(ERROR) << std::hex << result;
 382     return result;
 383   }
 384
 385   var.Set(item_name_.c_str());
 386   result = automation_->CreatePropertyCondition(UIA_NamePropertyId,
 387                                                 var,
 388                                                 item_name_condition.Receive());
 389   var.Reset();
 390   if (FAILED(result)) {
 391     LOG(ERROR) << std::hex << result;
 392     return result;
 393   }
 394
 395   result = automation_->get_ControlViewCondition(
 396       control_view_condition.Receive());
 397   if (FAILED(result)) {
 398     LOG(ERROR) << std::hex << result;
 399     return result;
 400   }
 401
 402   std::vector<IUIAutomationCondition*> conditions;
 403   conditions.push_back(invokable_condition.get());
 404   conditions.push_back(item_name_condition.get());
 405   conditions.push_back(control_view_condition.get());
 406   result = automation_->CreateAndConditionFromNativeArray(
 407       &conditions[0], conditions.size(), condition.Receive());
 408   if (FAILED(result)) {
 409     LOG(ERROR) << std::hex << result;
 410     return result;
 411   }
 412
 413   // Cache invokable pattern for the item.
 414   result = automation_->CreateCacheRequest(cache_request.Receive());
 415   if (FAILED(result)) {
 416     LOG(ERROR) << std::hex << result;
 417     return result;
 418   }
 419   cache_request->AddPattern(UIA_InvokePatternId);
 420
 421   result = element->FindFirstBuildCache(
 422       static_cast<TreeScope>(TreeScope_Children | TreeScope_Descendants),
 423       condition.get(), cache_request.get(), target.Receive());
 424   if (FAILED(result)) {
 425     LOG(ERROR) << std::hex << result;
 426     return result;
 427   }
 428
 429   // If the item was found, invoke it.
 430   if (!target.get()) {
 431     LOG(WARNING) << "Failed to find desired item to invoke.";
 432     return E_FAIL;
 433   }
 434
 435   base::win::ScopedComPtr<IUIAutomationInvokePattern> invoker;
 436   result = target->GetCachedPatternAs(UIA_InvokePatternId, invoker.iid(),
 437                                       invoker.ReceiveVoid());
 438   if (FAILED(result)) {
 439     LOG(ERROR) << std::hex << result;
 440     return result;
 441   }
 442
 443   result = invoker->Invoke();
 444   if (FAILED(result)) {
 445     LOG(ERROR) << std::hex << result;
 446     return result;
 447   }
 448
 449   return S_OK;
 450 }
 451
 452 // Populates |choices| with the names of all invokable children of |element|.
 453 HRESULT UIAutomationClient::Context::GetInvokableItems(
 454     const base::win::ScopedComPtr<IUIAutomationElement>& element,
 455     std::vector<base::string16>* choices) {
 456   DCHECK(choices);
 457   DCHECK(thread_checker_.CalledOnValidThread());
 458
 459   HRESULT result = S_OK;
 460   base::win::ScopedVariant var;
 461   base::win::ScopedComPtr<IUIAutomationCondition> invokable_condition;
 462   base::win::ScopedComPtr<IUIAutomationCondition> control_view_condition;
 463   base::win::ScopedComPtr<IUIAutomationCondition> condition;
 464   base::win::ScopedComPtr<IUIAutomationCacheRequest> cache_request;
 465   base::win::ScopedComPtr<IUIAutomationElementArray> element_array;
 466   base::win::ScopedComPtr<IUIAutomationElement> child_element;
 467
 468   // Search for all invokable elements.
 469   var.Set(true);
 470   result = automation_->CreatePropertyCondition(
 471       UIA_IsInvokePatternAvailablePropertyId,
 472       var,
 473       invokable_condition.Receive());
 474   var.Reset();
 475   if (FAILED(result)) {
 476     LOG(ERROR) << std::hex << result;
 477     return result;
 478   }
 479
 480   result = automation_->get_ControlViewCondition(
 481       control_view_condition.Receive());
 482   if (FAILED(result)) {
 483     LOG(ERROR) << std::hex << result;
 484     return result;
 485   }
 486
 487   result = automation_->CreateAndCondition(invokable_condition.get(),
 488                                            control_view_condition.get(),
 489                                            condition.Receive());
 490   if (FAILED(result)) {
 491     LOG(ERROR) << std::hex << result;
 492     return result;
 493   }
 494
 495   // Cache item names.
 496   result = automation_->CreateCacheRequest(cache_request.Receive());
 497   if (FAILED(result)) {
 498     LOG(ERROR) << std::hex << result;
 499     return result;
 500   }
 501   cache_request->AddProperty(UIA_NamePropertyId);
 502
 503   result = element->FindAllBuildCache(
 504       static_cast<TreeScope>(TreeScope_Children | TreeScope_Descendants),
 505       condition.get(), cache_request.get(), element_array.Receive());
 506   if (FAILED(result)) {
 507     LOG(ERROR) << std::hex << result;
 508     return result;
 509   }
 510
 511   if (!element_array.get()) {
 512     LOG(ERROR) << "The window may have vanished.";
 513     return S_OK;
 514   }
 515
 516   int num_elements = 0;
 517   result = element_array->get_Length(&num_elements);
 518   if (FAILED(result)) {
 519     LOG(ERROR) << std::hex << result;
 520     return result;
 521   }
 522
 523   choices->clear();
 524   choices->reserve(num_elements);
 525   for (int i = 0; i < num_elements; ++i) {
 526     child_element.Release();
 527     result = element_array->GetElement(i, child_element.Receive());
 528     if (FAILED(result)) {
 529       LOG(ERROR) << std::hex << result;
 530       continue;
 531     }
 532     result = child_element->GetCachedPropertyValueEx(UIA_NamePropertyId, TRUE,
 533                                                      var.Receive());
 534     if (FAILED(result)) {
 535       LOG(ERROR) << std::hex << result;
 536       continue;
 537     }
 538     if (V_VT(var.ptr()) != VT_BSTR) {
 539       LOG(ERROR) << __FUNCTION__ << " name is not a BSTR: " << V_VT(var.ptr());
 540       continue;
 541     }
 542     choices->push_back(base::string16(V_BSTR(var.ptr())));
 543     var.Reset();
 544   }
 545
 546   return result;
 547 }
 548
 549 // Closes the element |window| by sending it an escape key.
 550 void UIAutomationClient::Context::CloseWindow(
 551     const base::win::ScopedComPtr<IUIAutomationElement>& window) {
 552   DCHECK(thread_checker_.CalledOnValidThread());
 553
 554   // It's tempting to get the Window pattern from |window| and invoke its Close
 555   // method. Unfortunately, this doesn't work. Sending an escape key does the
 556   // trick, though.
 557   HRESULT result = S_OK;
 558   base::win::ScopedVariant var;
 559
 560   result = window->GetCachedPropertyValueEx(
 561       UIA_NativeWindowHandlePropertyId,
 562       TRUE,
 563       var.Receive());
 564   if (FAILED(result)) {
 565     LOG(ERROR) << std::hex << result;
 566     return;
 567   }
 568
 569   if (V_VT(var.ptr()) != VT_I4) {
 570     LOG(ERROR) << __FUNCTION__
 571                << " window handle is not an int: " << V_VT(var.ptr());
 572     return;
 573   }
 574
 575   HWND handle = reinterpret_cast<HWND>(V_I4(var.ptr()));
 576
 577   uint32 scan_code = MapVirtualKey(VK_ESCAPE, MAPVK_VK_TO_VSC);
 578   PostMessage(handle, WM_KEYDOWN, VK_ESCAPE,
 579               MAKELPARAM(1, scan_code));
 580   PostMessage(handle, WM_KEYUP, VK_ESCAPE,
 581               MAKELPARAM(1, scan_code | KF_REPEAT | KF_UP));
 582 }
 583
 584 UIAutomationClient::UIAutomationClient()
 585     : automation_thread_("UIAutomation") {}
 586
 587 UIAutomationClient::~UIAutomationClient() {
 588   DCHECK(thread_checker_.CalledOnValidThread());
 589
 590   // context_ is still valid when the caller destroys the instance before the
 591   // callback(s) have fired. In this case, delete the context on the automation
 592   // thread before joining with it.
 593   automation_thread_.message_loop()->PostTask(
 594       FROM_HERE,
 595       base::Bind(&UIAutomationClient::Context::DeleteOnAutomationThread,
 596                  context_));
 597 }
 598
 599 void UIAutomationClient::Begin(const wchar_t* class_name,
 600                                const base::string16& item_name,
 601                                const InitializedCallback& init_callback,
 602                                const ResultCallback& result_callback) {
 603   DCHECK(thread_checker_.CalledOnValidThread());
 604   DCHECK_EQ(context_.get(), static_cast<Context*>(NULL));
 605
 606   // Start the automation thread and initialize our automation client on it.
 607   context_ = Context::Create();
 608   automation_thread_.init_com_with_mta(true);
 609   automation_thread_.Start();
 610   automation_thread_.message_loop()->PostTask(
 611       FROM_HERE,
 612       base::Bind(&UIAutomationClient::Context::Initialize,
 613                  context_,
 614                  base::ThreadTaskRunnerHandle::Get(),
 615                  base::string16(class_name),
 616                  item_name,
 617                  init_callback,
 618                  result_callback));
 619 }
 620
 621 }  // namespace internal
 622 }  // namespace win8