win8/test/ui_automation_client.cc

   1 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "win8/test/ui_automation_client.h"
   6
   7 #include <atlbase.h>
   8 #include <atlcom.h>
   9 #include <oleauto.h>
  10 #include <uiautomation.h>
  11
  12 #include <algorithm>
  13
  14 #include "base/bind.h"
  15 #include "base/callback.h"
  16 #include "base/memory/ref_counted.h"
  17 #include "base/strings/string_number_conversions.h"
  18 #include "base/strings/string_util.h"
  19 #include "base/thread_task_runner_handle.h"
  20 #include "base/win/scoped_comptr.h"
  21 #include "base/win/scoped_variant.h"
  22
  23 namespace win8 {
  24 namespace internal {
  25
  26 // The guts of the UI automation client which runs on a dedicated thread in the
  27 // multi-threaded COM apartment. An instance may be constructed on any thread,
  28 // but Initialize() must be invoked on a thread in the MTA.
  29 class UIAutomationClient::Context {
  30  public:
  31   // Returns a new instance ready for initialization and use on another thread.
  32   static base::WeakPtr<Context> Create();
  33
  34   // Deletes the instance.
  35   void DeleteOnAutomationThread();
  36
  37   // Initializes the context, invoking |init_callback| via |client_runner| when
  38   // done. On success, |result_callback| will eventually be called after the
  39   // window has been processed. On failure, this instance self-destructs after
  40   // posting |init_callback|.
  41   void Initialize(
  42       scoped_refptr<base::SingleThreadTaskRunner> client_runner,
  43       base::string16 class_name,
  44       base::string16 item_name,
  45       UIAutomationClient::InitializedCallback init_callback,
  46       UIAutomationClient::ResultCallback result_callback);
  47
  48   // Methods invoked by event handlers via weak pointers.
  49   void HandleAutomationEvent(
  50       base::win::ScopedComPtr<IUIAutomationElement> sender,
  51       EVENTID eventId);
  52
  53  private:
  54   class EventHandler;
  55
  56   // The only and only method that may be called from outside of the automation
  57   // thread.
  58   Context();
  59   ~Context();
  60
  61   HRESULT InstallWindowObserver();
  62   HRESULT RemoveWindowObserver();
  63
  64   void HandleWindowOpen(
  65       const base::win::ScopedComPtr<IUIAutomationElement>& window);
  66   void ProcessWindow(
  67       const base::win::ScopedComPtr<IUIAutomationElement>& window);
  68   HRESULT InvokeDesiredItem(
  69       const base::win::ScopedComPtr<IUIAutomationElement>& element);
  70   HRESULT GetInvokableItems(
  71       const base::win::ScopedComPtr<IUIAutomationElement>& element,
  72       std::vector<base::string16>* choices);
  73   void CloseWindow(const base::win::ScopedComPtr<IUIAutomationElement>& window);
  74
  75   base::ThreadChecker thread_checker_;
  76
  77   // The loop on which the client itself lives.
  78   scoped_refptr<base::SingleThreadTaskRunner> client_runner_;
  79
  80   // The class name of the window for which the client waits.
  81   base::string16 class_name_;
  82
  83   // The name of the item to invoke.
  84   base::string16 item_name_;
  85
  86   // The consumer's result callback.
  87   ResultCallback result_callback_;
  88
  89   // The automation client.
  90   base::win::ScopedComPtr<IUIAutomation> automation_;
  91
  92   // A handler of Window open events.
  93   base::win::ScopedComPtr<IUIAutomationEventHandler> event_handler_;
  94
  95   // Weak pointers to the context are given to event handlers.
  96   base::WeakPtrFactory<UIAutomationClient::Context> weak_ptr_factory_;
  97
  98   DISALLOW_COPY_AND_ASSIGN(Context);
  99 };
 100
 101 class UIAutomationClient::Context::EventHandler
 102     : public CComObjectRootEx<CComMultiThreadModel>,
 103       public IUIAutomationEventHandler {
 104  public:
 105   BEGIN_COM_MAP(UIAutomationClient::Context::EventHandler)
 106     COM_INTERFACE_ENTRY(IUIAutomationEventHandler)
 107   END_COM_MAP()
 108
 109   EventHandler();
 110   virtual ~EventHandler();
 111
 112   // Initializes the object with its parent UI automation client context's
 113   // message loop and pointer. Events are dispatched back to the context on
 114   // the given loop.
 115   void Initialize(
 116       const scoped_refptr<base::SingleThreadTaskRunner>& context_runner,
 117       const base::WeakPtr<UIAutomationClient::Context>& context);
 118
 119   // IUIAutomationEventHandler methods.
 120   STDMETHOD(HandleAutomationEvent)(IUIAutomationElement* sender,
 121                                    EVENTID eventId);
 122
 123  private:
 124   // The task runner for the UI automation client context.
 125   scoped_refptr<base::SingleThreadTaskRunner> context_runner_;
 126
 127   // The parent UI automation client context.
 128   base::WeakPtr<UIAutomationClient::Context> context_;
 129
 130   DISALLOW_COPY_AND_ASSIGN(EventHandler);
 131 };
 132
 133 UIAutomationClient::Context::EventHandler::EventHandler() {}
 134
 135 UIAutomationClient::Context::EventHandler::~EventHandler() {}
 136
 137 void UIAutomationClient::Context::EventHandler::Initialize(
 138     const scoped_refptr<base::SingleThreadTaskRunner>& context_runner,
 139     const base::WeakPtr<UIAutomationClient::Context>& context) {
 140   context_runner_ = context_runner;
 141   context_ = context;
 142 }
 143
 144 HRESULT UIAutomationClient::Context::EventHandler::HandleAutomationEvent(
 145     IUIAutomationElement* sender,
 146     EVENTID eventId) {
 147   // Event handlers are invoked on an arbitrary thread in the MTA. Send the
 148   // event back to the main UI automation thread for processing.
 149   context_runner_->PostTask(
 150       FROM_HERE,
 151       base::Bind(&UIAutomationClient::Context::HandleAutomationEvent, context_,
 152                  base::win::ScopedComPtr<IUIAutomationElement>(sender),
 153                  eventId));
 154
 155   return S_OK;
 156 }
 157
 158 base::WeakPtr<UIAutomationClient::Context>
 159     UIAutomationClient::Context::Create() {
 160   Context* context = new Context();
 161   return context->weak_ptr_factory_.GetWeakPtr();
 162 }
 163
 164 void UIAutomationClient::Context::DeleteOnAutomationThread() {
 165   DCHECK(thread_checker_.CalledOnValidThread());
 166   delete this;
 167 }
 168
 169 UIAutomationClient::Context::Context() : weak_ptr_factory_(this) {}
 170
 171 UIAutomationClient::Context::~Context() {
 172   DCHECK(thread_checker_.CalledOnValidThread());
 173
 174   if (event_handler_.get()) {
 175     event_handler_ = NULL;
 176     HRESULT result = automation_->RemoveAllEventHandlers();
 177     LOG_IF(ERROR, FAILED(result)) << std::hex << result;
 178   }
 179 }
 180
 181 void UIAutomationClient::Context::Initialize(
 182     scoped_refptr<base::SingleThreadTaskRunner> client_runner,
 183     base::string16 class_name,
 184     base::string16 item_name,
 185     UIAutomationClient::InitializedCallback init_callback,
 186     UIAutomationClient::ResultCallback result_callback) {
 187   // This and all other methods must be called on the automation thread.
 188   DCHECK(!client_runner->BelongsToCurrentThread());
 189   // Bind the checker to this thread.
 190   thread_checker_.DetachFromThread();
 191   DCHECK(thread_checker_.CalledOnValidThread());
 192
 193   client_runner_ = client_runner;
 194   class_name_ = class_name;
 195   item_name_ = item_name;
 196   result_callback_ = result_callback;
 197
 198   HRESULT result = automation_.CreateInstance(CLSID_CUIAutomation, NULL,
 199                                               CLSCTX_INPROC_SERVER);
 200   if (FAILED(result) || !automation_.get())
 201     LOG(ERROR) << std::hex << result;
 202   else
 203     result = InstallWindowObserver();
 204
 205   // Tell the client that initialization is complete.
 206   client_runner_->PostTask(FROM_HERE, base::Bind(init_callback, result));
 207
 208   // Self-destruct if the overall operation failed.
 209   if (FAILED(result))
 210     delete this;
 211 }
 212
 213 // Installs the window observer.
 214 HRESULT UIAutomationClient::Context::InstallWindowObserver() {
 215   DCHECK(thread_checker_.CalledOnValidThread());
 216   DCHECK(automation_.get());
 217   DCHECK(!event_handler_.get());
 218
 219   HRESULT result = S_OK;
 220   base::win::ScopedComPtr<IUIAutomationElement> root_element;
 221   base::win::ScopedComPtr<IUIAutomationCacheRequest> cache_request;
 222
 223   // Observe the opening of all windows.
 224   result = automation_->GetRootElement(root_element.Receive());
 225   if (FAILED(result)) {
 226     LOG(ERROR) << std::hex << result;
 227     return result;
 228   }
 229
 230   // Cache Window class, HWND, and window pattern for opened windows.
 231   result = automation_->CreateCacheRequest(cache_request.Receive());
 232   if (FAILED(result)) {
 233     LOG(ERROR) << std::hex << result;
 234     return result;
 235   }
 236   cache_request->AddProperty(UIA_ClassNamePropertyId);
 237   cache_request->AddProperty(UIA_NativeWindowHandlePropertyId);
 238
 239   // Create the observer.
 240   CComObject<EventHandler>* event_handler_obj = NULL;
 241   result = CComObject<EventHandler>::CreateInstance(&event_handler_obj);
 242   if (FAILED(result)) {
 243     LOG(ERROR) << std::hex << result;
 244     return result;
 245   }
 246   event_handler_obj->Initialize(base::ThreadTaskRunnerHandle::Get(),
 247                                 weak_ptr_factory_.GetWeakPtr());
 248   base::win::ScopedComPtr<IUIAutomationEventHandler> event_handler(
 249       event_handler_obj);
 250
 251   result = automation_->AddAutomationEventHandler(
 252       UIA_Window_WindowOpenedEventId,
 253       root_element,
 254       TreeScope_Descendants,
 255       cache_request,
 256       event_handler);
 257
 258   if (FAILED(result)) {
 259     LOG(ERROR) << std::hex << result;
 260     return result;
 261   }
 262
 263   event_handler_ = event_handler;
 264   return S_OK;
 265 }
 266
 267 // Removes this instance's window observer.
 268 HRESULT UIAutomationClient::Context::RemoveWindowObserver() {
 269   DCHECK(thread_checker_.CalledOnValidThread());
 270   DCHECK(automation_.get());
 271   DCHECK(event_handler_.get());
 272
 273   HRESULT result = S_OK;
 274   base::win::ScopedComPtr<IUIAutomationElement> root_element;
 275
 276   // The opening of all windows are observed.
 277   result = automation_->GetRootElement(root_element.Receive());
 278   if (FAILED(result)) {
 279     LOG(ERROR) << std::hex << result;
 280     return result;
 281   }
 282
 283   result = automation_->RemoveAutomationEventHandler(
 284       UIA_Window_WindowOpenedEventId,
 285       root_element,
 286       event_handler_);
 287   if (FAILED(result)) {
 288     LOG(ERROR) << std::hex << result;
 289     return result;
 290   }
 291
 292   event_handler_ = NULL;
 293   return S_OK;
 294 }
 295
 296 // Handles an automation event. If the event results in the processing for which
 297 // this context was created, the context self-destructs after posting the
 298 // results to the client.
 299 void UIAutomationClient::Context::HandleAutomationEvent(
 300     base::win::ScopedComPtr<IUIAutomationElement> sender,
 301     EVENTID eventId) {
 302   DCHECK(thread_checker_.CalledOnValidThread());
 303   if (eventId == UIA_Window_WindowOpenedEventId)
 304     HandleWindowOpen(sender);
 305 }
 306
 307 // Handles a WindowOpen event. If |window| is the one for which this instance is
 308 // waiting, it is processed and this instance self-destructs after posting the
 309 // results to the client.
 310 void UIAutomationClient::Context::HandleWindowOpen(
 311     const base::win::ScopedComPtr<IUIAutomationElement>& window) {
 312   DCHECK(thread_checker_.CalledOnValidThread());
 313   HRESULT hr = S_OK;
 314   base::win::ScopedVariant var;
 315
 316   hr = window->GetCachedPropertyValueEx(UIA_ClassNamePropertyId, TRUE,
 317                                         var.Receive());
 318   if (FAILED(hr)) {
 319     LOG(ERROR) << std::hex << hr;
 320     return;
 321   }
 322
 323   if (V_VT(&var) != VT_BSTR) {
 324     LOG(ERROR) << __FUNCTION__ " class name is not a BSTR: " << V_VT(&var);
 325     return;
 326   }
 327
 328   base::string16 class_name(V_BSTR(&var));
 329
 330   // Window class names are atoms, which are case-insensitive.
 331   if (class_name.size() == class_name_.size() &&
 332       std::equal(class_name.begin(), class_name.end(), class_name_.begin(),
 333                  base::CaseInsensitiveCompare<wchar_t>())) {
 334     RemoveWindowObserver();
 335     ProcessWindow(window);
 336   }
 337 }
 338
 339 // Processes |window| by invoking the desired child item. If the item cannot be
 340 // found or invoked, an attempt is made to get a list of all invokable children.
 341 // The results are posted back to the client on |client_runner_|, and this
 342 // instance self-destructs.
 343 void UIAutomationClient::Context::ProcessWindow(
 344     const base::win::ScopedComPtr<IUIAutomationElement>& window) {
 345   DCHECK(thread_checker_.CalledOnValidThread());
 346
 347   HRESULT result = S_OK;
 348   std::vector<base::string16> choices;
 349   result = InvokeDesiredItem(window);
 350   if (FAILED(result)) {
 351     GetInvokableItems(window, &choices);
 352     CloseWindow(window);
 353   }
 354
 355   client_runner_->PostTask(FROM_HERE,
 356                            base::Bind(result_callback_, result, choices));
 357
 358   // Self-destruct since there's nothing more to be done here.
 359   delete this;
 360 }
 361
 362 // Invokes the desired child of |element|.
 363 HRESULT UIAutomationClient::Context::InvokeDesiredItem(
 364     const base::win::ScopedComPtr<IUIAutomationElement>& element) {
 365   DCHECK(thread_checker_.CalledOnValidThread());
 366
 367   HRESULT result = S_OK;
 368   base::win::ScopedVariant var;
 369   base::win::ScopedComPtr<IUIAutomationCondition> invokable_condition;
 370   base::win::ScopedComPtr<IUIAutomationCondition> item_name_condition;
 371   base::win::ScopedComPtr<IUIAutomationCondition> control_view_condition;
 372   base::win::ScopedComPtr<IUIAutomationCondition> condition;
 373   base::win::ScopedComPtr<IUIAutomationCacheRequest> cache_request;
 374   base::win::ScopedComPtr<IUIAutomationElement> target;
 375
 376   // Search for an invokable element named item_name.
 377   var.Set(true);
 378   result = automation_->CreatePropertyCondition(
 379       UIA_IsInvokePatternAvailablePropertyId,
 380       var,
 381       invokable_condition.Receive());
 382   var.Reset();
 383   if (FAILED(result)) {
 384     LOG(ERROR) << std::hex << result;
 385     return false;
 386   }
 387
 388   var.Set(item_name_.c_str());
 389   result = automation_->CreatePropertyCondition(UIA_NamePropertyId,
 390                                                 var,
 391                                                 item_name_condition.Receive());
 392   var.Reset();
 393   if (FAILED(result)) {
 394     LOG(ERROR) << std::hex << result;
 395     return result;
 396   }
 397
 398   result = automation_->get_ControlViewCondition(
 399       control_view_condition.Receive());
 400   if (FAILED(result)) {
 401     LOG(ERROR) << std::hex << result;
 402     return result;
 403   }
 404
 405   std::vector<IUIAutomationCondition*> conditions;
 406   conditions.push_back(invokable_condition.get());
 407   conditions.push_back(item_name_condition.get());
 408   conditions.push_back(control_view_condition.get());
 409   result = automation_->CreateAndConditionFromNativeArray(
 410       &conditions[0], conditions.size(), condition.Receive());
 411   if (FAILED(result)) {
 412     LOG(ERROR) << std::hex << result;
 413     return result;
 414   }
 415
 416   // Cache invokable pattern for the item.
 417   result = automation_->CreateCacheRequest(cache_request.Receive());
 418   if (FAILED(result)) {
 419     LOG(ERROR) << std::hex << result;
 420     return result;
 421   }
 422   cache_request->AddPattern(UIA_InvokePatternId);
 423
 424   result = element->FindFirstBuildCache(
 425       static_cast<TreeScope>(TreeScope_Children | TreeScope_Descendants),
 426       condition,
 427       cache_request,
 428       target.Receive());
 429   if (FAILED(result)) {
 430     LOG(ERROR) << std::hex << result;
 431     return result;
 432   }
 433
 434   // If the item was found, invoke it.
 435   if (!target.get()) {
 436     LOG(WARNING) << "Failed to find desired item to invoke.";
 437     return E_FAIL;
 438   }
 439
 440   base::win::ScopedComPtr<IUIAutomationInvokePattern> invoker;
 441   result = target->GetCachedPatternAs(UIA_InvokePatternId, invoker.iid(),
 442                                       invoker.ReceiveVoid());
 443   if (FAILED(result)) {
 444     LOG(ERROR) << std::hex << result;
 445     return result;
 446   }
 447
 448   result = invoker->Invoke();
 449   if (FAILED(result)) {
 450     LOG(ERROR) << std::hex << result;
 451     return result;
 452   }
 453
 454   return S_OK;
 455 }
 456
 457 // Populates |choices| with the names of all invokable children of |element|.
 458 HRESULT UIAutomationClient::Context::GetInvokableItems(
 459     const base::win::ScopedComPtr<IUIAutomationElement>& element,
 460     std::vector<base::string16>* choices) {
 461   DCHECK(choices);
 462   DCHECK(thread_checker_.CalledOnValidThread());
 463
 464   HRESULT result = S_OK;
 465   base::win::ScopedVariant var;
 466   base::win::ScopedComPtr<IUIAutomationCondition> invokable_condition;
 467   base::win::ScopedComPtr<IUIAutomationCondition> control_view_condition;
 468   base::win::ScopedComPtr<IUIAutomationCondition> condition;
 469   base::win::ScopedComPtr<IUIAutomationCacheRequest> cache_request;
 470   base::win::ScopedComPtr<IUIAutomationElementArray> element_array;
 471   base::win::ScopedComPtr<IUIAutomationElement> child_element;
 472
 473   // Search for all invokable elements.
 474   var.Set(true);
 475   result = automation_->CreatePropertyCondition(
 476       UIA_IsInvokePatternAvailablePropertyId,
 477       var,
 478       invokable_condition.Receive());
 479   var.Reset();
 480   if (FAILED(result)) {
 481     LOG(ERROR) << std::hex << result;
 482     return result;
 483   }
 484
 485   result = automation_->get_ControlViewCondition(
 486       control_view_condition.Receive());
 487   if (FAILED(result)) {
 488     LOG(ERROR) << std::hex << result;
 489     return result;
 490   }
 491
 492   result = automation_->CreateAndCondition(
 493       invokable_condition, control_view_condition, condition.Receive());
 494   if (FAILED(result)) {
 495     LOG(ERROR) << std::hex << result;
 496     return result;
 497   }
 498
 499   // Cache item names.
 500   result = automation_->CreateCacheRequest(cache_request.Receive());
 501   if (FAILED(result)) {
 502     LOG(ERROR) << std::hex << result;
 503     return result;
 504   }
 505   cache_request->AddProperty(UIA_NamePropertyId);
 506
 507   result = element->FindAllBuildCache(
 508       static_cast<TreeScope>(TreeScope_Children | TreeScope_Descendants),
 509       condition,
 510       cache_request,
 511       element_array.Receive());
 512   if (FAILED(result)) {
 513     LOG(ERROR) << std::hex << result;
 514     return result;
 515   }
 516
 517   if (!element_array.get()) {
 518     LOG(ERROR) << "The window may have vanished.";
 519     return S_OK;
 520   }
 521
 522   int num_elements = 0;
 523   result = element_array->get_Length(&num_elements);
 524   if (FAILED(result)) {
 525     LOG(ERROR) << std::hex << result;
 526     return result;
 527   }
 528
 529   choices->clear();
 530   choices->reserve(num_elements);
 531   for (int i = 0; i < num_elements; ++i) {
 532     child_element.Release();
 533     result = element_array->GetElement(i, child_element.Receive());
 534     if (FAILED(result)) {
 535       LOG(ERROR) << std::hex << result;
 536       continue;
 537     }
 538     result = child_element->GetCachedPropertyValueEx(UIA_NamePropertyId, TRUE,
 539                                                      var.Receive());
 540     if (FAILED(result)) {
 541       LOG(ERROR) << std::hex << result;
 542       continue;
 543     }
 544     if (V_VT(&var) != VT_BSTR) {
 545       LOG(ERROR) << __FUNCTION__ " name is not a BSTR: " << V_VT(&var);
 546       continue;
 547     }
 548     choices->push_back(base::string16(V_BSTR(&var)));
 549     var.Reset();
 550   }
 551
 552   return result;
 553 }
 554
 555 // Closes the element |window| by sending it an escape key.
 556 void UIAutomationClient::Context::CloseWindow(
 557     const base::win::ScopedComPtr<IUIAutomationElement>& window) {
 558   DCHECK(thread_checker_.CalledOnValidThread());
 559
 560   // It's tempting to get the Window pattern from |window| and invoke its Close
 561   // method. Unfortunately, this doesn't work. Sending an escape key does the
 562   // trick, though.
 563   HRESULT result = S_OK;
 564   base::win::ScopedVariant var;
 565
 566   result = window->GetCachedPropertyValueEx(
 567       UIA_NativeWindowHandlePropertyId,
 568       TRUE,
 569       var.Receive());
 570   if (FAILED(result)) {
 571     LOG(ERROR) << std::hex << result;
 572     return;
 573   }
 574
 575   if (V_VT(&var) != VT_I4) {
 576     LOG(ERROR) << __FUNCTION__ " window handle is not an int: " << V_VT(&var);
 577     return;
 578   }
 579
 580   HWND handle = reinterpret_cast<HWND>(V_I4(&var));
 581
 582   uint32 scan_code = MapVirtualKey(VK_ESCAPE, MAPVK_VK_TO_VSC);
 583   PostMessage(handle, WM_KEYDOWN, VK_ESCAPE,
 584               MAKELPARAM(1, scan_code));
 585   PostMessage(handle, WM_KEYUP, VK_ESCAPE,
 586               MAKELPARAM(1, scan_code | KF_REPEAT | KF_UP));
 587 }
 588
 589 UIAutomationClient::UIAutomationClient()
 590     : automation_thread_("UIAutomation") {}
 591
 592 UIAutomationClient::~UIAutomationClient() {
 593   DCHECK(thread_checker_.CalledOnValidThread());
 594
 595   // context_ is still valid when the caller destroys the instance before the
 596   // callback(s) have fired. In this case, delete the context on the automation
 597   // thread before joining with it.
 598   automation_thread_.message_loop()->PostTask(
 599       FROM_HERE,
 600       base::Bind(&UIAutomationClient::Context::DeleteOnAutomationThread,
 601                  context_));
 602 }
 603
 604 void UIAutomationClient::Begin(const wchar_t* class_name,
 605                                const base::string16& item_name,
 606                                const InitializedCallback& init_callback,
 607                                const ResultCallback& result_callback) {
 608   DCHECK(thread_checker_.CalledOnValidThread());
 609   DCHECK_EQ(context_.get(), static_cast<Context*>(NULL));
 610
 611   // Start the automation thread and initialize our automation client on it.
 612   context_ = Context::Create();
 613   automation_thread_.init_com_with_mta(true);
 614   automation_thread_.Start();
 615   automation_thread_.message_loop()->PostTask(
 616       FROM_HERE,
 617       base::Bind(&UIAutomationClient::Context::Initialize,
 618                  context_,
 619                  base::ThreadTaskRunnerHandle::Get(),
 620                  base::string16(class_name),
 621                  item_name,
 622                  init_callback,
 623                  result_callback));
 624 }
 625
 626 }  // namespace internal
 627 }  // namespace win8