win8/test/ui_automation_client.cc

   1 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "win8/test/ui_automation_client.h"
   6
   7 #include <atlbase.h>
   8 #include <atlcom.h>
   9 #include <oleauto.h>
  10 #include <uiautomation.h>
  11
  12 #include <algorithm>
  13
  14 #include "base/bind.h"
  15 #include "base/callback.h"
  16 #include "base/memory/ref_counted.h"
  17 #include "base/strings/string_number_conversions.h"
  18 #include "base/strings/string_util.h"
  19 #include "base/thread_task_runner_handle.h"
  20 #include "base/win/scoped_comptr.h"
  21 #include "base/win/scoped_variant.h"
  22
  23 namespace win8 {
  24 namespace internal {
  25
  26 // The guts of the UI automation client which runs on a dedicated thread in the
  27 // multi-threaded COM apartment. An instance may be constructed on any thread,
  28 // but Initialize() must be invoked on a thread in the MTA.
  29 class UIAutomationClient::Context {
  30  public:
  31   // Returns a new instance ready for initialization and use on another thread.
  32   static base::WeakPtr<Context> Create();
  33
  34   // Deletes the instance.
  35   void DeleteOnAutomationThread();
  36
  37   // Initializes the context, invoking |init_callback| via |client_runner| when
  38   // done. On success, |result_callback| will eventually be called after the
  39   // window has been processed. On failure, this instance self-destructs after
  40   // posting |init_callback|.
  41   void Initialize(
  42       scoped_refptr<base::SingleThreadTaskRunner> client_runner,
  43       base::string16 class_name,
  44       base::string16 item_name,
  45       UIAutomationClient::InitializedCallback init_callback,
  46       UIAutomationClient::ResultCallback result_callback);
  47
  48   // Methods invoked by event handlers via weak pointers.
  49   void HandleAutomationEvent(
  50       base::win::ScopedComPtr<IUIAutomationElement> sender,
  51       EVENTID eventId);
  52
  53  private:
  54   class EventHandler;
  55
  56   // The only and only method that may be called from outside of the automation
  57   // thread.
  58   Context();
  59   ~Context();
  60
  61   HRESULT InstallWindowObserver();
  62   HRESULT RemoveWindowObserver();
  63
  64   void HandleWindowOpen(
  65       const base::win::ScopedComPtr<IUIAutomationElement>& window);
  66   void ProcessWindow(
  67       const base::win::ScopedComPtr<IUIAutomationElement>& window);
  68   HRESULT InvokeDesiredItem(
  69       const base::win::ScopedComPtr<IUIAutomationElement>& element);
  70   HRESULT GetInvokableItems(
  71       const base::win::ScopedComPtr<IUIAutomationElement>& element,
  72       std::vector<base::string16>* choices);
  73   void CloseWindow(const base::win::ScopedComPtr<IUIAutomationElement>& window);
  74
  75   base::ThreadChecker thread_checker_;
  76
  77   // The loop on which the client itself lives.
  78   scoped_refptr<base::SingleThreadTaskRunner> client_runner_;
  79
  80   // The class name of the window for which the client waits.
  81   base::string16 class_name_;
  82
  83   // The name of the item to invoke.
  84   base::string16 item_name_;
  85
  86   // The consumer's result callback.
  87   ResultCallback result_callback_;
  88
  89   // The automation client.
  90   base::win::ScopedComPtr<IUIAutomation> automation_;
  91
  92   // A handler of Window open events.
  93   base::win::ScopedComPtr<IUIAutomationEventHandler> event_handler_;
  94
  95   // Weak pointers to the context are given to event handlers.
  96   base::WeakPtrFactory<UIAutomationClient::Context> weak_ptr_factory_;
  97
  98   DISALLOW_COPY_AND_ASSIGN(Context);
  99 };
 100
 101 class UIAutomationClient::Context::EventHandler
 102     : public CComObjectRootEx<CComMultiThreadModel>,
 103       public IUIAutomationEventHandler {
 104  public:
 105   BEGIN_COM_MAP(UIAutomationClient::Context::EventHandler)
 106     COM_INTERFACE_ENTRY(IUIAutomationEventHandler)
 107   END_COM_MAP()
 108
 109   EventHandler();
 110   virtual ~EventHandler();
 111
 112   // Initializes the object with its parent UI automation client context's
 113   // message loop and pointer. Events are dispatched back to the context on
 114   // the given loop.
 115   void Initialize(
 116       const scoped_refptr<base::SingleThreadTaskRunner>& context_runner,
 117       const base::WeakPtr<UIAutomationClient::Context>& context);
 118
 119   // IUIAutomationEventHandler methods.
 120   STDMETHOD(HandleAutomationEvent)(IUIAutomationElement* sender,
 121                                    EVENTID eventId);
 122
 123  private:
 124   // The task runner for the UI automation client context.
 125   scoped_refptr<base::SingleThreadTaskRunner> context_runner_;
 126
 127   // The parent UI automation client context.
 128   base::WeakPtr<UIAutomationClient::Context> context_;
 129
 130   DISALLOW_COPY_AND_ASSIGN(EventHandler);
 131 };
 132
 133 UIAutomationClient::Context::EventHandler::EventHandler() {}
 134
 135 UIAutomationClient::Context::EventHandler::~EventHandler() {}
 136
 137 void UIAutomationClient::Context::EventHandler::Initialize(
 138     const scoped_refptr<base::SingleThreadTaskRunner>& context_runner,
 139     const base::WeakPtr<UIAutomationClient::Context>& context) {
 140   context_runner_ = context_runner;
 141   context_ = context;
 142 }
 143
 144 HRESULT UIAutomationClient::Context::EventHandler::HandleAutomationEvent(
 145     IUIAutomationElement* sender,
 146     EVENTID eventId) {
 147   // Event handlers are invoked on an arbitrary thread in the MTA. Send the
 148   // event back to the main UI automation thread for processing.
 149   context_runner_->PostTask(
 150       FROM_HERE,
 151       base::Bind(&UIAutomationClient::Context::HandleAutomationEvent, context_,
 152                  base::win::ScopedComPtr<IUIAutomationElement>(sender),
 153                  eventId));
 154
 155   return S_OK;
 156 }
 157
 158 base::WeakPtr<UIAutomationClient::Context>
 159     UIAutomationClient::Context::Create() {
 160   Context* context = new Context();
 161   return context->weak_ptr_factory_.GetWeakPtr();
 162 }
 163
 164 void UIAutomationClient::Context::DeleteOnAutomationThread() {
 165   DCHECK(thread_checker_.CalledOnValidThread());
 166   delete this;
 167 }
 168
 169 UIAutomationClient::Context::Context() : weak_ptr_factory_(this) {}
 170
 171 UIAutomationClient::Context::~Context() {
 172   DCHECK(thread_checker_.CalledOnValidThread());
 173
 174   if (event_handler_.get()) {
 175     event_handler_ = NULL;
 176     HRESULT result = automation_->RemoveAllEventHandlers();
 177     LOG_IF(ERROR, FAILED(result)) << std::hex << result;
 178   }
 179 }
 180
 181 void UIAutomationClient::Context::Initialize(
 182     scoped_refptr<base::SingleThreadTaskRunner> client_runner,
 183     base::string16 class_name,
 184     base::string16 item_name,
 185     UIAutomationClient::InitializedCallback init_callback,
 186     UIAutomationClient::ResultCallback result_callback) {
 187   // This and all other methods must be called on the automation thread.
 188   DCHECK(!client_runner->BelongsToCurrentThread());
 189   // Bind the checker to this thread.
 190   thread_checker_.DetachFromThread();
 191   DCHECK(thread_checker_.CalledOnValidThread());
 192
 193   client_runner_ = client_runner;
 194   class_name_ = class_name;
 195   item_name_ = item_name;
 196   result_callback_ = result_callback;
 197
 198   HRESULT result = automation_.CreateInstance(CLSID_CUIAutomation, NULL,
 199                                               CLSCTX_INPROC_SERVER);
 200   if (FAILED(result) || !automation_.get())
 201     LOG(ERROR) << std::hex << result;
 202   else
 203     result = InstallWindowObserver();
 204
 205   // Tell the client that initialization is complete.
 206   client_runner_->PostTask(FROM_HERE, base::Bind(init_callback, result));
 207
 208   // Self-destruct if the overall operation failed.
 209   if (FAILED(result))
 210     delete this;
 211 }
 212
 213 // Installs the window observer.
 214 HRESULT UIAutomationClient::Context::InstallWindowObserver() {
 215   DCHECK(thread_checker_.CalledOnValidThread());
 216   DCHECK(automation_.get());
 217   DCHECK(!event_handler_.get());
 218
 219   HRESULT result = S_OK;
 220   base::win::ScopedComPtr<IUIAutomationElement> root_element;
 221   base::win::ScopedComPtr<IUIAutomationCacheRequest> cache_request;
 222
 223   // Observe the opening of all windows.
 224   result = automation_->GetRootElement(root_element.Receive());
 225   if (FAILED(result)) {
 226     LOG(ERROR) << std::hex << result;
 227     return result;
 228   }
 229
 230   // Cache Window class, HWND, and window pattern for opened windows.
 231   result = automation_->CreateCacheRequest(cache_request.Receive());
 232   if (FAILED(result)) {
 233     LOG(ERROR) << std::hex << result;
 234     return result;
 235   }
 236   cache_request->AddProperty(UIA_ClassNamePropertyId);
 237   cache_request->AddProperty(UIA_NativeWindowHandlePropertyId);
 238
 239   // Create the observer.
 240   CComObject<EventHandler>* event_handler_obj = NULL;
 241   result = CComObject<EventHandler>::CreateInstance(&event_handler_obj);
 242   if (FAILED(result)) {
 243     LOG(ERROR) << std::hex << result;
 244     return result;
 245   }
 246   event_handler_obj->Initialize(base::ThreadTaskRunnerHandle::Get(),
 247                                 weak_ptr_factory_.GetWeakPtr());
 248   base::win::ScopedComPtr<IUIAutomationEventHandler> event_handler(
 249       event_handler_obj);
 250
 251   result = automation_->AddAutomationEventHandler(
 252       UIA_Window_WindowOpenedEventId, root_element.get(), TreeScope_Descendants,
 253       cache_request.get(), event_handler.get());
 254
 255   if (FAILED(result)) {
 256     LOG(ERROR) << std::hex << result;
 257     return result;
 258   }
 259
 260   event_handler_ = event_handler;
 261   return S_OK;
 262 }
 263
 264 // Removes this instance's window observer.
 265 HRESULT UIAutomationClient::Context::RemoveWindowObserver() {
 266   DCHECK(thread_checker_.CalledOnValidThread());
 267   DCHECK(automation_.get());
 268   DCHECK(event_handler_.get());
 269
 270   HRESULT result = S_OK;
 271   base::win::ScopedComPtr<IUIAutomationElement> root_element;
 272
 273   // The opening of all windows are observed.
 274   result = automation_->GetRootElement(root_element.Receive());
 275   if (FAILED(result)) {
 276     LOG(ERROR) << std::hex << result;
 277     return result;
 278   }
 279
 280   result = automation_->RemoveAutomationEventHandler(
 281       UIA_Window_WindowOpenedEventId, root_element.get(), event_handler_.get());
 282   if (FAILED(result)) {
 283     LOG(ERROR) << std::hex << result;
 284     return result;
 285   }
 286
 287   event_handler_ = NULL;
 288   return S_OK;
 289 }
 290
 291 // Handles an automation event. If the event results in the processing for which
 292 // this context was created, the context self-destructs after posting the
 293 // results to the client.
 294 void UIAutomationClient::Context::HandleAutomationEvent(
 295     base::win::ScopedComPtr<IUIAutomationElement> sender,
 296     EVENTID eventId) {
 297   DCHECK(thread_checker_.CalledOnValidThread());
 298   if (eventId == UIA_Window_WindowOpenedEventId)
 299     HandleWindowOpen(sender);
 300 }
 301
 302 // Handles a WindowOpen event. If |window| is the one for which this instance is
 303 // waiting, it is processed and this instance self-destructs after posting the
 304 // results to the client.
 305 void UIAutomationClient::Context::HandleWindowOpen(
 306     const base::win::ScopedComPtr<IUIAutomationElement>& window) {
 307   DCHECK(thread_checker_.CalledOnValidThread());
 308   HRESULT hr = S_OK;
 309   base::win::ScopedVariant var;
 310
 311   hr = window->GetCachedPropertyValueEx(UIA_ClassNamePropertyId, TRUE,
 312                                         var.Receive());
 313   if (FAILED(hr)) {
 314     LOG(ERROR) << std::hex << hr;
 315     return;
 316   }
 317
 318   if (V_VT(&var) != VT_BSTR) {
 319     LOG(ERROR) << __FUNCTION__ << " class name is not a BSTR: " << V_VT(&var);
 320     return;
 321   }
 322
 323   base::string16 class_name(V_BSTR(&var));
 324
 325   // Window class names are atoms, which are case-insensitive.
 326   if (class_name.size() == class_name_.size() &&
 327       std::equal(class_name.begin(), class_name.end(), class_name_.begin(),
 328                  base::CaseInsensitiveCompare<wchar_t>())) {
 329     RemoveWindowObserver();
 330     ProcessWindow(window);
 331   }
 332 }
 333
 334 // Processes |window| by invoking the desired child item. If the item cannot be
 335 // found or invoked, an attempt is made to get a list of all invokable children.
 336 // The results are posted back to the client on |client_runner_|, and this
 337 // instance self-destructs.
 338 void UIAutomationClient::Context::ProcessWindow(
 339     const base::win::ScopedComPtr<IUIAutomationElement>& window) {
 340   DCHECK(thread_checker_.CalledOnValidThread());
 341
 342   HRESULT result = S_OK;
 343   std::vector<base::string16> choices;
 344   result = InvokeDesiredItem(window);
 345   if (FAILED(result)) {
 346     GetInvokableItems(window, &choices);
 347     CloseWindow(window);
 348   }
 349
 350   client_runner_->PostTask(FROM_HERE,
 351                            base::Bind(result_callback_, result, choices));
 352
 353   // Self-destruct since there's nothing more to be done here.
 354   delete this;
 355 }
 356
 357 // Invokes the desired child of |element|.
 358 HRESULT UIAutomationClient::Context::InvokeDesiredItem(
 359     const base::win::ScopedComPtr<IUIAutomationElement>& element) {
 360   DCHECK(thread_checker_.CalledOnValidThread());
 361
 362   HRESULT result = S_OK;
 363   base::win::ScopedVariant var;
 364   base::win::ScopedComPtr<IUIAutomationCondition> invokable_condition;
 365   base::win::ScopedComPtr<IUIAutomationCondition> item_name_condition;
 366   base::win::ScopedComPtr<IUIAutomationCondition> control_view_condition;
 367   base::win::ScopedComPtr<IUIAutomationCondition> condition;
 368   base::win::ScopedComPtr<IUIAutomationCacheRequest> cache_request;
 369   base::win::ScopedComPtr<IUIAutomationElement> target;
 370
 371   // Search for an invokable element named item_name.
 372   var.Set(true);
 373   result = automation_->CreatePropertyCondition(
 374       UIA_IsInvokePatternAvailablePropertyId,
 375       var,
 376       invokable_condition.Receive());
 377   var.Reset();
 378   if (FAILED(result)) {
 379     LOG(ERROR) << std::hex << result;
 380     return result;
 381   }
 382
 383   var.Set(item_name_.c_str());
 384   result = automation_->CreatePropertyCondition(UIA_NamePropertyId,
 385                                                 var,
 386                                                 item_name_condition.Receive());
 387   var.Reset();
 388   if (FAILED(result)) {
 389     LOG(ERROR) << std::hex << result;
 390     return result;
 391   }
 392
 393   result = automation_->get_ControlViewCondition(
 394       control_view_condition.Receive());
 395   if (FAILED(result)) {
 396     LOG(ERROR) << std::hex << result;
 397     return result;
 398   }
 399
 400   std::vector<IUIAutomationCondition*> conditions;
 401   conditions.push_back(invokable_condition.get());
 402   conditions.push_back(item_name_condition.get());
 403   conditions.push_back(control_view_condition.get());
 404   result = automation_->CreateAndConditionFromNativeArray(
 405       &conditions[0], conditions.size(), condition.Receive());
 406   if (FAILED(result)) {
 407     LOG(ERROR) << std::hex << result;
 408     return result;
 409   }
 410
 411   // Cache invokable pattern for the item.
 412   result = automation_->CreateCacheRequest(cache_request.Receive());
 413   if (FAILED(result)) {
 414     LOG(ERROR) << std::hex << result;
 415     return result;
 416   }
 417   cache_request->AddPattern(UIA_InvokePatternId);
 418
 419   result = element->FindFirstBuildCache(
 420       static_cast<TreeScope>(TreeScope_Children | TreeScope_Descendants),
 421       condition.get(), cache_request.get(), target.Receive());
 422   if (FAILED(result)) {
 423     LOG(ERROR) << std::hex << result;
 424     return result;
 425   }
 426
 427   // If the item was found, invoke it.
 428   if (!target.get()) {
 429     LOG(WARNING) << "Failed to find desired item to invoke.";
 430     return E_FAIL;
 431   }
 432
 433   base::win::ScopedComPtr<IUIAutomationInvokePattern> invoker;
 434   result = target->GetCachedPatternAs(UIA_InvokePatternId, invoker.iid(),
 435                                       invoker.ReceiveVoid());
 436   if (FAILED(result)) {
 437     LOG(ERROR) << std::hex << result;
 438     return result;
 439   }
 440
 441   result = invoker->Invoke();
 442   if (FAILED(result)) {
 443     LOG(ERROR) << std::hex << result;
 444     return result;
 445   }
 446
 447   return S_OK;
 448 }
 449
 450 // Populates |choices| with the names of all invokable children of |element|.
 451 HRESULT UIAutomationClient::Context::GetInvokableItems(
 452     const base::win::ScopedComPtr<IUIAutomationElement>& element,
 453     std::vector<base::string16>* choices) {
 454   DCHECK(choices);
 455   DCHECK(thread_checker_.CalledOnValidThread());
 456
 457   HRESULT result = S_OK;
 458   base::win::ScopedVariant var;
 459   base::win::ScopedComPtr<IUIAutomationCondition> invokable_condition;
 460   base::win::ScopedComPtr<IUIAutomationCondition> control_view_condition;
 461   base::win::ScopedComPtr<IUIAutomationCondition> condition;
 462   base::win::ScopedComPtr<IUIAutomationCacheRequest> cache_request;
 463   base::win::ScopedComPtr<IUIAutomationElementArray> element_array;
 464   base::win::ScopedComPtr<IUIAutomationElement> child_element;
 465
 466   // Search for all invokable elements.
 467   var.Set(true);
 468   result = automation_->CreatePropertyCondition(
 469       UIA_IsInvokePatternAvailablePropertyId,
 470       var,
 471       invokable_condition.Receive());
 472   var.Reset();
 473   if (FAILED(result)) {
 474     LOG(ERROR) << std::hex << result;
 475     return result;
 476   }
 477
 478   result = automation_->get_ControlViewCondition(
 479       control_view_condition.Receive());
 480   if (FAILED(result)) {
 481     LOG(ERROR) << std::hex << result;
 482     return result;
 483   }
 484
 485   result = automation_->CreateAndCondition(invokable_condition.get(),
 486                                            control_view_condition.get(),
 487                                            condition.Receive());
 488   if (FAILED(result)) {
 489     LOG(ERROR) << std::hex << result;
 490     return result;
 491   }
 492
 493   // Cache item names.
 494   result = automation_->CreateCacheRequest(cache_request.Receive());
 495   if (FAILED(result)) {
 496     LOG(ERROR) << std::hex << result;
 497     return result;
 498   }
 499   cache_request->AddProperty(UIA_NamePropertyId);
 500
 501   result = element->FindAllBuildCache(
 502       static_cast<TreeScope>(TreeScope_Children | TreeScope_Descendants),
 503       condition.get(), cache_request.get(), element_array.Receive());
 504   if (FAILED(result)) {
 505     LOG(ERROR) << std::hex << result;
 506     return result;
 507   }
 508
 509   if (!element_array.get()) {
 510     LOG(ERROR) << "The window may have vanished.";
 511     return S_OK;
 512   }
 513
 514   int num_elements = 0;
 515   result = element_array->get_Length(&num_elements);
 516   if (FAILED(result)) {
 517     LOG(ERROR) << std::hex << result;
 518     return result;
 519   }
 520
 521   choices->clear();
 522   choices->reserve(num_elements);
 523   for (int i = 0; i < num_elements; ++i) {
 524     child_element.Release();
 525     result = element_array->GetElement(i, child_element.Receive());
 526     if (FAILED(result)) {
 527       LOG(ERROR) << std::hex << result;
 528       continue;
 529     }
 530     result = child_element->GetCachedPropertyValueEx(UIA_NamePropertyId, TRUE,
 531                                                      var.Receive());
 532     if (FAILED(result)) {
 533       LOG(ERROR) << std::hex << result;
 534       continue;
 535     }
 536     if (V_VT(&var) != VT_BSTR) {
 537       LOG(ERROR) << __FUNCTION__ << " name is not a BSTR: " << V_VT(&var);
 538       continue;
 539     }
 540     choices->push_back(base::string16(V_BSTR(&var)));
 541     var.Reset();
 542   }
 543
 544   return result;
 545 }
 546
 547 // Closes the element |window| by sending it an escape key.
 548 void UIAutomationClient::Context::CloseWindow(
 549     const base::win::ScopedComPtr<IUIAutomationElement>& window) {
 550   DCHECK(thread_checker_.CalledOnValidThread());
 551
 552   // It's tempting to get the Window pattern from |window| and invoke its Close
 553   // method. Unfortunately, this doesn't work. Sending an escape key does the
 554   // trick, though.
 555   HRESULT result = S_OK;
 556   base::win::ScopedVariant var;
 557
 558   result = window->GetCachedPropertyValueEx(
 559       UIA_NativeWindowHandlePropertyId,
 560       TRUE,
 561       var.Receive());
 562   if (FAILED(result)) {
 563     LOG(ERROR) << std::hex << result;
 564     return;
 565   }
 566
 567   if (V_VT(&var) != VT_I4) {
 568     LOG(ERROR) << __FUNCTION__
 569                << " window handle is not an int: " << V_VT(&var);
 570     return;
 571   }
 572
 573   HWND handle = reinterpret_cast<HWND>(V_I4(&var));
 574
 575   uint32 scan_code = MapVirtualKey(VK_ESCAPE, MAPVK_VK_TO_VSC);
 576   PostMessage(handle, WM_KEYDOWN, VK_ESCAPE,
 577               MAKELPARAM(1, scan_code));
 578   PostMessage(handle, WM_KEYUP, VK_ESCAPE,
 579               MAKELPARAM(1, scan_code | KF_REPEAT | KF_UP));
 580 }
 581
 582 UIAutomationClient::UIAutomationClient()
 583     : automation_thread_("UIAutomation") {}
 584
 585 UIAutomationClient::~UIAutomationClient() {
 586   DCHECK(thread_checker_.CalledOnValidThread());
 587
 588   // context_ is still valid when the caller destroys the instance before the
 589   // callback(s) have fired. In this case, delete the context on the automation
 590   // thread before joining with it.
 591   automation_thread_.message_loop()->PostTask(
 592       FROM_HERE,
 593       base::Bind(&UIAutomationClient::Context::DeleteOnAutomationThread,
 594                  context_));
 595 }
 596
 597 void UIAutomationClient::Begin(const wchar_t* class_name,
 598                                const base::string16& item_name,
 599                                const InitializedCallback& init_callback,
 600                                const ResultCallback& result_callback) {
 601   DCHECK(thread_checker_.CalledOnValidThread());
 602   DCHECK_EQ(context_.get(), static_cast<Context*>(NULL));
 603
 604   // Start the automation thread and initialize our automation client on it.
 605   context_ = Context::Create();
 606   automation_thread_.init_com_with_mta(true);
 607   automation_thread_.Start();
 608   automation_thread_.message_loop()->PostTask(
 609       FROM_HERE,
 610       base::Bind(&UIAutomationClient::Context::Initialize,
 611                  context_,
 612                  base::ThreadTaskRunnerHandle::Get(),
 613                  base::string16(class_name),
 614                  item_name,
 615                  init_callback,
 616                  result_callback));
 617 }
 618
 619 }  // namespace internal
 620 }  // namespace win8