UiPath doesn't recognize children/ fake UIElements

274f2b6343dcbfc5cee00ced8 · November 13, 2025, 4:41pm

Hi,
I was given the task to see if our app can be automated with UiPath. Before getting into the real deal I made a small test program in Delphi. In this program i made a “FakeGrid” that is a descendant from TWinControl, and exposes IRawElementProviderSimple, IValueProvider, IGridProvider, IRawElementProviderFragment,IRawElementProviderFragmentRoot. Then there is an object list of “FakeGridItems” that are just TInterfacedObject and expose IGridItemProvider,IInvokeProvider, IRawElementProviderSimple,IValueProvider, IRawElementProviderFragment. these items would be the cells of a grid.

I implemented all the methods from the providers as simple as i could. When exploring/experimenting with Inspect.exe, i can see the children from the grid and the interfaces return their respective “boundingrectangles”. when i try to simulate a click with UiPath the cells are not detected and using the Find Children activity it returns 0 children. also UIExplorer doesnt seem to see the children.

274f2b6343dcbfc5cee00ced8 · November 13, 2025, 4:46pm

unit MainForm;

interface

uses
  Winapi.Windows, Winapi.Messages, System.SysUtils, System.Variants, System.Classes, Vcl.Graphics,
  Vcl.Controls, Vcl.Forms, Vcl.Dialogs, clCell, Vcl.ExtCtrls, Vcl.StdCtrls,ComObj,System.Generics.Collections,
  ActiveX,
  utButton,
  clGrids,
  utTypes,
  UIAutomationCore_TLB, utSplitterPanel, utScrollingCtrl, utSysUtils,
  utCmpContainer, utComponentTree, Vcl.ComCtrls, clTable, clFields;

type

  TclAutoCell = class(TclCell, IRawElementProviderSimple, IValueProvider{,IGridItemProvider}{, ISelectionProvider})
    public constructor Create(aOwner: TComponent); override;
    public destructor Destroy; override;
    //
    procedure WMGetObject(var Message: TMessage); message WM_GETOBJECT;
    //
    private FRawElementProviderSimple : IRawElementProviderSimple;
    //
    // IRawElementProviderSimple
    function Get_ProviderOptions(out pRetVal: ProviderOptions): HResult; stdcall;
    function GetPatternProvider(patternId: SYSINT; out pRetVal: IUnknown): HResult; stdcall;
    function GetPropertyValue(propertyId: SYSINT; out pRetVal: OleVariant): HResult; stdcall;
    function Get_HostRawElementProvider(out pRetVal: IRawElementProviderSimple): HResult; stdcall;
    // IValueProvider
    private FValue: String;
    function SetValue(val: PWideChar): HResult; stdcall;
    function Get_Value(out pRetVal: WideString): HResult; stdcall;
    function Get_IsReadOnly(out pRetVal: Integer): HResult; stdcall;
    // ISelectionProvider
    {
    function GetSelection(out pRetVal: PSafeArray): HResult; stdcall;
    function Get_CanSelectMultiple(out pRetVal: Integer): HResult; stdcall;
    function Get_IsSelectionRequired(out pRetVal: Integer): HResult; stdcall;
    }
  end;

type
  TclAutoButton = class(TButton,IRawElementProviderSimple, IValueProvider)
    public constructor Create(aOwner: TComponent); override;
    public destructor Destroy; override;
    //
    procedure WMGetObject(var Message: TMessage); message WM_GETOBJECT;
    //
    private FRawElementProviderSimple : IRawElementProviderSimple;
    //
     //IRawElementProviderSimple
    function Get_ProviderOptions(out pRetVal: ProviderOptions): HResult; stdcall;
    function GetPatternProvider(patternId: SYSINT; out pRetVal: IUnknown): HResult; stdcall;
    function GetPropertyValue(propertyId: SYSINT; out pRetVal: OleVariant): HResult; stdcall;
    function Get_HostRawElementProvider(out pRetVal: IRawElementProviderSimple): HResult; stdcall;
    // IValueProvider
    private FValue: String;
    function SetValue(val: PWideChar): HResult; stdcall;
    function Get_Value(out pRetVal: WideString): HResult; stdcall;
    function Get_IsReadOnly(out pRetVal: Integer): HResult; stdcall;
  end;

type
  TclAutoFakeGridItem = class(TInterfacedObject,IGridItemProvider,IInvokeProvider, IRawElementProviderSimple,IValueProvider, IRawElementProviderFragment)
    public constructor Create(aOwner: TWinControl; aFragmentRoot: IRawElementProviderFragmentRoot;aRow,aColumn: TutInteger);
    public destructor Destroy; override;
    //
    //procedure WMGetObject(var Message: TMessage); message WM_GETOBJECT;
    //
    private FRawElementProviderSimple: IRawElementProviderSimple;
    //
    private FLeft: Integer;
    private FTop: Integer;
    private FWidth: Integer;
    private FHeight: Integer;
    //
    published property Left: Integer read FLeft write FLeft;
    published property Top: Integer read FTop write FTop;
    published property Width: Integer read FWidth write FWidth;
    published property Height: Integer read FHeight write FHeight;
    //
    private FRow: TutInteger;
    private FColumn: TutInteger;
    //
    private FRoot: IRawElementProviderFragmentRoot;
    //
    private FIsFocused: Boolean;
    public property IsFocused: Boolean read FIsFocused write FIsFocused;
    //
    private FOwner: TWinControl;
    protected function GetHandle: HWND;
    public property Handle: HWND read GetHandle;
    // IRawElementProviderSimple
    function Get_ProviderOptions(out pRetVal: ProviderOptions): HResult; stdcall;
    function GetPatternProvider(patternId: SYSINT; out pRetVal: IUnknown): HResult; stdcall;
    function GetPropertyValue(propertyId: SYSINT; out pRetVal: OleVariant): HResult; stdcall;
    function Get_HostRawElementProvider(out pRetVal: IRawElementProviderSimple): HResult; stdcall;
    // IValueProvider
    private FValue: String;
    function SetValue(val: PWideChar): HResult; stdcall;
    function Get_Value(out pRetVal: WideString): HResult; stdcall;
    function Get_IsReadOnly(out pRetVal: Integer): HResult; stdcall;
    //IGridItemProvider
    function Get_row(out pRetVal: SYSINT): HResult; stdcall;
    function Get_column(out pRetVal: SYSINT): HResult; stdcall;
    function Get_RowSpan(out pRetVal: SYSINT): HResult; stdcall;
    function Get_ColumnSpan(out pRetVal: SYSINT): HResult; stdcall;
    function Get_ContainingGrid(out pRetVal: IRawElementProviderSimple): HResult; stdcall;
    //IRawElementProviderFragment
    function Navigate(direction: NavigateDirection; out pRetVal: IRawElementProviderFragment): HResult; stdcall;
    function GetRuntimeId(out pRetVal: PSafeArray): HResult; stdcall;
    function get_BoundingRectangle(out pRetVal: UiaRect): HResult; stdcall;
    function GetEmbeddedFragmentRoots(out pRetVal: PSafeArray): HResult; stdcall;
    function SetFocus: HResult; stdcall;
    function Get_FragmentRoot(out pRetVal: IRawElementProviderFragmentRoot): HResult; stdcall;
    //
    function Invoke: HResult; stdcall;
  end;

type
  TclAutoFakeGrid = class(TPanel,IRawElementProviderSimple, IValueProvider, IGridProvider, IRawElementProviderFragment,IRawElementProviderFragmentRoot)
    public constructor Create(aOwner: TComponent); override;
    public destructor Destroy; override;
    //
    //
    procedure WMGetObject(var Message: TMessage); message WM_GETOBJECT;
    //
    private FChildItemList: TObjectList<TclAutoFakeGridItem>;
//    public property ChildItemList: TclAutoFakeItemList read FChildItemList write FChildItemList;
    public procedure UnfocusAll;

    private FRawElementProviderSimple: IRawElementProviderSimple;
    //private FIRawElementProviderFragmentRoot: IRawElementProviderFragmentRoot;
    //
    //IRawElementProviderSimple
    function Get_ProviderOptions(out pRetVal: ProviderOptions): HResult; stdcall;
    function GetPatternProvider(patternId: SYSINT; out pRetVal: IUnknown): HResult; stdcall;
    function GetPropertyValue(propertyId: SYSINT; out pRetVal: OleVariant): HResult; stdcall;
    function Get_HostRawElementProvider(out pRetVal: IRawElementProviderSimple): HResult; stdcall;
    // IValueProvider
    private FValue: String;
    function SetValue(val: PWideChar): HResult; stdcall;
    function Get_Value(out pRetVal: WideString): HResult; stdcall;
    function Get_IsReadOnly(out pRetVal: Integer): HResult; stdcall;
    //IGridProvider
    function GetItem(row: SYSINT; column: SYSINT; out pRetVal: IRawElementProviderSimple): HResult; stdcall;
    function Get_RowCount(out pRetVal: SYSINT): HResult; stdcall;
    function Get_ColumnCount(out pRetVal: SYSINT): HResult; stdcall;
    //IRawElementProviderFragmentRoot
    function ElementProviderFromPoint(x: Double; y: Double; out pRetVal: IRawElementProviderFragment): HResult; stdcall;
    function GetFocus(out pRetVal: IRawElementProviderFragment): HResult; stdcall;
    //IRawElementProviderFragment
    function Navigate(direction: NavigateDirection; out pRetVal: IRawElementProviderFragment): HResult; stdcall;
    function GetRuntimeId(out pRetVal: PSafeArray): HResult; stdcall;
    function get_BoundingRectangle(out pRetVal: UiaRect): HResult; stdcall;
    function GetEmbeddedFragmentRoots(out pRetVal: PSafeArray): HResult; stdcall;
    function SetFocus: HResult; stdcall;
    function Get_FragmentRoot(out pRetVal: IRawElementProviderFragmentRoot): HResult; stdcall;
  end;

type
  TForm1 = class(TForm)
    utSplitterPanel1Spn: TutSplitterPanel;
    NormalInp: TutInnerPanel;
    N2520701127Spt: TutSplitArea;
    AutomationInp: TutInnerPanel;
    Label1: TLabel;
    Label2: TLabel;
    Button1: TButton;
    Edit1: TEdit;
    Item: TclAutoFakeGridItem;
//    FFakeGrid: TclAutoFakeGrid;
    ListView1: TListView;
    Panel1: TPanel;
    Panel2: TPanel;
    procedure FormCreate(Sender: TObject);
    procedure Button1Click(Sender: TObject);
    procedure AutoButtonClick(Sender: TObject);
    procedure GridClick(Sender: TObject);
    procedure ListEnter(Sender: TObject);
    procedure MouseMovelist(Sender: TObject;Shift: TShiftState; x,y: Integer);
  private
    { Private-Deklarationen }
    FFakeGrid: TclAutoFakeGrid;
  public
    { Public-Deklarationen }
  end;

const UiaRootObjectId = $FFFFFFE7;

var
  Form1: TForm1;

implementation

{$R *.dfm}

{ TclAutoCell }

constructor TclAutoCell.Create(aOwner: TComponent);
begin
  inherited Create(aOwner);
  FValue := 'Das ist der Wert vom TclAutoCell';
end;

destructor TclAutoCell.Destroy;
begin
  inherited Destroy;
end;

function TclAutoCell.GetPatternProvider(patternId: SYSINT; out pRetVal: IInterface): HResult;
begin
  Result := S_OK;
  pRetval := nil;
  if (patternID = UIA_ValuePatternID) {or (patternID = UIA_SelectionPatternId)} then
  begin
    pRetVal := self;
  end;
end;

function TclAutoCell.GetPropertyValue(propertyId: SYSINT; out pRetVal: OleVariant): HResult;
begin
  if(propertyId = UIA_ClassNamePropertyId) then
  begin
    TVarData(pRetVal).VType := varOleStr;
    TVarData(pRetVal).VOleStr := pWideChar('Das ist die Klasse vom TclAutoCell');
    result := S_OK;
  end
  else if(propertyId = UIA_NamePropertyId) then
  begin
    TVarData(pRetVal).VType := varOleStr;
    TVarData(pRetVal).VOleStr := pWideChar('Das ist der Name vom TclAutoCell');
    result := S_OK;
  end
  else if(propertyId = UIA_AutomationIdPropertyId) then
  begin
    TVarData(pRetVal).VType := varOleStr;
    TVarData(pRetVal).VOleStr := pWideChar({'Das ist die ID vom TclAutoCell'}'3045489874');
    result := S_OK;
  end
  else if(propertyId = UIA_ControlTypePropertyId) then
  begin
    TVarData(pRetVal).VType := varInteger;
    TVarData(pRetVal).VInteger := UIA_CustomControlTypeId;
    result := S_OK;
  end
  else if (propertyId = UIA_FrameworkIdPropertyId) then
  begin
    pRetVal := 'Win32';
    result := S_OK;
  end
  else
  begin
    result := S_FALSE;
  end;
end;

{
function TclAutoCell.GetSelection(out pRetVal: PSafeArray): HResult;
begin

end;

function TclAutoCell.Get_CanSelectMultiple(out pRetVal: Integer): HResult;
begin

end;

function TclAutoCell.Get_IsSelectionRequired(out pRetVal: Integer): HResult;
begin

end;
}
//
//function TclAutoCell.Get_column(out pRetVal: SYSINT): HResult;
//begin
//  pRetVal := 5;
//  Result := S_OK;
//end;
//
//function TclAutoCell.Get_ColumnSpan(out pRetVal: SYSINT): HResult;
//begin
//  pRetVal := 1;
//  Result := S_OK;
//end;
//
//function TclAutoCell.Get_ContainingGrid(
//  out pRetVal: IRawElementProviderSimple): HResult;
//begin
//  pRetVal := Owner as IRawElementProviderSimple;
//  Result := S_OK;
//end;

function TclAutoCell.Get_HostRawElementProvider(out pRetVal: IRawElementProviderSimple): HResult;
begin
  Result := UiaHostProviderFromHwnd(self.Handle, pRetVal);
end;

function TclAutoCell.Get_IsReadOnly(out pRetVal: Integer): HResult;
begin
  pRetVal := 0;   // Maybe?
  Result := S_OK;
end;

function TclAutoCell.Get_ProviderOptions(out pRetVal: ProviderOptions): HResult;
begin
  pRetVal := ProviderOptions_ServerSideProvider;
  Result := S_OK;
end;
//
//function TclAutoCell.Get_row(out pRetVal: SYSINT): HResult;
//begin
//  pRetVal := 4;
//  Result := S_OK;
//end;
//
//function TclAutoCell.Get_RowSpan(out pRetVal: SYSINT): HResult;
//begin
//  pRetVal := 1;
//  Result := S_OK;
//end;

function TclAutoCell.Get_Value(out pRetVal: WideString): HResult;
begin
  pRetVal := FValue;
  Result := S_OK;
end;

function TclAutoCell.SetValue(val: PWideChar): HResult;
begin
  FValue := val;
  Result := S_OK;
end;

procedure TclAutoCell.WMGetObject(var Message: TMessage);
const
  lOBJID_CLIENT = -4;
begin
  if (Message.Msg = WM_GETOBJECT) and (Message.LParam = -25) then // probably should include this, not tested (https://learn.microsoft.com/en-us/windows/win32/winauto/wm-getobject)
  begin
    QueryInterface(IID_IRawElementProviderSimple, FRawElementProviderSimple);
    message.Result := UiaReturnRawElementProvider(self.Handle, Message.WParam, Message.LParam, FRawElementProviderSimple);
  end
  else
  begin
    Message.Result := DefWindowProc(self.Handle, Message.Msg, Message.WParam, Message.LParam);
  end;
end;

procedure TForm1.AutoButtonClick;
var
  lRoot: IRawElementProviderFragmentRoot;
  lItem: TclAutoFakeGridItem;
  lRect: TRect;
begin
  GetWindowRect(Handle,lRect);
  //FFakeGrid.QueryInterface(IRawElementProviderFragmentRoot,lRoot);
  Supports(FFakeGrid,IRawElementProviderFragmentRoot,lRoot);
  lItem := TclAutoFakeGridItem.Create(FFakeGrid,lRoot,0,0);
  lItem.Left := 100;//lRect.Left;
  lItem.Top := 100;//lRect.Top;
  lItem.Width := 10;
  lItem.Height := 10;
  lItem.IsFocused := True;
  //
  FFakeGrid.FChildItemList.Add(lItem);
  lRoot._AddRef;
  lItem := TclAutoFakeGridItem.Create(FFakeGrid,lRoot,0,1);
  lItem.Left := 10;
  lItem.Top := 1;
  lItem.Width := 10;
  lItem.Height := 10;
  FFakeGrid.FChildItemList.Add(lItem);
  lRoot._AddRef;
  lItem := TclAutoFakeGridItem.Create(FFakeGrid,lRoot,0,2);
  lItem.Left := 20;
  lItem.Top := 10;
  lItem.Width := 10;
  lItem.Height := 10;
  FFakeGrid.FChildItemList.Add(lItem);
  lRoot._AddRef;
  lItem := TclAutoFakeGridItem.Create(FFakeGrid,lRoot,0,0);
end;

procedure TForm1.Button1Click(Sender: TObject);
begin
  Edit1.Text := 'NormalButtonClicked';

end;

procedure TForm1.FormCreate(Sender: TObject);
var
  lNormalCell: TclCell;
  lAutoCell: TclAutoCell;
  lAutoButton: TclAutoButton;
  lGrid: TclAutoFakeGrid;
  //lFakeList: TclAutoFakeList;
  lListItem: TListItem;
begin
  lNormalCell := TclCell.Create(Self);
  lNormalCell.Left := 0;
  lNormalCell.Top := 25;
  lNormalCell.Width := 100;
  lNormalCell.Height := 20;
  lNormalCell.Parent := NormalInp;
  //
  lAutoCell := TclAutoCell.Create(Self);
  lAutoCell.Left := 0;
  lAutoCell.Top := 25;
  lAutoCell.Width := 100;
  lAutoCell.Height := 20;
  lAutoCell.Parent := AutomationInp;
  //
  lAutoButton := TclAutoButton.Create(Self);
  lAutoButton.Left := 80;
  lAutoButton.Top := 80;
  lAutoButton.Width := 10;
  lAutoButton.Height := 50;
  lAutoButton.Parent := AutomationInp;
  lAutoButton.OnClick := AutoButtonClick;
  //
  FFakeGrid := TclAutoFakeGrid.Create(Self);
  FFakeGrid.Caption := 'AutoFakegrid';
  FFakeGrid.Left := 150;
  FFakeGrid.Top := 100;
  FFakeGrid.Width := 80;
  FFakeGrid.Height := 50;
  FFakeGrid.Parent := AutomationInp;
  //FFakeGrid.OnClick := GridClick;
  lListItem := ListView1.Items[0];
  lListItem.Left := 200;
  lListItem.Top := 200;

  //ListView1.OnMouseEnter := ListEnter;
  Item := TclAutoFakegridItem.Create(Self,nil,4,4);
  Item.Left := 200;
  item.Top := 100;
  Item.Width := 20;
  Item.Height := 50;
  ListView1.OnMouseMove := MouseMovelist;
end;

procedure TForm1.GridClick(Sender: TObject);
var
  lFakeItem: TclAutoFakeGridItem;
begin
  Edit1.Text := 'GridClicked';
end;


procedure TForm1.ListEnter(Sender: TObject);
begin
  Edit1.Text := 'ListEnter';
end;

procedure TForm1.MouseMovelist(Sender: TObject; Shift: TShiftState; x,
  y: Integer);
begin
  MouseMove(Shift,x,y);
end;

{ TclAutoButton }

constructor TclAutoButton.Create(aOwner: TComponent);
begin
  inherited Create(aOwner);
  FValue := 'Das ist der Wert vom TclAutoButton';
end;

destructor TclAutoButton.Destroy;
begin
  inherited Destroy;
end;

function TclAutoButton.GetPatternProvider(patternId: SYSINT;
  out pRetVal: IInterface): HResult;
begin
  pRetval := nil;
  if (patternID = UIA_ValuePatternID) {or (patternID = UIA_SelectionPatternId)} then
  begin
    pRetVal := self;
    Result := S_OK
  end
  else
  begin
    Result := S_OK;
  end;
end;

function TclAutoButton.GetPropertyValue(propertyId: SYSINT;
  out pRetVal: OleVariant): HResult;
begin
  if(propertyId = UIA_ClassNamePropertyId) then
  begin
    TVarData(pRetVal).VType := varOleStr;
    TVarData(pRetVal).VOleStr := pWideChar('Das ist die Klasse vom TclAutoButton');
    result := S_OK;
  end
  else if(propertyId = UIA_NamePropertyId) then
  begin
    TVarData(pRetVal).VType := varOleStr;
    TVarData(pRetVal).VOleStr := pWideChar('Das ist der Name vom TclAutoButton');
    result := S_OK;
  end
  else if(propertyId = UIA_AutomationIdPropertyId) then
  begin
    TVarData(pRetVal).VType := varOleStr;
    TVarData(pRetVal).VOleStr := pWideChar('3045594506');
    result := S_OK;
  end
  else if(propertyId = UIA_ControlTypePropertyId) then
  begin
    TVarData(pRetVal).VType := varInteger;
    TVarData(pRetVal).VInteger := UIA_ButtonControlTypeId;
    result := S_OK;
  end
  else if (propertyId = UIA_IsControlElementPropertyId) then
  begin
    pRetVal := True;
    result := S_OK;
  end
  else if (propertyId = UIA_IsContentElementPropertyId) then
  begin
    pRetVal := True;
    result := S_OK;
  end
  else if (propertyId = UIA_FrameworkIdPropertyId) then
  begin
    pRetVal := 'Win32';
    result := S_OK;
  end
  else
  begin
    pRetVal := VT_EMPTY;
    result := S_OK;
  end;
end;

function TclAutoButton.Get_HostRawElementProvider(
  out pRetVal: IRawElementProviderSimple): HResult;
begin
  result := UiaHostProviderFromHwnd(self.Handle, pRetVal);
end;

function TclAutoButton.Get_IsReadOnly(out pRetVal: Integer): HResult;
begin
  pRetVal := 0;   // Maybe?
  Result := S_OK;
end;

function TclAutoButton.Get_ProviderOptions(
  out pRetVal: ProviderOptions): HResult;
begin
  pRetVal := ProviderOptions_ServerSideProvider;
  Result := S_OK;
end;

function TclAutoButton.Get_Value(out pRetVal: WideString): HResult;
begin
  pRetVal := FValue;
  Result := S_OK;
end;

function TclAutoButton.SetValue(val: PWideChar): HResult;
begin
  FValue := val;
  Result := S_OK;
end;

procedure TclAutoButton.WMGetObject(var Message: TMessage);
const
  lOBJID_CLIENT = -4;
begin
  if (Message.Msg = WM_GETOBJECT) and ((Message.LParam = {UiaRootObjectId}-25) {or (Message.LParam = -4)}) then // probably should include this, not tested (https://learn.microsoft.com/en-us/windows/win32/winauto/wm-getobject)
  begin
    QueryInterface(IID_IRawElementProviderSimple, FRawElementProviderSimple);
    message.Result := UiaReturnRawElementProvider(self.Handle, Message.WParam, Message.LParam, FRawElementProviderSimple);
  end
  else
  begin
    Message.Result := DefWindowProc(self.Handle, Message.Msg, Message.WParam, Message.LParam);
  end;
end;

{ TclAutoFakeGrid }

constructor TclAutoFakeGrid.Create(aOwner: TComponent);
begin
  inherited Create(aOwner);
  FValue := 'Das ist der Wert vom TclAutoFakeGrid';
  FChildItemList := TObjectList<TclAutoFakeGridItem>.Create(True);
  //
end;

destructor TclAutoFakeGrid.Destroy;
begin
  inherited Destroy;
end;

function TclAutoFakeGrid.ElementProviderFromPoint(x, y: Double;
  out pRetVal: IRawElementProviderFragment): HResult;
var
  lRect: TRect;
  lChildItem: TclAutoFakeGridItem;
begin
  GetWindowRect(Handle,lRect);
  if (x > lRect.Left) and (x < lRect.Left + 40) then  //size of grid halved
  begin
    lChildItem := FChildItemList[0];
    Supports(lChildItem,{IID_}IRawElementProviderFragment,pRetVal);
    //OutputDebugString(PChar('elementproviderfrompoiont-child'));
  end
  else
  begin
    //Supports({FChildItemList}Self,IRawElementProviderFragment,pRetVal);
    pRetVal := Self as IRawElementProviderFragment;
    pRetVal._AddRef;
    //pRetVal := FChildItemList as IRawElementProviderFragment;
    //pRetVal._AddRef;
    //OutputDebugString(PChar('elementproviderfrompoiont-grid'));
  end;
  Result := S_OK;
end;

function TclAutoFakeGrid.GetEmbeddedFragmentRoots(
  out pRetVal: PSafeArray): HResult;
begin
  pRetval := nil;
  Result := S_OK;
  //OutputDebugString(PChar('GetEmbeddedFragmentRoots-grid'));
end;

function TclAutoFakeGrid.GetFocus(
  out pRetVal: IRawElementProviderFragment): HResult;
var
  lChildItem: TclAutoFakeGridItem;
  I: Integer;
begin
  for I := 0 to FChildItemList.Count - 1 do
  begin
    lChildItem := FChildItemList[I];
    if lChildItem.IsFocused then
    begin
      Break;
    end;
  end;
  //lChildItem := FChildItemList[0];
  Supports(lChildItem,IID_IRawElementProviderFragment,pRetVal);
  Result := S_OK;
end;

function TclAutoFakeGrid.GetItem(row, column: SYSINT;
  out pRetVal: IRawElementProviderSimple): HResult;
var
  lChildItem: TclAutoFakeGridItem;
begin
  lChildItem := FChildItemList[0];
  if (row = 0) and (column = 0) then
  begin
    Supports(lChildItem,IID_IRawElementProviderSimple,pRetVal);
    Result := S_OK;
    //Edit.Text := Edit.Text + 'getitem-called';
  end
  else if (row = 0) and (column = 1) then
  begin
    lChildItem := FChildItemList[1];
    Supports(lChildItem,IID_IRawElementProviderSimple,pRetVal);
    Result := S_OK;
    //Edit.Text := Edit.Text + 'getitem2-called';
  end
  else
  begin
    pRetVal := nil;
    Result := S_OK;
  end;
end;

function TclAutoFakeGrid.GetPatternProvider(patternId: SYSINT;
  out pRetVal: IUnknown): HResult;
begin
  if (patternID = UIA_GridPatternId) then
  begin
    //self.QueryInterface(IGridProvider,pRetVal);
    pRetVal := Self as IGridProvider;
    pRetVal._AddRef;
    Result := S_OK;
  end
  else if (patternID = UIA_ValuePatternId) then
  begin
    pRetVal := self as  IValueProvider;
    pRetVal._AddRef;
    //self.QueryInterface(IValueProvider,pRetVal);
    Result := S_OK;
  end
  else
  begin
    pRetval := nil;
    Result := S_OK;
    //OutputDebugString(PChar('GetPatternProvider-else-'+IntToStr(patternID)));
  end;
  //Edit.Text := Edit.Text + 'GetPatternProvider-called';
end;

function TclAutoFakeGrid.GetPropertyValue(propertyId: SYSINT;
  out pRetVal: OleVariant): HResult;
begin
  if(propertyId = UIA_ClassNamePropertyId) then
  begin
//    TVarData(pRetVal).VType := varOleStr;
//    TVarData(pRetVal).VOleStr := pWideChar('Das ist die Klasse vom TclAutoFakeGrid');
    pRetVal := 'Das ist die Klasse vom TclAutoFakeGrid';
    result := S_OK;
  end
  else if(propertyId = UIA_NamePropertyId) then
  begin
    //TVarData(pRetVal).VType := varOleStr;
    //TVarData(pRetVal).VOleStr := pWideChar('Das ist der Name vom TclAutoFakeGrid');
    pRetVal := 'Das ist der Name vom TclAutoFakeGrid';
    result := S_OK;
  end
  else if(propertyId = UIA_AutomationIdPropertyId) then
  begin
    //TVarData(pRetVal).VType := varOleStr;
    //TVarData(pRetVal).VOleStr := pWideChar('3045715169');
    pRetVal:= '3045715169';
    result := S_OK;
  end
  else if (propertyId = UIA_IsControlElementPropertyId) then
  begin
    //TVarData(pRetVal).VType := varBoolean;
    //TVarData(pRetVal).VBoolean := True;
    pRetVal := True;
    Result := S_OK;
  end
  else if (propertyId = UIA_IsContentElementPropertyId) then
  begin
    //TVarData(pRetVal).VType := varBoolean;
    //TVarData(pRetVal).VBoolean := True;
    pRetVal := True;
    Result := S_OK;
  end
  else if(propertyId = UIA_ControlTypePropertyId) then
  begin
    //TVarData(pRetVal).VType := varInteger;
    //TVarData(pRetVal).VInteger := UIA_DataGridControlTypeId;
    pRetVal := UIA_DataGridControlTypeId;
    result := S_OK;
  end
  else if (propertyId = UIA_FrameworkIdPropertyId) then
  begin
    //TVarData(pRetVal).VType := varOleStr;
    //TVarData(pRetVal).VOleStr := pWideChar('UIAutomation');
    pRetVal := 'Win32';
    result := S_OK;
  end
  else if (propertyId = UIA_LocalizedControlTypePropertyId) then
  begin
    //TVarData(pRetVal).VType := varOleStr;
    //TVarData(pRetVal).VOleStr := pWideChar('Grid-Auto');
    pRetVal := 'Grid-Auto';
    result := S_OK;
  end
  else if (propertyId = UIA_BoundingRectanglePropertyId) then
  begin
    //TVarData(pRetVal).VType := VT_ARRAY;
    //TVarData(pRetVal).VArray := VarArrayOf([150, 100, 230, 150]);
    pRetval := VarArrayOf([150.0, 100.0, 230.0, 150.0]);
    Result := S_OK;
  end
  else if (propertyId = UIA_IsGridPatternAvailablePropertyId) then
  begin
//    TVarData(pRetVal).VType := varBoolean;
//    TVarData(pRetVal).VBoolean := True;
    pRetVal := True;
    Result := S_OK;
  end
  else
  begin
    pRetval := VT_EMPTY;
    Result := S_OK;
  end;
end;

function TclAutoFakeGrid.GetRuntimeId(out pRetVal: PSafeArray): HResult;
var
  lSafeArray: PSafeArray;
  lBounds: TSafeArrayBound;
  pData: PInteger;
begin
  lBounds.cElements := 2;
  lBounds.lLbound := 0;
  lSafeArray := SafeArrayCreate(VT_I4,1,@lBounds);
  OleCheck(SafeArrayAccessData(lSafeArray, Pointer(pData)));
  try
    pData^ := -2;
    Inc(pData);
    pData^ := 1110;
  finally
    SafeArrayUnaccessData(lSafeArray);
  end;
  //Edit.Text := Edit.Text + 'runtimeid-called';
  pRetVal := lSafeArray;
  Result := S_OK;
end;
////
function TclAutoFakeGrid.get_BoundingRectangle(out pRetVal: UiaRect): HResult;
var
  lRect: TRect;
begin
  GetWindowRect(Handle,lRect);
  pRetVal.left := lRect.Left;
  pRetVal.top := lRect.Top;
  pRetVal.width := lRect.Width;
  pRetVal.height := lRect.Height;
end;
//
function TclAutoFakeGrid.Get_ColumnCount(out pRetVal: SYSINT): HResult;
begin
  pRetVal := 3;
  Result := S_OK;
end;
//
function TclAutoFakeGrid.Get_FragmentRoot(
  out pRetVal: IRawElementProviderFragmentRoot): HResult;
begin
  pRetVal := Self as IRawElementProviderFragmentRoot;
  pRetVal._AddRef;
  Result := S_OK;
  //Edit.Text := Edit.Text + 'getfragmentroot-called';
end;

function TclAutoFakeGrid.Get_HostRawElementProvider(
  out pRetVal: IRawElementProviderSimple): HResult;
begin
  //pRetVal := nil;
//  Result := UiaHostProviderFromHwnd(Handle, pRetVal);
  pRetVal := Self as IRawElementProviderSimple;
  pRetVal._AddRef;
  Result := S_OK;
end;

function TclAutoFakeGrid.Get_IsReadOnly(out pRetVal: Integer): HResult;
begin
  pRetVal := 0;   // Maybe?
  Result := S_OK;
end;

function TclAutoFakeGrid.Get_ProviderOptions(
  out pRetVal: ProviderOptions): HResult;
begin
  pRetVal := ProviderOptions_ServerSideProvider;
  Result := S_OK;
end;

function TclAutoFakeGrid.Get_RowCount(out pRetVal: SYSINT): HResult;
begin
  pRetVal := 1;
  Result := S_OK;
end;

function TclAutoFakeGrid.Get_Value(out pRetVal: WideString): HResult;
begin
  pRetVal := FValue;
  Result := S_OK;
end;

function TclAutoFakeGrid.Navigate(direction: NavigateDirection;
  out pRetVal: IRawElementProviderFragment): HResult;
var
  lSimpleProvider: IRawElementProviderFragment;
  lProviderSimple: IRawElementProviderSimple;
  H_Result: HResult;
  lColumn: TutInteger;
  lOleVariantTest: OleVariant;
begin
  case direction of
  NavigateDirection_Parent:
    begin
      Result := S_OK;
      //pRetVal := nil;//Self as IRawElementProviderFragment;
      UiaHostProviderFromHwnd(Parent.Handle, lProviderSimple);
      pRetVal := lSimpleProvider as IRawElementProviderFragment;
      //OutputDebugString(PChar('navigate-grid-parent'));
    end;
  NavigateDirection_NextSibling:
    begin
      Result := S_OK;
      pRetVal := nil;
      //OutputDebugString(PChar('navigate-grid-next-sibling'));
    end;
  NavigateDirection_PreviousSibling:
    begin
      Result := S_OK;
      pRetVal := nil;
      //OutputDebugString(PChar('navigate-grid-previous-sibling'));
    end;
  NavigateDirection_FirstChild:
    begin

      lSimpleProvider := FChildItemList[0] as IRawElementProviderFragment;
      lSimpleProvider._AddRef;
      pRetVal := lSimpleProvider;
      Result := S_OK;
      //OutputDebugString(PChar('navigate-grid-first-child'));
    end;
  NavigateDirection_LastChild:
    begin
      lSimpleProvider := FChildItemList[2] as IRawElementProviderFragment;
      lSimpleProvider._AddRef;
      pRetVal := lSimpleProvider;
      Result := S_OK;
      //OutputDebugString(PChar('navigate-grid-lastchild'));
    end;
  end;

end;

function TclAutoFakeGrid.SetFocus: HResult;
begin
  Result := S_OK;
end;

function TclAutoFakeGrid.SetValue(val: PWideChar): HResult;
begin
  FValue := val;
  Result := S_OK;
end;

procedure TclAutoFakeGrid.UnfocusAll;
var
  I: Integer;
  lChildItem: TclAutoFakeGridItem;
begin
  for I := 0 to FChildItemList.Count - 1 do
  begin
    lChildItem := FChildItemList[I];
    lChildItem.IsFocused := False;
  end;
end;

procedure TclAutoFakeGrid.WMGetObject(var Message: TMessage);
var
  lControl: TclAutoFakeGrid;
  lProvider: IRawElementProviderSimple;
begin
  OutputDebugString(PWideChar(IntToStr(Message.LParam)));
  if (Message.Msg = WM_GETOBJECT) and (Message.LParam = -25{UiaRootObjectId}) then // probably should include this, not tested (https://learn.microsoft.com/en-us/windows/win32/winauto/wm-getobject)
  begin
    QueryInterface(IID_IRawElementProviderSimple, FRawElementProviderSimple);
    message.Result := UiaReturnRawElementProvider(self.Handle, Message.WParam, Message.LParam, FRawElementProviderSimple);
  end
  else
  begin
    Message.Result := DefWindowProc(self.Handle, Message.Msg, Message.WParam, Message.LParam);
  end;
end;


{ TclAutoFakeGridItem }

constructor TclAutoFakeGridItem.Create(aOwner: TWinControl; aFragmentRoot: IRawElementProviderFragmentRoot;aRow,aColumn: TutInteger);
var
  lRect: TRect;
  lhandle: hwnd;
begin
  inherited Create;
  FValue := 'Das ist der Wert vom TclAutoFakeGridItem';
  FOwner := aOwner;
  FRoot := aFragmentRoot;
  FRow := aRow;
  FColumn := aColumn;
end;

destructor TclAutoFakeGridItem.Destroy;
begin
  inherited;
end;

function TclAutoFakeGridItem.GetEmbeddedFragmentRoots(
  out pRetVal: PSafeArray): HResult;
begin
  pRetval := nil;
  Result := S_OK;
end;

function TclAutoFakeGridItem.GetHandle: HWND;
begin
  Result := (FOwner as TclAutoFakeGrid).Handle;
end;

function TclAutoFakeGridItem.GetPatternProvider(patternId: SYSINT;
  out pRetVal: IInterface): HResult;
begin
  pRetval := nil;
  if (patternID = UIA_ValuePatternID) then
  begin
    pRetVal := self  as IValueProvider;
    pRetVal._AddRef;
    Result := S_OK
  end
  else if (patternID = UIA_GridItemPatternId) then
  begin
    pRetVal := self as IGridItemProvider;
    pRetVal._AddRef;
    Result := S_OK;
  end
  else if (patternID = UIA_InvokePatternId) then
  begin
    pRetVal := self as IInvokeProvider;
    pRetVal._AddRef;
    Result := S_OK;
  end
  else
  begin
    Result := S_OK;
  end;
end;

function TclAutoFakeGridItem.GetPropertyValue(propertyId: SYSINT;
  out pRetVal: OleVariant): HResult;
begin
  if(propertyId = UIA_ClassNamePropertyId) then
  begin
    TVarData(pRetVal).VType := varOleStr;
    TVarData(pRetVal).VOleStr := pWideChar('Das ist die Klasse vom TclAutoFakeGridItem');
    result := S_OK;
  end
  else if(propertyId = UIA_NamePropertyId) then
  begin
    TVarData(pRetVal).VType := varOleStr;
    if (self.FColumn = 0) then
    begin
      TVarData(pRetVal).VOleStr := pWideChar('First-Item');
    end
    else if self.FColumn = 1 then
    begin
      TVarData(pRetVal).VOleStr := pWideChar('Second-Item');
    end
    else if self.FColumn = 2 then
    begin
      TVarData(pRetVal).VOleStr := pWideChar('Third-Item');
    end;
    result := S_OK;
  end
  else if(propertyId = UIA_AutomationIdPropertyId) then
  begin
    TVarData(pRetVal).VType := varOleStr;
    if (self.FColumn = 0) then
    begin
      TVarData(pRetVal).VOleStr := pWideChar('3058732320');
    end
    else if self.FColumn = 1 then
    begin
      TVarData(pRetVal).VOleStr := pWideChar('3058732321');
    end
    else if self.FColumn = 2 then
    begin
      TVarData(pRetVal).VOleStr := pWideChar('3058732322');
    end;
    result := S_OK;
  end
  else if(propertyId = UIA_ControlTypePropertyId) then
  begin
    TVarData(pRetVal).VType := varInteger;
    TVarData(pRetVal).VInteger := {UIA_DataItemControlTypeId}UIA_CustomControlTypeId;
    result := S_OK;
  end
  else if (propertyId = UIA_IsControlElementPropertyId) then
  begin
    pRetVal := True;
    result := S_OK;
  end
  else if (propertyId = UIA_IsContentElementPropertyId) then
  begin
    pRetVal := True;
    result := S_OK;
  end
  else if (propertyId = UIA_FrameworkIdPropertyId) then
  begin
    pRetVal := 'Win32';
    result := S_OK;
  end
  else
  begin
    pRetVal := VT_EMPTY;
    result := S_OK;
  end;
end;

function TclAutoFakeGridItem.GetRuntimeId(out pRetVal: PSafeArray): HResult;
var
  lSafeArray: PSafeArray;
  lBounds: TSafeArrayBound;
  pData: PInteger;
  lInteger: TutInteger;
begin
  lBounds.cElements := 2;
  lBounds.lLbound := 0;
  lSafeArray := SafeArrayCreate(VT_I4,1,@lBounds);
  OleCheck(SafeArrayAccessData(lSafeArray, Pointer(pData)));
  try
    pData^ := -2;
    Inc(pData);
    Self.Get_column(lInteger);
    pData^ := 1111+lInteger;
  finally
    SafeArrayUnaccessData(lSafeArray);
  end;
  //Edit.Text := Edit.Text + 'runtimeid-called';
  pRetVal := lSafeArray;
  Result := S_OK;
end;

function TclAutoFakeGridItem.get_BoundingRectangle(
  out pRetVal: UiaRect): HResult;
var
  lRect: TRect;
begin
  GetWindowRect(Handle,lRect);
  if ((FRow = 0) and (FColumn = 0)) then
  begin
    pRetVal.left := lRect.Left;
    pRetVal.top := lRect.Top;
    pRetVal.width := 10;
    pRetVal.height := 10;//lRect.Height;
  end
  else if FColumn = 1 then
  begin
    pRetVal.left := lRect.Left+10;
    pRetVal.top := lRect.Top;
    pRetVal.width := 10;
    pRetVal.height := lRect.Height;
  end;

  Result := S_OK;
end;

function TclAutoFakeGridItem.Get_column(out pRetVal: SYSINT): HResult;
begin
  pRetVal := FColumn;
  Result := S_OK;
end;

function TclAutoFakeGridItem.Get_ColumnSpan(out pRetVal: SYSINT): HResult;
begin
  pRetVal := 1;
  Result := S_OK;
end;

function TclAutoFakeGridItem.Get_ContainingGrid(
  out pRetVal: IRawElementProviderSimple): HResult;
begin
  pRetVal := FOwner as IRawElementProviderSimple;
  pRetVal._AddRef;
  Result := S_OK;
end;

function TclAutoFakeGridItem.Get_FragmentRoot(
  out pRetVal: IRawElementProviderFragmentRoot): HResult;
begin
  //pRetval := FRoot;
  pRetVal := FOwner as IRawElementProviderFragmentRoot;
  pRetval._AddRef;
  Result := S_OK;
end;

function TclAutoFakeGridItem.Get_HostRawElementProvider(
  out pRetVal: IRawElementProviderSimple): HResult;
begin
  pRetVal := nil;
  //UiaHostProviderFromHwnd(self.Handle, pRetVal);
  Result := S_OK;
end;

function TclAutoFakeGridItem.Get_IsReadOnly(out pRetVal: Integer): HResult;
begin
  pRetVal := 0;
  Result := S_OK;
end;

function TclAutoFakeGridItem.Get_ProviderOptions(
  out pRetVal: ProviderOptions): HResult;
begin
  pRetVal := ProviderOptions_ServerSideProvider;
  Result := S_OK;
end;

function TclAutoFakeGridItem.Get_row(out pRetVal: SYSINT): HResult;
begin
  pRetVal := FRow;
  Result := S_OK;
end;

function TclAutoFakeGridItem.Get_RowSpan(out pRetVal: SYSINT): HResult;
begin
  pRetVal := 1;
  Result := S_OK;
end;

function TclAutoFakeGridItem.Get_Value(out pRetVal: WideString): HResult;
begin
  pRetVal := FValue;
  Result := S_OK;
end;

function TclAutoFakeGridItem.Invoke: HResult;
begin
  OutputDebugString('Item invoked');
end;

function TclAutoFakeGridItem.Navigate(direction: NavigateDirection;
  out pRetVal: IRawElementProviderFragment): HResult;
var
  lSimpleProvider: IRawElementProviderSimple;
  lFragmentProvider: IRawElementProviderFragment;
  lGrid: TclAutoFakeGrid;
  H_Result: HResult;
  lColumn: TutInteger;
begin
  lGrid := (FOwner as TclAutoFakeGrid);
  case direction of
  NavigateDirection_Parent:
    begin
      pRetVal := lGrid as IRawElementProviderFragment;
      pRetVal._AddRef;
      Result := S_OK;
      //OutputDebugString(PChar('navigate-griditem-parent'));
    end;
  NavigateDirection_NextSibling:
    begin
      Result := S_OK;
      H_Result := Get_Column(lColumn);
      if lColumn < 2 then
      begin
        lFragmentProvider := lGrid.FChildItemList[lColumn+1] as IRawElementProviderFragment;
        lFragmentProvider._AddRef;
        pRetVal := lFragmentProvider;
        //OutputDebugString(PChar('navigate-griditem-next-sibling'));
      end
      else
      begin
        pRetVal := nil;
        Result := S_OK;
      end;
    end;
  NavigateDirection_PreviousSibling:
    begin

      H_Result := Get_Column(lColumn);

      if lColumn > 0 then
      begin
        lFragmentProvider := lGrid.FChildItemList[lColumn - 1] as IRawElementProviderFragment;
        lFragmentProvider._AddRef;
        pRetVal := lFragmentProvider;
        Result := S_OK;
        //OutputDebugString(PChar('navigate-griditem-previous-sibling'));
      end
      else
      begin
        pRetVal := nil;
        Result := S_OK;
      end;
    end;
  NavigateDirection_FirstChild:
    begin
      Result := S_OK;
      pRetVal := nil;
      //OutputDebugString(PChar('navigate-griditem-first-Child'));
    end;
  NavigateDirection_LastChild:
    begin
      Result := S_OK;
      pRetVal := nil;
      //OutputDebugString(PChar('navigate-griditem-last-child'));
    end;
  end;
  Result := S_OK;
end;

function TclAutoFakeGridItem.SetFocus: HResult;
begin
  (FOwner as TclAutoFakeGrid).UnfocusAll;
  IsFocused := True;
  Result := S_OK;
end;

function TclAutoFakeGridItem.SetValue(val: PWideChar): HResult;
begin
  FValue := val;
  Result := S_OK;
end;
end.

Topic		Replies	Views
UI Automation related activities Activities uiautomation , activities , question	0	989	June 9, 2021
UIPATH reframework io_TransactionData Help uiautomation , activities , question	1	788	November 26, 2020
UI Activities are not consistent while automating a tool called "Aggresso" Help uiautomation , activities	6	3850	October 29, 2018
New UI Automation framework! :pray: StudioX uiautomation , studio , news	63	8709	October 31, 2022
About "The UI element is invalid" error StudioX uiautomation , bug	1	180	May 10, 2024

UiPath doesn't recognize children/ fake UIElements

Related topics