Invoke Python Method goes into errors!

Hi Guys,

I’ve seen the python invoke method demo and it works fine.
I’m now trying to replicate this with my script.
Code is here:

# load yolov3 model and perform object detection
# based on https://github.com/experiencor/keras-yolo3
import numpy as np
from numpy import expand_dims
from keras.models import load_model
from tensorflow.keras.utils import load_img
from tensorflow.keras.utils import img_to_array
from matplotlib import pyplot
from matplotlib.patches import Rectangle
import sys


class BoundBox:
	def __init__(self, xmin, ymin, xmax, ymax, objness = None, classes = None):
		self.xmin = xmin
		self.ymin = ymin
		self.xmax = xmax
		self.ymax = ymax
		self.objness = objness
		self.classes = classes
		self.label = -1
		self.score = -1

	def get_label(self):
		if self.label == -1:
			self.label = np.argmax(self.classes)

		return self.label

	def get_score(self):
		if self.score == -1:
			self.score = self.classes[self.get_label()]

		return self.score

def _sigmoid(x):
	return 1. / (1. + np.exp(-x))

def decode_netout(netout, anchors, obj_thresh, net_h, net_w):
	grid_h, grid_w = netout.shape[:2]
	nb_box = 3
	netout = netout.reshape((grid_h, grid_w, nb_box, -1))
	nb_class = netout.shape[-1] - 5
	boxes = []
	netout[..., :2]  = _sigmoid(netout[..., :2])
	netout[..., 4:]  = _sigmoid(netout[..., 4:])
	netout[..., 5:]  = netout[..., 4][..., np.newaxis] * netout[..., 5:]
	netout[..., 5:] *= netout[..., 5:] > obj_thresh

	for i in range(grid_h*grid_w):
		row = i / grid_w
		col = i % grid_w
		for b in range(nb_box):
			# 4th element is objectness score
			objectness = netout[int(row)][int(col)][b][4]
			if(objectness.all() <= obj_thresh): continue
			# first 4 elements are x, y, w, and h
			x, y, w, h = netout[int(row)][int(col)][b][:4]
			x = (col + x) / grid_w # center position, unit: image width
			y = (row + y) / grid_h # center position, unit: image height
			w = anchors[2 * b + 0] * np.exp(w) / net_w # unit: image width
			h = anchors[2 * b + 1] * np.exp(h) / net_h # unit: image height
			# last elements are class probabilities
			classes = netout[int(row)][col][b][5:]
			box = BoundBox(x-w/2, y-h/2, x+w/2, y+h/2, objectness, classes)
			boxes.append(box)
	return boxes

def correct_yolo_boxes(boxes, image_h, image_w, net_h, net_w):
	new_w, new_h = net_w, net_h
	for i in range(len(boxes)):
		x_offset, x_scale = (net_w - new_w)/2./net_w, float(new_w)/net_w
		y_offset, y_scale = (net_h - new_h)/2./net_h, float(new_h)/net_h
		boxes[i].xmin = int((boxes[i].xmin - x_offset) / x_scale * image_w)
		boxes[i].xmax = int((boxes[i].xmax - x_offset) / x_scale * image_w)
		boxes[i].ymin = int((boxes[i].ymin - y_offset) / y_scale * image_h)
		boxes[i].ymax = int((boxes[i].ymax - y_offset) / y_scale * image_h)

def _interval_overlap(interval_a, interval_b):
	x1, x2 = interval_a
	x3, x4 = interval_b
	if x3 < x1:
		if x4 < x1:
			return 0
		else:
			return min(x2,x4) - x1
	else:
		if x2 < x3:
			 return 0
		else:
			return min(x2,x4) - x3

def bbox_iou(box1, box2):
	intersect_w = _interval_overlap([box1.xmin, box1.xmax], [box2.xmin, box2.xmax])
	intersect_h = _interval_overlap([box1.ymin, box1.ymax], [box2.ymin, box2.ymax])
	intersect = intersect_w * intersect_h
	w1, h1 = box1.xmax-box1.xmin, box1.ymax-box1.ymin
	w2, h2 = box2.xmax-box2.xmin, box2.ymax-box2.ymin
	union = w1*h1 + w2*h2 - intersect
	return float(intersect) / union

def do_nms(boxes, nms_thresh):
	if len(boxes) > 0:
		nb_class = len(boxes[0].classes)
	else:
		return
	for c in range(nb_class):
		sorted_indices = np.argsort([-box.classes[c] for box in boxes])
		for i in range(len(sorted_indices)):
			index_i = sorted_indices[i]
			if boxes[index_i].classes[c] == 0: continue
			for j in range(i+1, len(sorted_indices)):
				index_j = sorted_indices[j]
				if bbox_iou(boxes[index_i], boxes[index_j]) >= nms_thresh:
					boxes[index_j].classes[c] = 0

# load and prepare an image
def load_image_pixels(filename, shape):
	# load the image to get its shape
	image = load_img(filename)
	width, height = image.size
	# load the image with the required size
	image = load_img(filename, target_size=shape)
	# convert to numpy array
	image = img_to_array(image)
	# scale pixel values to [0, 1]
	image = image.astype('float32')
	image /= 255.0
	# add a dimension so that we have one sample
	image = expand_dims(image, 0)
	return image, width, height

# get all of the results above a threshold
def get_boxes(boxes, labels, thresh):
	v_boxes, v_labels, v_scores = list(), list(), list()
	# enumerate all boxes
	for box in boxes:
		# enumerate all possible labels
		for i in range(len(labels)):
			# check if the threshold for this label is high enough
			if box.classes[i] > thresh:
				v_boxes.append(box)
				v_labels.append(labels[i])
				v_scores.append(box.classes[i]*100)
				# don't break, many labels may trigger for one box
	return v_boxes, v_labels, v_scores

# draw all results
def draw_boxes(filename, v_boxes, v_labels, v_scores):
	# load the image
	data = pyplot.imread(filename)
	# plot the image
	pyplot.imshow(data)
	# get the context for drawing boxes
	ax = pyplot.gca()
	# plot each box
	for i in range(len(v_boxes)):
		box = v_boxes[i]
		# get coordinates
		y1, x1, y2, x2 = box.ymin, box.xmin, box.ymax, box.xmax
		# calculate width and height of the box
		width, height = x2 - x1, y2 - y1
		# create the shape
		rect = Rectangle((x1, y1), width, height, fill=False, color='white')
		# draw the box
		ax.add_patch(rect)
		# draw text and score in top left corner
		label = "%s (%.3f)" % (v_labels[i], v_scores[i])
		pyplot.text(x1, y1, label, color='white')
	# show the plot
	pyplot.show()


def init():	
	# load yolov3 model
	model = load_model('./yolov3_model.h5', compile=False)
	# define the expected input shape for the model
	input_w, input_h = 416, 416
	# define our new photo
	photo_filename = './Hydrant.png'
	# load and prepare image
	image, image_w, image_h = load_image_pixels(photo_filename, (input_w, input_h))
	# make prediction
	yhat = model.predict(image)
	# summarize the shape of the list of arrays
	print([a.shape for a in yhat])
	# define the anchors
	anchors = [[116,90, 156,198, 373,326], [30,61, 62,45, 59,119], [10,13, 16,30, 33,23]]
	# define the probability threshold for detected objects
	class_threshold = 0.6
	boxes = list()
	for i in range(len(yhat)):
		# decode the output of the network
		boxes += decode_netout(yhat[i][0], anchors[i], class_threshold, input_h, input_w)
	# correct the sizes of the bounding boxes for the shape of the image
	correct_yolo_boxes(boxes, image_h, image_w, input_h, input_w)
	# suppress non-maximal boxes
	do_nms(boxes, 0.5)
	# define the labels
	labels = ["person", "bicycle", "car", "motorbike", "aeroplane", "bus", "train", "truck",
		"boat", "traffic light", "fire hydrant", "stop sign", "parking meter", "bench",
		"bird", "cat", "dog", "horse", "sheep", "cow", "elephant", "bear", "zebra", "giraffe",
		"backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", "skis", "snowboard",
		"sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard",
		"tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana",
		"apple", "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake",
		"chair", "sofa", "pottedplant", "bed", "diningtable", "toilet", "tvmonitor", "laptop", "mouse",
		"remote", "keyboard", "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator",
		"book", "clock", "vase", "scissors", "teddy bear", "hair drier", "toothbrush"]
	# get the details of the detected objects
	v_boxes, v_labels, v_scores = get_boxes(boxes, labels, class_threshold)
	# summarize what we found
	for i in range(len(v_boxes)):
		print(v_labels[i], v_scores[i])

	#draw what we found
	draw_boxes(photo_filename, v_boxes, v_labels, v_scores)


if __name__ == '__main__':
	init()



And this is my UiPath workflow:

The code works fine if I run the script in a windows shell, but it raises this error with UiPath:

22.4.1+Branch.master.Sha.ae3dd5d96272ef2a8c7136934fe08624b9ad1069

Source: Invoke Python Method

Message: One or more errors occurred.

Exception Type: System.AggregateException

RemoteException wrapping System.AggregateException: One or more errors occurred.  ---> RemoteException wrapping System.InvalidOperationException: Error invoking Python method  ---> RemoteException wrapping System.NullReferenceException: Object reference not set to an instance of an object. 
   at UiPath.Python.Impl.OutOfProcessEngine.<>c__DisplayClass15_0.<InvokeMethod>b__0()
   at System.Threading.Tasks.Task`1.InnerInvoke()
   at System.Threading.Tasks.Task.Execute()
--- End of stack trace from previous location where exception was thrown ---
   at System.Runtime.CompilerServices.TaskAwaiter.ThrowForNonSuccess(Task task)
   at System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)
   at UiPath.Python.Activities.InvokeMethod.<ExecuteAsync>d__16.MoveNext()
	--- End of inner exception stack trace ---
   at UiPath.Python.Activities.InvokeMethod.<ExecuteAsync>d__16.MoveNext()
	--- End of inner exception stack trace ---
   at UiPath.Shared.Activities.AsyncTaskCodeActivity.EndExecute(AsyncCodeActivityContext context, IAsyncResult result)
   at System.Activities.AsyncCodeActivity.System.Activities.IAsyncCodeActivity.FinishExecution(AsyncCodeActivityContext context, IAsyncResult result)
   at System.Activities.AsyncCodeActivity.CompleteAsyncCodeActivityData.CompleteAsyncCodeActivityWorkItem.Execute(ActivityExecutor executor, BookmarkManager bookmarkManager)

Don’t know why, could it be because of the import at the beginning of my script?

Someone could help me fix this, please? Thanks a lot :slight_smile:

Hi @Re_Walter ,

Could you also show us the Properties of the Python Scope Activity ?

Assuming that a Simple Python Code works fine, Could you add any one/all of the import statements to the code which is working and Check if it produces the same error ?

This way we’ll be able to understand if it is required to install the dependencies using the cmd prompt first.

Hi @supermanPunch, thanks for your reply :slight_smile:

Could you also show us the Properties of the Python Scope Activity ?

I don’t think the problem is here, because it is the same configuration I used to run the demo, and it works fine there.

Assuming that a Simple Python Code works fine, Could you add any one/all of the import statements to the code which is working and Check if it produces the same error ?

Sorry, I don’t understand your question. Import statements are already inside the python code, they are:

# load yolov3 model and perform object detection
# based on https://github.com/experiencor/keras-yolo3
import numpy as np
from numpy import expand_dims
from keras.models import load_model
from tensorflow.keras.utils import load_img
from tensorflow.keras.utils import img_to_array
from matplotlib import pyplot
from matplotlib.patches import Rectangle
import sys

And those works fine if I run the script normally, inside a windows shell.

My question is whether UiPath can import these libraries before invoking the method.

Thanks for you help :slight_smile:

Hello @Re_Walter
Provide a working folder in properties as a path of your code Location(File.py) and try again. It may works

@Re_Walter ,

Just wanted to check when added the import statements to the working python code, if errors are being generated. If this check has been already done, we can go for other checks.

Hi @Gokul_Jayakumar many thanks for your reply. :slight_smile:

I tried, but the error persists with the same output

Hi @supermanPunch thanks for your reply :slight_smile:

In my python code, the import statements are already in, and they work fine, as expected.

Or did you mean to remove them and add one statement at a time when I run the script with UiPath, to see if there are any changes?

@Re_Walter ,

Apologies if being confused. Let me just highlight what I have got to know and if it is in sync with your understanding as well.

  1. Tested with Python Invoke Method demo workflow (Python Code 1) - Working Fine. - No Import statements as such.

  2. Tested with your Python Code of Object Detection (Python Code 2)- Received Errors - import Statements are present.

Suggestion was to add the import statements from Python Code 2 to Python Code 1, to check if the errors are being generated for Python Code 1 (Only the import statements are added), as we already have one version of Python Code which does work fine.

This is done to just understand whether an installation of the modules is required separately using the command prompt. Most of the times we would require to install the python modules using cmd at first.

Hey @Re_Walter, I hope you’re well.

I get this error quite often when working with Python scripts in UiPath and these are my steps to debug, hopefully one of them help you:

  1. Run your Python script from the Command Prompt to see what output you get. Reference your Python Interpreter in Command Prompt to ensure you get the same output. This should tell you what the error is.
  2. I like to double check that all libraries are installed and working, especially if I’m recreating virtual environments, sometimes that causes a lot of issues but if you are using the venv interpreter, Step 1 will show if there’s an error.
  3. Double check indentation in a Python IDE, that has caught me out a few times as well.

Hope this helps.

For more info, please feel free to read through my post that details how to use Python in UiPath: https://thejpanda.com/2020/01/09/rpa-using-the-python-activity-in-uipath/

Hi @supermanPunch, thanks for your reply.

Sorry, I didn’t catch you meant this. I’ve done a try, to see what happens.

I ran this code:

def impor():
	import numpy as np


def add(num1,num2):
	impor()
	n = np.array([1,2,3])
	n1 = n[0]
	return num1 + num2 + n1
	
def dif(num1,num2):
	return num1 - num2

Both with the “impor()” function and without, with the statement “import numpy as np” as the first line.
Both results gave me an error:

22.4.1+Branch.master.Sha.ae3dd5d96272ef2a8c7136934fe08624b9ad1069

Source: Invoke Python Method

Message: Pipe is broken.

Exception Type: System.IO.IOException

RemoteException wrapping System.IO.IOException: Pipe is broken. 
   at System.IO.Pipes.PipeStream.CheckWriteOperations()
   at System.IO.Pipes.PipeStream.Flush()
   at System.IO.StreamWriter.Flush(Boolean flushStream, Boolean flushEncoder)
   at System.IO.StreamWriter.Dispose(Boolean disposing)
   at System.IO.TextWriter.Dispose()
   at UiPath.Python.Service.PythonProxy.RequestAsync(PythonRequest request, CancellationToken ct)
   at UiPath.Python.Service.PythonProxy.Shutdown()
   at UiPath.Python.Impl.OutOfProcessEngine.Release()
   at UiPath.Python.Activities.PythonScope.Cleanup()
   at UiPath.Python.Activities.PythonScope.OnFaulted(NativeActivityFaultContext faultContext, Exception propagatedException, ActivityInstance propagatedFrom)
   at System.Activities.Runtime.FaultCallbackWrapper.Invoke(NativeActivityFaultContext faultContext, Exception propagatedException, ActivityInstance propagatedFrom)
   at System.Activities.Runtime.FaultCallbackWrapper.FaultWorkItem.Execute(ActivityExecutor executor, BookmarkManager bookmarkManager)

Which is different from the other one.

I also try to import numpy without using it, and in that case, it works fine.

Hi @Jacqui_M , thanks for your reply :slight_smile:

The problem is that when I run the script in the command prompt (cmd) it works fine, but the error arises when I try to use it inside a UiPath.

I’m not using a Python IDE, since I’m using AI libraries, such as TensorFlow, and I tried the script on Google Colab, and then I download it.

Thanks for your help :slight_smile:

Ah okay great! Thank you for clarifying. Have you tried invoking a basic script with only one function? It could be an inherited scope issue.

Hi @Jacqui_M, thanks for your reply. :slight_smile:

I tried the demo https://docs.uipath.com/activities/docs/invoke-method and it works fine, but if I try to use my script, it gives me the error I reported.

UPDATE:

I’m trying to split the code in order to see in which part the error arises, and it seems to be at the code line:

model = load_model('./yolov3_model.h5', compile=False)

This is the test I have done:

Script:

from keras.models import load_model



def main():
    model = load_model('./yolov3_model.h5', compile=False)
    if(model):
        return 1

Workflow:

And the error is the same:

22.4.1+Branch.master.Sha.ae3dd5d96272ef2a8c7136934fe08624b9ad1069

Source: Invoke Python Method

Message: One or more errors occurred.

Exception Type: System.AggregateException

RemoteException wrapping System.AggregateException: One or more errors occurred.  ---> RemoteException wrapping System.InvalidOperationException: Error invoking Python method  ---> RemoteException wrapping System.NullReferenceException: Object reference not set to an instance of an object. 
   at UiPath.Python.Impl.OutOfProcessEngine.<>c__DisplayClass15_0.<InvokeMethod>b__0()
   at System.Threading.Tasks.Task`1.InnerInvoke()
   at System.Threading.Tasks.Task.Execute()
--- End of stack trace from previous location where exception was thrown ---
   at System.Runtime.CompilerServices.TaskAwaiter.ThrowForNonSuccess(Task task)
   at System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)
   at UiPath.Python.Activities.InvokeMethod.<ExecuteAsync>d__16.MoveNext()
	--- End of inner exception stack trace ---
   at UiPath.Python.Activities.InvokeMethod.<ExecuteAsync>d__16.MoveNext()
	--- End of inner exception stack trace ---
   at UiPath.Shared.Activities.AsyncTaskCodeActivity.EndExecute(AsyncCodeActivityContext context, IAsyncResult result)
   at System.Activities.AsyncCodeActivity.System.Activities.IAsyncCodeActivity.FinishExecution(AsyncCodeActivityContext context, IAsyncResult result)
   at System.Activities.AsyncCodeActivity.CompleteAsyncCodeActivityData.CompleteAsyncCodeActivityWorkItem.Execute(ActivityExecutor executor, BookmarkManager bookmarkManager)

Does anyone know a solution? Thanks