Grass boundary detection when not 100% green

edit

First of all, thank you so much for all the useful information on this forum. I managed to put together a python script to detect the lawn boundary for an autonomous lawn mower project I am working on. It can detect the boundary fine if the lawn is 100% green (below).

However, I encounter an issue if the lawn is not 100% green. This mage below has some patches of other colors, causing the boundary to be broken (below).

Please let me know if you have any suggestion on how I can fine-tune the script to better detect the boundary for the lawn in the second photo. Original image is here. .

My script is as follows:

    #RGB to HSV
    img_hsv = cv2.cvtColor(image, cv2.COLOR_RGB2HSV)

    #green mask
    lower_green = np.array( [40,40,40], dtype = "uint8")
    upper_green = np.array( [70,255,255], dtype = "uint8")
    mask_green = cv2.inRange(img_hsv, lower_green, upper_green)
    output = cv2.bitwise_and(image, image, mask=mask_green)
    cv2.imshow('output',output)

    #gaussian blur
    kernel_size = 3
    gauss = gaussian_blur(mask_green,kernel_size)
    cv2.imshow('gauss',gauss)

    #dilation
    kernel = np.ones((kernel_size*2,kernel_size*2),np.uint8)
    dilation_image = cv2.dilate(mask_green, kernel, iterations=1)
    cv2.imshow('dilation_image',dilation_image)

    #morph close
    closing = cv2.morphologyEx(dilation_image, cv2.MORPH_CLOSE, kernel)
    cv2.imshow('Closing',closing)

    #remove small blobs
    nb_components, output, stats, centroids = cv2.connectedComponentsWithStats(closing, connectivity=8)
    #connectedComponentswithStats yields every separated component with information on each of them, such as size
    sizes = stats[1:, -1]; nb_components = nb_components - 1

    min_size = 150  #num pixels

    img2 = np.zeros((output.shape))
    for i in range(0, nb_components):
        if sizes[i] >= min_size:
            img2[output == i + 1] = 255

    cv2.imshow('final',img2)