ValueError: need at least one array to concatenate

Question

I am having issues with

ValueError: need at least one array to concatenate

Below is the whole error message.

    Training mode
Traceback (most recent call last):
  File "bcf.py", line 342, in <module>
    bcf.train()
  File "bcf.py", line 321, in train
    self._learn_codebook()
  File "bcf.py", line 142, in _learn_codebook
    feats_sc = np.concatenate(feats_sc, axis=1).transpose()
ValueError: need at least one array to concatenate

Below is the area of the problem.

    def _learn_codebook(self):
    MAX_CFS = 800 # max number of contour fragments per image; if above, sample randomly
    CLUSTERING_CENTERS = 1500
    feats_sc = []
    for image in self.data.values():
        feats = image['cfs']
        feat_sc = feats[1]
        if feat_sc.shape[1] > MAX_CFS:
            # Sample MAX_CFS from contour fragments
            rand_indices = np.random.permutation(feat_sc.shape[1])
            feat_sc = feat_sc[:, rand_indices[:MAX_CFS]]
        feats_sc.append(feat_sc)
    feats_sc = np.concatenate(feats_sc, axis=1).transpose()
    print("Running KMeans...")
    self.kmeans = sklearn.cluster.KMeans(min(CLUSTERING_CENTERS, feats_sc.shape[0]), n_jobs=-1, algorithm='elkan').fit(feats_sc)
    print("Saving codebook...")
    self._save_kmeans(self.kmeans)
    return self.kmeans

Below is the complete CLASS

class BCF():
def __init__(self):
    self.DATA_DIR = "/Users/minniemouse/TRAIN/bcf-master5/data/cuauv/"
    self.PERC_TRAINING_PER_CLASS = 0.5
    self.CODEBOOK_FILE = "codebook.data"
    self.CLASSIFIER_FILE = "classifier"
    self.LABEL_TO_CLASS_MAPPING_FILE = "labels_to_classes.data"
    self.classes = defaultdict(list)
    self.data = defaultdict(dict)
    self.counter = defaultdict(int)
    self.kmeans = None
    self.clf = None
    self.label_to_class_mapping = None

def _load_classes(self):
    for dir_name, subdir_list, file_list in os.walk(self.DATA_DIR):
        if subdir_list:
            continue
        for f in sorted(file_list, key=hash):
            self.classes[dir_name.split('/')[-1]].append(os.path.join(dir_name, f))

def _load_training(self):
    for cls in self.classes:
        images = self.classes[cls]
        for image in images[:int(len(images) * self.PERC_TRAINING_PER_CLASS)]:
            image_id = self._get_image_identifier(cls)
            self.data[image_id]['image'] = cv2.imread(image, cv2.IMREAD_GRAYSCALE)
            if self.data[image_id]['image'] is None:
                print("Failed to load " + image)

def _load_testing(self):
    for cls in self.classes:
        images = self.classes[cls]
        for image in images[int(len(images) * self.PERC_TRAINING_PER_CLASS):]:
            image_id = self._get_image_identifier(cls)
            self.data[image_id]['image'] = cv2.imread(image, cv2.IMREAD_GRAYSCALE)
            if self.data[image_id]['image'] is None:
                print("Failed to load " + image)

def _load_single(self, image):
    # Load single image data
    self.data.clear()
    image_id = self._get_image_identifier(None)
    self.data[image_id]['image'] = image

def _save_label_to_class_mapping(self):
    self.label_to_class_mapping = {hash(cls): cls for cls in self.classes}
    with open(self.LABEL_TO_CLASS_MAPPING_FILE, 'wb') as out_file:
        pickle.dump(self.label_to_class_mapping, out_file, -1)

def _load_label_to_class_mapping(self):
    if self.label_to_class_mapping is None:
        with open(self.LABEL_TO_CLASS_MAPPING_FILE, 'rb') as in_file:
            self.label_to_class_mapping = pickle.load(in_file)
    return self.label_to_class_mapping

def _normalize_shapes(self):
    for (cls, idx) in self.data.keys():
        image = self.data[(cls, idx)]['image']
        # Remove void space
        y, x = np.where(image > 50)
        max_y = y.max()
        min_y = y.min()
        max_x = x.max()
        min_x = x.min()
        trimmed = image[min_y:max_y, min_x:max_x] > 50
        trimmed = trimmed.astype('uint8')
        trimmed[trimmed > 0] = 255
        self.data[(cls, idx)]['normalized_image'] = trimmed

def _extract_cf(self):
    for (cls, idx) in self.data.keys():
        image = self.data[(cls, idx)]['normalized_image']
        images,contours, _ = cv2.findContours(image, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
        contour = sorted(contours, key=len)[-1]
        mat = np.zeros(image.shape, np.int8)
        cv2.drawContours(mat, [contour], -1, (255, 255, 255))
        #self.show(mat)
        MAX_CURVATURE = 1.5
        N_CONTSAMP = 50
        N_PNTSAMP = 10
        C = None
        for pnt in contour:
            if C is None:
                C = np.array([[pnt[0][0], pnt[0][1]]])
            else:
                C = np.append(C, [[pnt[0][0], pnt[0][1]]], axis=0)
        cfs = self._extr_raw_points(C, MAX_CURVATURE, N_CONTSAMP, N_PNTSAMP)
        tmp = mat.copy()
        for cf in cfs:
            for pnt in cf:
                cv2.circle(tmp, (pnt[0], pnt[1]), 2, (255, 0, 0))
            #self.show(tmp)
        num_cfs = len(cfs)
        print("Extracted %s points" % (num_cfs))
        feat_sc = np.zeros((300, num_cfs))
        xy = np.zeros((num_cfs, 2))

        for i in range(num_cfs):
            cf = cfs[i]
            sc, _, _, _ = shape_context(cf)
            # shape context is 60x5 (60 bins at 5 reference points)
            sc = sc.flatten(order='F')
            sc /= np.sum(sc) # normalize
            feat_sc[:, i] = sc
            # shape context descriptor sc for each cf is 300x1
            # save a point at the midpoint of the contour fragment
            xy[i, 0:2] = cf[np.round(len(cf) / 2. - 1).astype('int32'), :]
        sz = image.shape
        self.data[(cls, idx)]['cfs'] = (cfs, feat_sc, xy, sz)

def _learn_codebook(self):
    MAX_CFS = 800 # max number of contour fragments per image; if above, sample randomly
    CLUSTERING_CENTERS = 1500
    feats_sc = []
    for image in self.data.values():
        feats = image['cfs']
        feat_sc = feats[1]
        if feat_sc.shape[1] > MAX_CFS:
            # Sample MAX_CFS from contour fragments
            rand_indices = np.random.permutation(feat_sc.shape[1])
            feat_sc = feat_sc[:, rand_indices[:MAX_CFS]]
        feats_sc.append(feat_sc)
    feats_sc = np.concatenate(feats_sc, axis=1).transpose()
    print("Running KMeans...")
    self.kmeans = sklearn.cluster.KMeans(min(CLUSTERING_CENTERS,  feats_sc.shape[0]), n_jobs=-1, algorithm='elkan').fit(feats_sc)
    print("Saving codebook...")
    self._save_kmeans(self.kmeans)
    return self.kmeans

I have read through the various posts on ValueError already described, but I am not having much luck on figuring it out. I have now attached the CLASS and full error message information.

Please, can someone point out what I am missing?

Thank you

Answer 1

the problem comes from the lenght of your array. Check if your array/list is longer than to 0 print(len(feats_sc)).

Don't forget to checkout the documentation numpy.concatenate — NumPy v1.16 Manual

Answer 2

The problem seems to be in np.concatenate where it expects an array of arrays and it's not receiving that.

Refer: Scipy docs

numpy.concatenate((a1, a2, ...), axis=0, out=None)

Join a sequence of arrays along an existing axis.

Parameters:
a1, a2, … : sequence of array_like The arrays must have the same shape, except in the dimension corresponding to axis (the first, by default).

axis : int, optional The axis along which the arrays will be joined. If axis is None, arrays are flattened before use. Default is 0.

out : ndarray, optional If provided, the destination to place the result. The shape must be correct, matching that of what concatenate would have returned if no out argument were specified.

Returns: res : ndarray The concatenated array.

In your case, check what feats_sc contains.

You can debug using pdb

python -m pdb <your-code>.py
(pdb) b fullpath/to/your-code.py:line-number-to-break
(pdb) c

• c will continue until break point in encountered
• n will move to next line
• b is to set break point
• q is to quit

Answer 3

Just to make it clearer, running the following piece of code throws the same ValueError: need at least one array to concatenate error.

import numpy as np
feats_sc = np.array([])
feats_sc = np.concatenate(feats_sc, axis=1)

whereas the following code does not.

import numpy as np
feats_sc = np.array(([[0, 1, 2], [3, 4, 5]], [[6, 7, 8], [1 ,2 ,3]]))
feats_sc = np.concatenate(feats_sc, axis=1)

The reason is that in the former, the numpy array is empty, and in the latter, it is not.