"""
CUB-200 Dataset Loader
** THIS DATASET NEEDS TO BE DOWNLOADED BEFORE BEING ABLE TO USE THE LOADER **
################################################################################
## DOWNLOAD INSTRUCTIONS
################################################################################
**** OPTION #1 *****
The simplest way to get the CUB dataset, is to download the pre-processed CUB
dataset by Koh et al. [CBM Paper]. This can be downloaded from their
public colab notebook at: https://worksheets.codalab.org/worksheets/0x362911581fcd4e048ddfd84f47203fd2.
You will need to download the original CUB dataset from that notebook (found
here: https://worksheets.codalab.org/bundles/0xd013a7ba2e88481bbc07e787f73109f5)
and the preprocessed "CUB_preprocessed" dataset (which can be directly accessed
here: https://worksheets.codalab.org/bundles/0x5b9d528d2101418b87212db92fea6683)
**** OPTION #2 *****
Follow the download the preprocess instructions found in Koh et al.'s original
repository here: https://github.com/yewsiang/ConceptBottleneck.
Specifically, here: https://github.com/yewsiang/ConceptBottleneck/blob/master/CUB/
################################################################################
[IMPORTANT] After downloading the files, they need to follow the following
structure:
This loader has been adapted/inspired by that of found in Koh et al.'s
repository (https://github.com/yewsiang/ConceptBottleneck/blob/master/CUB/cub_loader.py)
as well as in Espinosa Zarlenga and Barbiero's et al.'s repository
(https://github.com/mateoespinosa/cem).
"""
import numpy as np
import os
import pickle
import torch
import torchvision.transforms as transforms
from collections import defaultdict
from PIL import Image
from torch.utils.data import Dataset
########################################################
## GENERAL DATASET GLOBAL VARIABLES
########################################################
N_CLASSES = 200
# CAN BE OVERWRITTEN WITH AN ENV VARIABLE CUB_DIR
CUB_DIR = os.environ.get("CUB_DIR", './CUB200/')
#########################################################
## CONCEPT INFORMATION REGARDING CUB
#########################################################
# CUB Class names
CLASS_NAMES = [
"Black_footed_Albatross",
"Laysan_Albatross",
"Sooty_Albatross",
"Groove_billed_Ani",
"Crested_Auklet",
"Least_Auklet",
"Parakeet_Auklet",
"Rhinoceros_Auklet",
"Brewer_Blackbird",
"Red_winged_Blackbird",
"Rusty_Blackbird",
"Yellow_headed_Blackbird",
"Bobolink",
"Indigo_Bunting",
"Lazuli_Bunting",
"Painted_Bunting",
"Cardinal",
"Spotted_Catbird",
"Gray_Catbird",
"Yellow_breasted_Chat",
"Eastern_Towhee",
"Chuck_will_Widow",
"Brandt_Cormorant",
"Red_faced_Cormorant",
"Pelagic_Cormorant",
"Bronzed_Cowbird",
"Shiny_Cowbird",
"Brown_Creeper",
"American_Crow",
"Fish_Crow",
"Black_billed_Cuckoo",
"Mangrove_Cuckoo",
"Yellow_billed_Cuckoo",
"Gray_crowned_Rosy_Finch",
"Purple_Finch",
"Northern_Flicker",
"Acadian_Flycatcher",
"Great_Crested_Flycatcher",
"Least_Flycatcher",
"Olive_sided_Flycatcher",
"Scissor_tailed_Flycatcher",
"Vermilion_Flycatcher",
"Yellow_bellied_Flycatcher",
"Frigatebird",
"Northern_Fulmar",
"Gadwall",
"American_Goldfinch",
"European_Goldfinch",
"Boat_tailed_Grackle",
"Eared_Grebe",
"Horned_Grebe",
"Pied_billed_Grebe",
"Western_Grebe",
"Blue_Grosbeak",
"Evening_Grosbeak",
"Pine_Grosbeak",
"Rose_breasted_Grosbeak",
"Pigeon_Guillemot",
"California_Gull",
"Glaucous_winged_Gull",
"Heermann_Gull",
"Herring_Gull",
"Ivory_Gull",
"Ring_billed_Gull",
"Slaty_backed_Gull",
"Western_Gull",
"Anna_Hummingbird",
"Ruby_throated_Hummingbird",
"Rufous_Hummingbird",
"Green_Violetear",
"Long_tailed_Jaeger",
"Pomarine_Jaeger",
"Blue_Jay",
"Florida_Jay",
"Green_Jay",
"Dark_eyed_Junco",
"Tropical_Kingbird",
"Gray_Kingbird",
"Belted_Kingfisher",
"Green_Kingfisher",
"Pied_Kingfisher",
"Ringed_Kingfisher",
"White_breasted_Kingfisher",
"Red_legged_Kittiwake",
"Horned_Lark",
"Pacific_Loon",
"Mallard",
"Western_Meadowlark",
"Hooded_Merganser",
"Red_breasted_Merganser",
"Mockingbird",
"Nighthawk",
"Clark_Nutcracker",
"White_breasted_Nuthatch",
"Baltimore_Oriole",
"Hooded_Oriole",
"Orchard_Oriole",
"Scott_Oriole",
"Ovenbird",
"Brown_Pelican",
"White_Pelican",
"Western_Wood_Pewee",
"Sayornis",
"American_Pipit",
"Whip_poor_Will",
"Horned_Puffin",
"Common_Raven",
"White_necked_Raven",
"American_Redstart",
"Geococcyx",
"Loggerhead_Shrike",
"Great_Grey_Shrike",
"Baird_Sparrow",
"Black_throated_Sparrow",
"Brewer_Sparrow",
"Chipping_Sparrow",
"Clay_colored_Sparrow",
"House_Sparrow",
"Field_Sparrow",
"Fox_Sparrow",
"Grasshopper_Sparrow",
"Harris_Sparrow",
"Henslow_Sparrow",
"Le_Conte_Sparrow",
"Lincoln_Sparrow",
"Nelson_Sharp_tailed_Sparrow",
"Savannah_Sparrow",
"Seaside_Sparrow",
"Song_Sparrow",
"Tree_Sparrow",
"Vesper_Sparrow",
"White_crowned_Sparrow",
"White_throated_Sparrow",
"Cape_Glossy_Starling",
"Bank_Swallow",
"Barn_Swallow",
"Cliff_Swallow",
"Tree_Swallow",
"Scarlet_Tanager",
"Summer_Tanager",
"Artic_Tern",
"Black_Tern",
"Caspian_Tern",
"Common_Tern",
"Elegant_Tern",
"Forsters_Tern",
"Least_Tern",
"Green_tailed_Towhee",
"Brown_Thrasher",
"Sage_Thrasher",
"Black_capped_Vireo",
"Blue_headed_Vireo",
"Philadelphia_Vireo",
"Red_eyed_Vireo",
"Warbling_Vireo",
"White_eyed_Vireo",
"Yellow_throated_Vireo",
"Bay_breasted_Warbler",
"Black_and_white_Warbler",
"Black_throated_Blue_Warbler",
"Blue_winged_Warbler",
"Canada_Warbler",
"Cape_May_Warbler",
"Cerulean_Warbler",
"Chestnut_sided_Warbler",
"Golden_winged_Warbler",
"Hooded_Warbler",
"Kentucky_Warbler",
"Magnolia_Warbler",
"Mourning_Warbler",
"Myrtle_Warbler",
"Nashville_Warbler",
"Orange_crowned_Warbler",
"Palm_Warbler",
"Pine_Warbler",
"Prairie_Warbler",
"Prothonotary_Warbler",
"Swainson_Warbler",
"Tennessee_Warbler",
"Wilson_Warbler",
"Worm_eating_Warbler",
"Yellow_Warbler",
"Northern_Waterthrush",
"Louisiana_Waterthrush",
"Bohemian_Waxwing",
"Cedar_Waxwing",
"American_Three_toed_Woodpecker",
"Pileated_Woodpecker",
"Red_bellied_Woodpecker",
"Red_cockaded_Woodpecker",
"Red_headed_Woodpecker",
"Downy_Woodpecker",
"Bewick_Wren",
"Cactus_Wren",
"Carolina_Wren",
"House_Wren",
"Marsh_Wren",
"Rock_Wren",
"Winter_Wren",
"Common_Yellowthroat",
]
# Set of CUB attributes selected by Koh et al. [CBM Paper]
SELECTED_CONCEPTS = [
1,
4,
6,
7,
10,
14,
15,
20,
21,
23,
25,
29,
30,
35,
36,
38,
40,
44,
45,
50,
51,
53,
54,
56,
57,
59,
63,
64,
69,
70,
72,
75,
80,
84,
90,
91,
93,
99,
101,
106,
110,
111,
116,
117,
119,
125,
126,
131,
132,
134,
145,
149,
151,
152,
153,
157,
158,
163,
164,
168,
172,
178,
179,
181,
183,
187,
188,
193,
194,
196,
198,
202,
203,
208,
209,
211,
212,
213,
218,
220,
221,
225,
235,
236,
238,
239,
240,
242,
243,
244,
249,
253,
254,
259,
260,
262,
268,
274,
277,
283,
289,
292,
293,
294,
298,
299,
304,
305,
308,
309,
310,
311,
]
# Names of all CUB attributes
CONCEPT_SEMANTICS = [
"has_bill_shape::curved_(up_or_down)",
"has_bill_shape::dagger",
"has_bill_shape::hooked",
"has_bill_shape::needle",
"has_bill_shape::hooked_seabird",
"has_bill_shape::spatulate",
"has_bill_shape::all-purpose",
"has_bill_shape::cone",
"has_bill_shape::specialized",
"has_wing_color::blue",
"has_wing_color::brown",
"has_wing_color::iridescent",
"has_wing_color::purple",
"has_wing_color::rufous",
"has_wing_color::grey",
"has_wing_color::yellow",
"has_wing_color::olive",
"has_wing_color::green",
"has_wing_color::pink",
"has_wing_color::orange",
"has_wing_color::black",
"has_wing_color::white",
"has_wing_color::red",
"has_wing_color::buff",
"has_upperparts_color::blue",
"has_upperparts_color::brown",
"has_upperparts_color::iridescent",
"has_upperparts_color::purple",
"has_upperparts_color::rufous",
"has_upperparts_color::grey",
"has_upperparts_color::yellow",
"has_upperparts_color::olive",
"has_upperparts_color::green",
"has_upperparts_color::pink",
"has_upperparts_color::orange",
"has_upperparts_color::black",
"has_upperparts_color::white",
"has_upperparts_color::red",
"has_upperparts_color::buff",
"has_underparts_color::blue",
"has_underparts_color::brown",
"has_underparts_color::iridescent",
"has_underparts_color::purple",
"has_underparts_color::rufous",
"has_underparts_color::grey",
"has_underparts_color::yellow",
"has_underparts_color::olive",
"has_underparts_color::green",
"has_underparts_color::pink",
"has_underparts_color::orange",
"has_underparts_color::black",
"has_underparts_color::white",
"has_underparts_color::red",
"has_underparts_color::buff",
"has_breast_pattern::solid",
"has_breast_pattern::spotted",
"has_breast_pattern::striped",
"has_breast_pattern::multi-colored",
"has_back_color::blue",
"has_back_color::brown",
"has_back_color::iridescent",
"has_back_color::purple",
"has_back_color::rufous",
"has_back_color::grey",
"has_back_color::yellow",
"has_back_color::olive",
"has_back_color::green",
"has_back_color::pink",
"has_back_color::orange",
"has_back_color::black",
"has_back_color::white",
"has_back_color::red",
"has_back_color::buff",
"has_tail_shape::forked_tail",
"has_tail_shape::rounded_tail",
"has_tail_shape::notched_tail",
"has_tail_shape::fan-shaped_tail",
"has_tail_shape::pointed_tail",
"has_tail_shape::squared_tail",
"has_upper_tail_color::blue",
"has_upper_tail_color::brown",
"has_upper_tail_color::iridescent",
"has_upper_tail_color::purple",
"has_upper_tail_color::rufous",
"has_upper_tail_color::grey",
"has_upper_tail_color::yellow",
"has_upper_tail_color::olive",
"has_upper_tail_color::green",
"has_upper_tail_color::pink",
"has_upper_tail_color::orange",
"has_upper_tail_color::black",
"has_upper_tail_color::white",
"has_upper_tail_color::red",
"has_upper_tail_color::buff",
"has_head_pattern::spotted",
"has_head_pattern::malar",
"has_head_pattern::crested",
"has_head_pattern::masked",
"has_head_pattern::unique_pattern",
"has_head_pattern::eyebrow",
"has_head_pattern::eyering",
"has_head_pattern::plain",
"has_head_pattern::eyeline",
"has_head_pattern::striped",
"has_head_pattern::capped",
"has_breast_color::blue",
"has_breast_color::brown",
"has_breast_color::iridescent",
"has_breast_color::purple",
"has_breast_color::rufous",
"has_breast_color::grey",
"has_breast_color::yellow",
"has_breast_color::olive",
"has_breast_color::green",
"has_breast_color::pink",
"has_breast_color::orange",
"has_breast_color::black",
"has_breast_color::white",
"has_breast_color::red",
"has_breast_color::buff",
"has_throat_color::blue",
"has_throat_color::brown",
"has_throat_color::iridescent",
"has_throat_color::purple",
"has_throat_color::rufous",
"has_throat_color::grey",
"has_throat_color::yellow",
"has_throat_color::olive",
"has_throat_color::green",
"has_throat_color::pink",
"has_throat_color::orange",
"has_throat_color::black",
"has_throat_color::white",
"has_throat_color::red",
"has_throat_color::buff",
"has_eye_color::blue",
"has_eye_color::brown",
"has_eye_color::purple",
"has_eye_color::rufous",
"has_eye_color::grey",
"has_eye_color::yellow",
"has_eye_color::olive",
"has_eye_color::green",
"has_eye_color::pink",
"has_eye_color::orange",
"has_eye_color::black",
"has_eye_color::white",
"has_eye_color::red",
"has_eye_color::buff",
"has_bill_length::about_the_same_as_head",
"has_bill_length::longer_than_head",
"has_bill_length::shorter_than_head",
"has_forehead_color::blue",
"has_forehead_color::brown",
"has_forehead_color::iridescent",
"has_forehead_color::purple",
"has_forehead_color::rufous",
"has_forehead_color::grey",
"has_forehead_color::yellow",
"has_forehead_color::olive",
"has_forehead_color::green",
"has_forehead_color::pink",
"has_forehead_color::orange",
"has_forehead_color::black",
"has_forehead_color::white",
"has_forehead_color::red",
"has_forehead_color::buff",
"has_under_tail_color::blue",
"has_under_tail_color::brown",
"has_under_tail_color::iridescent",
"has_under_tail_color::purple",
"has_under_tail_color::rufous",
"has_under_tail_color::grey",
"has_under_tail_color::yellow",
"has_under_tail_color::olive",
"has_under_tail_color::green",
"has_under_tail_color::pink",
"has_under_tail_color::orange",
"has_under_tail_color::black",
"has_under_tail_color::white",
"has_under_tail_color::red",
"has_under_tail_color::buff",
"has_nape_color::blue",
"has_nape_color::brown",
"has_nape_color::iridescent",
"has_nape_color::purple",
"has_nape_color::rufous",
"has_nape_color::grey",
"has_nape_color::yellow",
"has_nape_color::olive",
"has_nape_color::green",
"has_nape_color::pink",
"has_nape_color::orange",
"has_nape_color::black",
"has_nape_color::white",
"has_nape_color::red",
"has_nape_color::buff",
"has_belly_color::blue",
"has_belly_color::brown",
"has_belly_color::iridescent",
"has_belly_color::purple",
"has_belly_color::rufous",
"has_belly_color::grey",
"has_belly_color::yellow",
"has_belly_color::olive",
"has_belly_color::green",
"has_belly_color::pink",
"has_belly_color::orange",
"has_belly_color::black",
"has_belly_color::white",
"has_belly_color::red",
"has_belly_color::buff",
"has_wing_shape::rounded-wings",
"has_wing_shape::pointed-wings",
"has_wing_shape::broad-wings",
"has_wing_shape::tapered-wings",
"has_wing_shape::long-wings",
"has_size::large_(16_-_32_in)",
"has_size::small_(5_-_9_in)",
"has_size::very_large_(32_-_72_in)",
"has_size::medium_(9_-_16_in)",
"has_size::very_small_(3_-_5_in)",
"has_shape::upright-perching_water-like",
"has_shape::chicken-like-marsh",
"has_shape::long-legged-like",
"has_shape::duck-like",
"has_shape::owl-like",
"has_shape::gull-like",
"has_shape::hummingbird-like",
"has_shape::pigeon-like",
"has_shape::tree-clinging-like",
"has_shape::hawk-like",
"has_shape::sandpiper-like",
"has_shape::upland-ground-like",
"has_shape::swallow-like",
"has_shape::perching-like",
"has_back_pattern::solid",
"has_back_pattern::spotted",
"has_back_pattern::striped",
"has_back_pattern::multi-colored",
"has_tail_pattern::solid",
"has_tail_pattern::spotted",
"has_tail_pattern::striped",
"has_tail_pattern::multi-colored",
"has_belly_pattern::solid",
"has_belly_pattern::spotted",
"has_belly_pattern::striped",
"has_belly_pattern::multi-colored",
"has_primary_color::blue",
"has_primary_color::brown",
"has_primary_color::iridescent",
"has_primary_color::purple",
"has_primary_color::rufous",
"has_primary_color::grey",
"has_primary_color::yellow",
"has_primary_color::olive",
"has_primary_color::green",
"has_primary_color::pink",
"has_primary_color::orange",
"has_primary_color::black",
"has_primary_color::white",
"has_primary_color::red",
"has_primary_color::buff",
"has_leg_color::blue",
"has_leg_color::brown",
"has_leg_color::iridescent",
"has_leg_color::purple",
"has_leg_color::rufous",
"has_leg_color::grey",
"has_leg_color::yellow",
"has_leg_color::olive",
"has_leg_color::green",
"has_leg_color::pink",
"has_leg_color::orange",
"has_leg_color::black",
"has_leg_color::white",
"has_leg_color::red",
"has_leg_color::buff",
"has_bill_color::blue",
"has_bill_color::brown",
"has_bill_color::iridescent",
"has_bill_color::purple",
"has_bill_color::rufous",
"has_bill_color::grey",
"has_bill_color::yellow",
"has_bill_color::olive",
"has_bill_color::green",
"has_bill_color::pink",
"has_bill_color::orange",
"has_bill_color::black",
"has_bill_color::white",
"has_bill_color::red",
"has_bill_color::buff",
"has_crown_color::blue",
"has_crown_color::brown",
"has_crown_color::iridescent",
"has_crown_color::purple",
"has_crown_color::rufous",
"has_crown_color::grey",
"has_crown_color::yellow",
"has_crown_color::olive",
"has_crown_color::green",
"has_crown_color::pink",
"has_crown_color::orange",
"has_crown_color::black",
"has_crown_color::white",
"has_crown_color::red",
"has_crown_color::buff",
"has_wing_pattern::solid",
"has_wing_pattern::spotted",
"has_wing_pattern::striped",
"has_wing_pattern::multi-colored",
]
# Generate a mapping containing all concept groups in CUB generated
# using a simple prefix tree
CONCEPT_GROUP_MAP = defaultdict(list)
for i, concept_name in enumerate(list(
np.array(CONCEPT_SEMANTICS)[SELECTED_CONCEPTS]
)):
group = concept_name[:concept_name.find("::")]
CONCEPT_GROUP_MAP[group].append(i)
# Definitions from CUB (certainties.txt)
# 1 not visible
# 2 guessing
# 3 probably
# 4 definitely
# Unc map represents a mapping from the discrete score to a "mental probability"
DEFAULT_UNC_MAP = [
{0: 0.5, 1: 0.5, 2: 0.5, 3:0.75, 4:1.0},
{0: 0.5, 1: 0.5, 2: 0.5, 3:0.75, 4:1.0},
]
##########################################################
## Helper Functions
##########################################################
def discrete_to_continuous_unc(unc_val, attr_label, unc_map):
'''
Yield a continuous prob representing discrete conf val
Inspired by CBM data processing
The selected probability should account for whether the concept is on or off
E.g., if a human is "probably" sure the concept is off
flip the prob in unc_map
'''
unc_val = unc_val.item()
attr_label = attr_label.item()
return float(unc_map[int(attr_label)][unc_val])
##########################################################
## Data Loaders
##########################################################
[docs]
class CUBDataset(Dataset):
"""
TODO
"""
[docs]
def __init__(
self,
split='train',
uncertain_concept_labels=False,
root=CUB_DIR,
path_transform=None,
sample_transform=None,
concept_transform=None,
label_transform=None,
uncertainty_based_random_labels=False,
unc_map=DEFAULT_UNC_MAP,
selected_concepts=None,
training_augment=True,
):
"""
TODO: Define different arguments
"""
if not (os.path.exists(root) and os.path.isdir(root)):
raise ValueError(
f'Provided CUB data directory "{root}" is not a valid or '
f'an existing directory.'
)
assert split in ['train', 'val', 'test'], (
f"CUB split must be in ['train', 'val', 'test'] but got '{split}'"
)
self.split = split
base_dir = os.path.join(root, 'class_attr_data_10')
self.pkl_file_path = os.path.join(base_dir, f'{split}.pkl')
self.name = 'CUB'
self.data = []
with open(self.pkl_file_path, 'rb') as f:
self.data.extend(pickle.load(f))
image_size = 299
if (split == 'train') and training_augment:
self.sample_transform = transforms.Compose([
transforms.ColorJitter(brightness=32/255, saturation=(0.5, 1.5)),
transforms.RandomResizedCrop(image_size),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(), #implicitly divides by 255
transforms.Normalize(mean = [0.5, 0.5, 0.5], std = [2, 2, 2]),
sample_transform or (lambda x: x),
])
else:
self.sample_transform = transforms.Compose([
transforms.CenterCrop(image_size),
transforms.ToTensor(), #implicitly divides by 255
transforms.Normalize(mean = [0.5, 0.5, 0.5], std = [2, 2, 2]),
sample_transform or (lambda x: x),
])
self.concept_transform = concept_transform or (lambda x: x)
self.label_transform = label_transform or (lambda x: x)
self.uncertain_concept_labels = uncertain_concept_labels
self.root = root
self.path_transform = path_transform
self.uncertainty_based_random_labels = uncertainty_based_random_labels
self.unc_map = unc_map
if selected_concepts is None:
selected_concepts = list(range(len(SELECTED_CONCEPTS)))
self.selected_concepts = selected_concepts
self.concept_names = self.concept_attr_names = list(
np.array(
CONCEPT_SEMANTICS
)[CONCEPT_SEMANTICS][selected_concepts]
)
self.task_names = self.task_attr_names = CLASS_NAMES
def __len__(self):
return len(self.data)
def __getitem__(self, idx):
img_data = self.data[idx]
img_path = img_data['img_path']
if self.path_transform is None:
# This is needed if the dataset is downloaded from the original
# CBM paper's repository/experiment code
img_path = img_path.replace(
'/juice/scr/scr102/scr/thaonguyen/CUB_supervision/datasets/',
self.root
)
try:
img = Image.open(img_path).convert('RGB')
except:
img_path_split = img_path.split('/')
img_path = '/'.join(
img_path_split[:2] + [self.split] + img_path_split[2:]
)
img = Image.open(img_path).convert('RGB')
else:
img = Image.open(self.path_transform(img_path)).convert('RGB')
class_label = self.label_transform(img_data['class_label'])
img = self.sample_transform(img)
if self.uncertain_concept_labels:
attr_label = img_data['uncertain_attribute_label']
else:
attr_label = img_data['attribute_label']
attr_label = self.concept_transform(
np.array(attr_label)[self.selected_concepts]
)
# We may want to randomly sample concept labels based on their provided
# annotator uncertainty
if self.uncertainty_based_random_labels:
discrete_unc_label = np.array(
img_data['attribute_certainty']
)[self.selected_concepts]
instance_attr_label = np.array(img_data['attribute_label'])
competencies = []
for (discrete_unc_val, hard_concept_val) in zip(
discrete_unc_label,
instance_attr_label,
):
competencies.append(
discrete_to_continuous_unc(
discrete_unc_val,
hard_concept_val,
self.unc_map,
)
)
attr_label = np.random.binomial(1, competencies)
return img, torch.FloatTensor(attr_label), class_label
[docs]
def concept_weights(self):
"""
Calculate class imbalance ratio for binary attribute labels
"""
imbalance_ratio = []
with open(self.pkl_file_path, 'rb') as f:
data = pickle.load(f)
n = len(data)
n_attr = len(data[0]['attribute_label'])
n_ones = [0] * n_attr
total = [n] * n_attr
for d in data:
labels = d['attribute_label']
for i in range(n_attr):
n_ones[i] += labels[i]
for j in range(len(n_ones)):
imbalance_ratio.append(total[j]/n_ones[j] - 1)
return np.array(imbalance_ratio)[self.selected_concepts]