How to handle imbalanced classes

60

Hi

This is how i am calculating weight per sample for weight_Sampler.

from tqdm.notebook import tqdm
import glob,json

def calculate_sample_weights(json_dir,su2id_json):

    json_files = sorted(glob.glob(os.path.join(json_dir,"*.json")))[:10000]
    sub2id = json.load(open(su2id_json,'r'))

    target = []
    for idx in tqdm(json_files):
        data = json.load(open(idx,'r'))
        label = data['subject']
        y = int(sub2id[label])
        target.append(y)

    class_sample_count = np.array(
        [len(np.where(target == t)[0]) for t in np.arange(45)])


    weight = 1. / (class_sample_count+1e-6)
    samples_weight = np.array([weight[t] for t in target])
    samples_weight = torch.from_numpy(samples_weight)
    samples_weight = samples_weight.double()

    return samples_weight