PatternFlow/recognition/45307915/dataset.py at 9cef7cb076f91c0f1fa2c5c044a477c3d63f2f35 · shakes76/PatternFlow · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
import tensorflow as tf

TRAIN_IMAGES_PATH = "./ISIC-2017_Training_Data/*.jpg"
TRAIN_MASKS_PATH = "./ISIC-2017_Training_Part1_GroundTruth/*.png"

TEST_IMAGES_PATH = "./ISIC-2017_Test_v2_Data/*.jpg"
TEST_MASKS_PATH = "./ISIC-2017_Test_v2_Part1_GroundTruth/*.png"

VALIDATE_IMAGES_PATH = "./ISIC-2017_Validation_Data/*.jpg"
VALIDATE_MASKS_PATH = "./ISIC-2017_Validation_Part1_GroundTruth/*.png"

IMAGE_HEIGHT = 256
IMAGE_WIDTH = 256

class DataLoader():

    def __init__(self, train_images_path=TRAIN_IMAGES_PATH, train_masks_path=TRAIN_MASKS_PATH,
                 test_images_path=TEST_IMAGES_PATH, test_masks_path=TEST_MASKS_PATH,
                 validate_images_path=VALIDATE_IMAGES_PATH, validate_masks_path=VALIDATE_MASKS_PATH,
                 img_width=IMAGE_WIDTH, img_height=IMAGE_HEIGHT):
        """
        Create a new Data Loader instance to load in the training, testing and validation data

        Parameters:
            images_path (str): Path of the dataset images
            masks_path (str):  Path of the dataset masks
            img_width (int): Image Width
            img_height (int): Image Height
        """
        self.train_images_path = train_images_path
        self.train_masks_path = train_masks_path

        self.test_images_path = test_images_path
        self.test_masks_path = test_masks_path

        self.validate_images_path = validate_images_path
        self.validate_masks_path = validate_masks_path

        self.img_width = img_width
        self.img_height = img_height

    def preprocessImages(self, filenames):
        """
        Load and preprocess the image files.

        Image loading code from TensorFlow
        https://www.tensorflow.org/api_docs/python/tf/io/read_file

        Parameters:
            filenames (tf.string): names of all image files

        Return:
            tf.Dataset: Dataset containing all the image file data

        """
        raw = tf.io.read_file(filenames)
        images = tf.io.decode_jpeg(raw, channels=3)

        #resize images
        images = tf.image.resize(images, [self.img_height, self.img_width])

        #Normalise
        images = images / 255.

        return images


    def preprocessMasks(self, filenames):
        """
        Load and preprocess the mask files.

        Image loading code from TensorFlow
        https://www.tensorflow.org/api_docs/python/tf/io/read_file

        Parameters:
            filenames (tf.string): names of all mask files

        Return:
            tf.Dataset: Dataset containing all the mask file data

        """
        raw = tf.io.read_file(filenames)
        images = tf.io.decode_png(raw, channels=1)

        #resize images
        images = tf.image.resize(images, [self.img_height, self.img_width])

        #Normalise
        images = images / 255.

        #Threshold image to 0-1
        images = tf.where(images > 0.5, 1.0, 0.0)

        return images

    def loadData(self):
        """
        Loads and prepocesses all the image and mask data, for the training, testing and validation datasets

        Return:
            (tf.Dataset, tf.Dataset, tf.Dataset): 3 datasets containing all the image and mask data
        """

        #train
        train_image_data = tf.data.Dataset.list_files(self.train_images_path, shuffle=False)
        train_processed_images = train_image_data.map(self.preprocessImages)

        train_mask_data = tf.data.Dataset.list_files(self.train_masks_path, shuffle=False)
        train_processed_masks = train_mask_data.map(self.preprocessMasks)

        train_dataset = tf.data.Dataset.zip((train_processed_images, train_processed_masks))

        #test
        test_image_data = tf.data.Dataset.list_files(self.test_images_path, shuffle=False)
        test_processed_images = test_image_data.map(self.preprocessImages)

        test_mask_data = tf.data.Dataset.list_files(self.test_masks_path, shuffle=False)
        test_processed_masks = test_mask_data.map(self.preprocessMasks)

        test_dataset = tf.data.Dataset.zip((test_processed_images, test_processed_masks))

        #validate
        validate_image_data = tf.data.Dataset.list_files(self.validate_images_path, shuffle=False)
        validate_processed_images = validate_image_data.map(self.preprocessImages)

        validate_mask_data = tf.data.Dataset.list_files(self.validate_masks_path, shuffle=False)
        validate_processed_masks = validate_mask_data.map(self.preprocessMasks)

        validate_dataset = tf.data.Dataset.zip((validate_processed_images, validate_processed_masks))

        return train_dataset, test_dataset, validate_dataset