v0.5.1: adds several datasets from the WILDS benchmark

KaiyangZhou · KaiyangZhou · commit 7f7eab8e22f6 · 2022-06-20T18:57:11.000+08:00
diff --git a/README.md b/README.md
@@ -20,6 +20,7 @@ A drawback of Dassl is that it doesn't (yet? hmm) support distributed multi-GPU
 We don't provide detailed documentations for Dassl, unlike another [project](https://kaiyangzhou.github.io/deep-person-reid/) of ours. This is because Dassl is developed for research purpose and as a researcher, we think it's important to be able to read source code and we highly encourage you to do so---definitely not because we are lazy. :-)
 
 ## What's new
+- Jun 2022: Adds three datasets from the [WILDS](https://wilds.stanford.edu/) benchmark: IWildCam, FMoW and Camelyon17.
 - May 2022: A new domain generalization method [DDG](https://arxiv.org/abs/2205.13913) developed by [Zhishu Sun](https://github.com/siaimes) and to appear at IJCAI'22 is added to this repo. See [here](https://github.com/MetaVisionLab/DDG) for more details.
 - Mar 2022: A new domain generalization method [EFDM](https://arxiv.org/abs/2203.07740) developed by [Yabin Zhang (PolyU)](https://ybzh.github.io/) and to appear at CVPR'22 is added to this repo. See [here](https://github.com/KaiyangZhou/Dassl.pytorch/pull/36) for more details.
 - Feb 2022: In case you don't know, a class in the painting domain of DomainNet (the official splits) only has test images (no training images), which could affect performance. See section 4.a in our [paper](https://arxiv.org/abs/2003.07325) for more details.
diff --git a/configs/datasets/dg/camelyon17.yaml b/configs/datasets/dg/camelyon17.yaml
@@ -0,0 +1,6 @@
+INPUT:
+  SIZE: (224, 224)
+  TRANSFORMS: ["random_resized_crop", "random_flip", "normalize"]
+
+DATASET:
+  NAME: "Camelyon17"
diff --git a/configs/datasets/dg/fmow.yaml b/configs/datasets/dg/fmow.yaml
@@ -0,0 +1,6 @@
+INPUT:
+  SIZE: (224, 224)
+  TRANSFORMS: ["random_resized_crop", "random_flip", "normalize"]
+
+DATASET:
+  NAME: "FMoW"
diff --git a/configs/datasets/dg/iwildcam.yaml b/configs/datasets/dg/iwildcam.yaml
@@ -0,0 +1,6 @@
+INPUT:
+  SIZE: (224, 224)
+  TRANSFORMS: ["random_resized_crop", "random_flip", "normalize"]
+
+DATASET:
+  NAME: "IWildCam"
diff --git a/dassl/__init__.py b/dassl/__init__.py
@@ -13,6 +13,6 @@
 }
 """
 
-__version__ = "0.5.0"
+__version__ = "0.5.1"
 __author__ = "Kaiyang Zhou"
 __homepage__ = "https://kaiyangzhou.github.io/"
diff --git a/dassl/data/datasets/base_dataset.py b/dassl/data/datasets/base_dataset.py
@@ -60,7 +60,6 @@ def __init__(self, train_x=None, train_u=None, val=None, test=None):
         self._train_u = train_u  # unlabeled training data (optional)
         self._val = val  # validation data (optional)
         self._test = test  # test data
-
         self._num_classes = self.get_num_classes(train_x)
         self._lab2cname, self._classnames = self.get_lab2cname(train_x)
 
diff --git a/dassl/data/datasets/dg/__init__.py b/dassl/data/datasets/dg/__init__.py
@@ -1,5 +1,6 @@
 from .pacs import PACS
 from .vlcs import VLCS
+from .wilds import *
 from .cifar_c import CIFAR10C, CIFAR100C
 from .digits_dg import DigitsDG
 from .digit_single import DigitSingle
diff --git a/dassl/data/datasets/dg/wilds/__init__.py b/dassl/data/datasets/dg/wilds/__init__.py
@@ -0,0 +1,3 @@
+from .fmow import FMoW
+from .iwildcam import IWildCam
+from .camelyon17 import Camelyon17
diff --git a/dassl/data/datasets/dg/wilds/camelyon17.py b/dassl/data/datasets/dg/wilds/camelyon17.py
@@ -0,0 +1,24 @@
+from dassl.data.datasets import DATASET_REGISTRY
+
+from .wilds_base import WILDSBase
+
+
+@DATASET_REGISTRY.register()
+class Camelyon17(WILDSBase):
+    """Tumor tissue recognition.
+
+    2 classes (whether a given region of tissue contains tumor tissue).
+
+    Reference:
+        - Bandi et al. "From detection of individual metastases to classification of lymph
+        node status at the patient level: the CAMELYON17 challenge." TMI 2021.
+        - Koh et al. "Wilds: A benchmark of in-the-wild distribution shifts." ICML 2021.
+    """
+
+    dataset_dir = "camelyon17_v1.0"
+
+    def __init__(self, cfg):
+        super().__init__(cfg)
+
+    def load_classnames(self):
+        return {0: "healthy tissue", 1: "tumor tissue"}
diff --git a/dassl/data/datasets/dg/wilds/fmow.py b/dassl/data/datasets/dg/wilds/fmow.py
@@ -0,0 +1,57 @@
+import os.path as osp
+
+from dassl.data.datasets import DATASET_REGISTRY
+
+from .wilds_base import WILDSBase
+
+CATEGORIES = [
+    "airport", "airport_hangar", "airport_terminal", "amusement_park",
+    "aquaculture", "archaeological_site", "barn", "border_checkpoint",
+    "burial_site", "car_dealership", "construction_site", "crop_field", "dam",
+    "debris_or_rubble", "educational_institution", "electric_substation",
+    "factory_or_powerplant", "fire_station", "flooded_road", "fountain",
+    "gas_station", "golf_course", "ground_transportation_station", "helipad",
+    "hospital", "impoverished_settlement", "interchange", "lake_or_pond",
+    "lighthouse", "military_facility", "multi-unit_residential",
+    "nuclear_powerplant", "office_building", "oil_or_gas_facility", "park",
+    "parking_lot_or_garage", "place_of_worship", "police_station", "port",
+    "prison", "race_track", "railway_bridge", "recreational_facility",
+    "road_bridge", "runway", "shipyard", "shopping_mall",
+    "single-unit_residential", "smokestack", "solar_farm", "space_facility",
+    "stadium", "storage_tank", "surface_mine", "swimming_pool", "toll_booth",
+    "tower", "tunnel_opening", "waste_disposal", "water_treatment_facility",
+    "wind_farm", "zoo"
+]
+
+
+@DATASET_REGISTRY.register()
+class FMoW(WILDSBase):
+    """Satellite imagery classification.
+
+    62 classes (building or land use categories).
+
+    Reference:
+        - Christie et al. "Functional Map of the World." CVPR 2018.
+        - Koh et al. "Wilds: A benchmark of in-the-wild distribution shifts." ICML 2021.
+    """
+
+    dataset_dir = "fmow_v1.1"
+
+    def __init__(self, cfg):
+        super().__init__(cfg)
+
+    def get_image_path(self, dataset, idx):
+        idx = dataset.full_idxs[idx]
+        image_name = f"rgb_img_{idx}.png"
+        image_path = osp.join(self.dataset_dir, "images", image_name)
+        return image_path
+
+    def get_domain(self, dataset, idx):
+        # number of regions: 5 or 6
+        # number of years: 16
+        region_id = int(dataset.metadata_array[idx][0])
+        year_id = int(dataset.metadata_array[idx][1])
+        return region_id*16 + year_id
+
+    def load_classnames(self):
+        return {i: cat for i, cat in enumerate(CATEGORIES)}
diff --git a/dassl/data/datasets/dg/wilds/iwildcam.py b/dassl/data/datasets/dg/wilds/iwildcam.py
@@ -0,0 +1,32 @@
+import os.path as osp
+import pandas as pd
+
+from dassl.data.datasets import DATASET_REGISTRY
+
+from .wilds_base import WILDSBase
+
+
+@DATASET_REGISTRY.register()
+class IWildCam(WILDSBase):
+    """Animal species recognition.
+
+    182 classes (species).
+
+    Reference:
+        - Beery et al. "The iwildcam 2021 competition dataset." arXiv 2021.
+        - Koh et al. "Wilds: A benchmark of in-the-wild distribution shifts." ICML 2021.
+    """
+
+    dataset_dir = "iwildcam_v2.0"
+
+    def __init__(self, cfg):
+        super().__init__(cfg)
+
+    def get_image_path(self, dataset, idx):
+        image_name = dataset._input_array[idx]
+        image_path = osp.join(self.dataset_dir, "train", image_name)
+        return image_path
+
+    def load_classnames(self):
+        df = pd.read_csv(osp.join(self.dataset_dir, "categories.csv"))
+        return dict(df["name"])
diff --git a/dassl/data/datasets/dg/wilds/wilds_base.py b/dassl/data/datasets/dg/wilds/wilds_base.py
@@ -0,0 +1,106 @@
+import pickle
+import os.path as osp
+from wilds import get_dataset as wilds_get_dataset
+
+from dassl.data.datasets import Datum, DatasetBase
+
+
+class WILDSBase(DatasetBase):
+
+    dataset_dir = ""
+    relabel_domain = True
+
+    def __init__(self, cfg):
+        root = osp.abspath(osp.expanduser(cfg.DATASET.ROOT))
+        name = self.dataset_dir.split("_")[0]
+        self.dataset_dir = osp.join(root, self.dataset_dir)
+        self.preloaded = osp.join(self.dataset_dir, "zhou_preloaded.pkl")
+
+        self.label_to_name = self.load_classnames()
+        assert isinstance(self.label_to_name, dict)
+
+        if osp.exists(self.preloaded):
+            with open(self.preloaded, "rb") as file:
+                dataset = pickle.load(file)
+                train = dataset["train"]
+                val = dataset["val"]
+                test = dataset["test"]
+        else:
+            dataset = wilds_get_dataset(
+                dataset=name, root_dir=root, download=False
+            )
+            subset_train = dataset.get_subset("train")
+            subset_val = dataset.get_subset("val")
+            subset_test = dataset.get_subset("test")
+
+            train = self.read_data(subset_train)
+            val = self.read_data(subset_val)
+            test = self.read_data(subset_test)
+
+            # Save time for data loading next time
+            preloaded = {"train": train, "val": val, "test": test}
+            with open(self.preloaded, "wb") as file:
+                pickle.dump(preloaded, file, protocol=pickle.HIGHEST_PROTOCOL)
+
+        # Few-shot learning
+        k = cfg.DATASET.NUM_SHOTS
+        if k > 0:
+            groups = self.split_dataset_by_domain(train)
+            groups = list(groups.values())
+            groups = self.generate_fewshot_dataset(*groups, num_shots=k)
+            train = []
+            for group in groups:
+                train.extend(group)
+
+        super().__init__(train_x=train, val=val, test=test)
+
+    def load_classnames(self):
+        raise NotImplementedError
+
+    def get_image_path(self, dataset, idx):
+        image_name = dataset._input_array[idx]
+        image_path = osp.join(self.dataset_dir, image_name)
+        return image_path
+
+    def get_label(self, dataset, idx):
+        return int(dataset.y_array[idx])
+
+    def get_domain(self, dataset, idx):
+        return int(dataset.metadata_array[idx][0])
+
+    def read_data(self, subset):
+        items = []
+        indices = subset.indices
+        dataset = subset.dataset
+
+        for idx in indices:
+            image_path = self.get_image_path(dataset, idx)
+            label = self.get_label(dataset, idx)
+            domain = self.get_domain(dataset, idx)
+            classname = self.label_to_name[label]
+            item = Datum(
+                impath=image_path,
+                label=label,
+                domain=domain,
+                classname=classname
+            )
+            items.append(item)
+
+        if self.relabel_domain:
+            domains = set([item.domain for item in items])
+            mapping = {domain: i for i, domain in enumerate(domains)}
+
+            items_new = []
+
+            for item in items:
+                item_new = Datum(
+                    impath=item.impath,
+                    label=item.label,
+                    domain=mapping[item.domain],
+                    classname=item.classname
+                )
+                items_new.append(item_new)
+
+            return items_new
+
+        return items
diff --git a/requirements.txt b/requirements.txt
@@ -8,3 +8,6 @@ future
 scipy
 scikit-learn
 tqdm
+ftfy
+regex
+wilds==1.2.2

Original file line number	Diff line number	Diff line change
`@@ -13,6 +13,6 @@`
`13`	`13`	`}`
`14`	`14`	`"""`
`15`	`15`
`16`		`-__version__ = "0.5.0"`
	`16`	`+__version__ = "0.5.1"`
`17`	`17`	`__author__ = "Kaiyang Zhou"`
`18`	`18`	`__homepage__ = "https://kaiyangzhou.github.io/"`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+from .fmow import FMoW`
	`2`	`+from .iwildcam import IWildCam`
	`3`	`+from .camelyon17 import Camelyon17`