change pipeline such that pca is immediately calculated when the max number of...

change pipeline such that pca is immediately calculated when the max number of features has been extracted. New features will instantly be scaled down to number of pca dims. This is to save memory. 5000 imgs of oxford with 1000 descriptors per image each containing 1024 dims saved as 32 byte floats would amount to ~164 GB of memory required. + Calculating PCA Matrix on this would require much more.

Also now standardizing features (0 mean, 1 std) before calculating/applying pca. This is identical to calculating PCA on correlation instead of covariance matrix.
parent 6ce1a0c0
......@@ -42,7 +42,7 @@ class ImageFolderWithPaths(torchvision.datasets.ImageFolder):
def get_path_data_loader(dataset_path, batch_size=1, num_workers=4):
dataset = ImageFolderWithPaths(root=dataset_path, transform=transforms.ToTensor())
return torch.utils.data.DataLoader(dataset=dataset, batch_size=batch_size, num_workers=num_workers, shuffle=False)
return torch.utils.data.DataLoader(dataset=dataset, batch_size=batch_size, num_workers=num_workers, shuffle=True)
"""
from pathlib import Path
import time
......
......@@ -198,7 +198,7 @@ class Delf(torch.nn.Module):
all_scale_boxes = all_scale_boxes[keep]
all_scale_centers = get_receptive_centers(all_scale_boxes)
all_scale_features = all_scale_features[keep]
return all_scale_features, all_scale_centers
return all_scale_features.numpy(), all_scale_centers.numpy()
def __load_weights_from__(module_dict, load_dict, module_names):
......
......@@ -18,6 +18,8 @@ RANDOM_CROP_SITE_KEYPOINT = 720
BATCH_PRINT_FREQ = 20
MAX_PCA_FEATURES = 500000
NUM_VIS_MATCH = 10
class ExperimentManager:
"""
ExperimentManager: Controls the various stages of the experiments. parameters can be provided as single values or
......@@ -349,6 +351,9 @@ class ExperimentManager:
with torch.no_grad():
accumulated_features = None
image_info = {}
pca = None
feature_mean = None
feature_std = None
data_loader = get_path_data_loader(images, num_workers=self.num_workers["retrieval"])
for ind, batch in enumerate(data_loader):
# get all image info from loader
......@@ -358,32 +363,64 @@ class ExperimentManager:
# get features and rf info from model
features, rf_centers = model.extract_features(image_input)
# Delf says normalize before pca
features = torch.nn.functional.normalize(features, p=2, dim=1)
# accumulate features for pca_calculations
if accumulated_features is None:
accumulated_features = features
features = normalize(features, norm='l2', axis=1)
if pca is not None:
features -= feature_mean
features /= feature_std
features = pca.transform(features)
if self.use_retrieval_normalization:
features = normalize(features, norm='l2', axis=1)
else:
accumulated_features = torch.cat((accumulated_features, features), dim=0)
# accumulate features for pca_calculations
if accumulated_features is None:
accumulated_features = features
elif len(accumulated_features) < MAX_PCA_FEATURES:
accumulated_features = np.concatenate((accumulated_features, features), axis=0)
# we have enough features to calculated pca
else:
print("Maximum number of features for PCA calculation reached.")
print("Training PCA Matrix")
# zero mean and normalize std
feature_mean = np.mean(accumulated_features, axis=0)
feature_std = np.std(accumulated_features, axis=0)
accumulated_features -= feature_mean
accumulated_features /= feature_std
pca = calculate_pca(accumulated_features, stage_path.joinpath(
f"{self.load_paths['retrieval'].stem}_{self.dataset['retrieval'].stem}.pca"), self.pca_log)
# update all features we have already calculated
for filename in image_info:
features, rf_centers = image_info[filename]
features -= feature_mean
features /= feature_std
features = pca.transform(features)
if self.use_retrieval_normalization:
features = normalize(features, norm='l2', axis=1)
image_info[filename] = [features, rf_centers]
# save the feature representation
image_info[filename] = [features, rf_centers]
print(f"Gathered features of [{ind+1}/{len(data_loader)}] images")
accumulated_features = accumulated_features.numpy()
# if we have too many features for pca we choose a random subset
if len(accumulated_features) > MAX_PCA_FEATURES:
print(f"Number of gathered features exceeds maximum of {MAX_PCA_FEATURES}. Selected random"
f" subset for PCA calculation!")
accumulated_features = np.random.permutation(accumulated_features)[:MAX_PCA_FEATURES]
# calculate and evaluate the pca matrix
print("Training PCA Matrix")
pca = calculate_pca(accumulated_features, stage_path.joinpath(
f"{self.load_paths['retrieval'].stem}_{self.dataset['retrieval'].stem}.pca"), self.pca_log)
for filename in image_info:
features, rf_centers = image_info[filename]
rf_centers = rf_centers.numpy()
features = pca.transform(features.numpy())
if self.use_retrieval_normalization:
features = normalize(features, norm='l2', axis=1)
image_info[filename] = [features, rf_centers]
# if we have not reached a critical number of features after processing all images
if pca is None:
print("Training PCA Matrix")
# zero mean, normalize std
feature_mean = np.mean(accumulated_features, axis=0)
feature_std = np.std(accumulated_features, axis=0)
accumulated_features -= feature_mean
accumulated_features /= feature_std
# calculated pca
pca = calculate_pca(accumulated_features, stage_path.joinpath(
f"{self.load_paths['retrieval'].stem}_{self.dataset['retrieval'].stem}.pca"), self.pca_log)
# update all features
for filename in image_info:
features, rf_centers = image_info[filename]
features -= feature_mean
features /= feature_std
features = pca.transform(features)
if self.use_retrieval_normalization:
features = normalize(features, norm='l2', axis=1)
image_info[filename] = [features, rf_centers]
print("Feature processing completed! Begin matching!")
with query.open("r") as query_file:
query_filenames = json.load(query_file)["queries"]
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment