- extract features now returns receptive field centers

- wrote matching function (currently ransac runs on cpu single core - kinda slow, kd tree is multithreaded and fast)
- matching works, tested on one of the query images, returns itself first, then the query images that show similar content then a few others rest is rejected

Todos: matching parameters (thresholds, trails etc. need estimation)
clean up
incorperate into experiment structure
process output information
parent 97df86b4
......@@ -4,7 +4,7 @@ import torch
import torchvision
import copy
from Training.layers import Flatten, SpatialAttention2d, WeightedSum2d, __gamma_rescale__
from Training.utils import get_receptive_boxes, get_top_k_index
from Training.utils import get_receptive_boxes, get_receptive_centers, get_top_k_index
LAYER_3_OUT_CHANNELS = 1024
LAYER_4_OUT_CHANNELS = 2048
SCALES = [2, 1.4142, 1, 0.7071, 0.5, 0.3536, 0.25]
......@@ -171,8 +171,6 @@ class Delf(torch.nn.Module):
features = features.detach().cpu()
# calculate the receptive boxes in original scale
receptive_boxes = get_receptive_boxes(features.size(2), features.size(3), scale)
# print(receptive_boxes)
# print(receptive_boxes.size())
# flatten h,w dimensions so we can append values from different scales
# spatial information can be inferred from the receptive boxes
features = features.view(features.size(1), -1).t()
......@@ -195,8 +193,9 @@ class Delf(torch.nn.Module):
keep = get_top_k_index(all_scale_scores, all_scale_boxes.cuda(), iou_threshold=0.8, k=1000)
# index select the best boxes
all_scale_boxes = all_scale_boxes[keep]
all_scale_centers = get_receptive_centers(all_scale_boxes)
all_scale_features = all_scale_features[keep]
return all_scale_features, all_scale_boxes
return all_scale_features, all_scale_centers
def __load_weights_from__(module_dict, load_dict, module_names):
......
......@@ -2,6 +2,7 @@ from Training.delf import Delf
from Training.dataloader import get_data_loaders, get_path_data_loader
from sklearn.decomposition import PCA
from pathlib import Path
from Training.utils import score_match
import torch
import numpy as np
import uuid
......@@ -379,16 +380,15 @@ class ExperimentManager:
# get all image info from loader
image_input, label, path = batch
image_input = image_input.to(self.device)
# TODO make model return box centers instead of boxes
# get features and rf info from model
features, boxes = model.extract_features(image_input)
features, rf_centers = model.extract_features(image_input)
# Delf says normalize before pca
features = torch.nn.functional.normalize(features, p=2, dim=1)
if pca:
# if we already have pca calculated we can apply it now and save space
features = pca.transform(features.numpy())
# create data entry for the image
index_extraction_list.append((Path(path[0]).stem, label, features, boxes.numpy()))
index_extraction_list.append((Path(path[0]).stem, label, features, rf_centers.numpy()))
if use_index_for_pca:
# if we still have to calculate pca: accumulate features
if accumulated_features is None:
......@@ -401,8 +401,25 @@ class ExperimentManager:
pca = calculate_pca(accumulated_features, Path("pca.pkl"))
index_extraction_list = [(entry[0], entry[1], pca.transform(entry[2].numpy()), entry[3]) for entry in index_extraction_list]
print(index_extraction_list[5][2].shape)
query_dataset = Path(query_dataset)
query_data_loader = get_path_data_loader(query_dataset)
for ind, batch in enumerate(query_data_loader):
# get all image info from loader
image_input, label, path = batch
filename = Path(path[0]).stem
image_input = image_input.to(self.device)
# get features and rf info from model
features, rf_centers = model.extract_features(image_input)
# Delf says normalize before pca
features = torch.nn.functional.normalize(features, p=2, dim=1)
# apply pca
features = pca.transform(features.numpy())
rf_centers = rf_centers.numpy()
now= time.time()
for index_representation in index_extraction_list:
inliers = score_match(index_representation[2], features, index_representation[3], rf_centers)
print(f"query image {filename}, index image{index_representation[0]} inliers {inliers}")
print(time.time()-now)
def calculate_pca(data, save_path=None):
pca = PCA(n_components=PCA_COMPONENTS, whiten=True)
......@@ -440,6 +457,6 @@ def check_experiment_wide_parameter(parameter, parameter_name, required_type, al
torch.backends.cudnn.benchmark = True
exp = ExperimentManager("variable_target_layer", {"retrieval"}, "../../Datasets/Oxford/index", load_from={"retrieval":"../Experiments/variable target layer/keypoints/5db43e8d_dbb65c50.pth"}).perform_retrieval("../Experiments/variable target layer/keypoints/5db43e8d_dbb65c50.pth", "../../Datasets/Oxford/index", "asd", pca_dataset="../../Datasets/Oxford/index", pca_save="pca.pkl")
exp = ExperimentManager("variable_target_layer", {"retrieval"}, "../../Datasets/Oxford/index", load_from={"retrieval":"../Experiments/variable target layer/keypoints/5db43e8d_dbb65c50.pth"}).perform_retrieval("../Experiments/variable target layer/keypoints/5db43e8d_dbb65c50.pth", "../../Datasets/Oxford/index", "../../Datasets/Oxford/query", pca_load="pca.pkl")
#exp = ExperimentManager("variable target layer", {"finetuning","keypoints"}, "../Datasets/Landmarks", epochs=1)
#exp = ExperimentManager("variable target layer", {"keypoints"}, "../Datasets/Landmarks", epochs=1, load_from={"keypoints":"Experiments/variable target layer/finetuning/5db43e8d.pth"})
import torch
import torchvision
import numpy as np
from skimage.measure import ransac
from scipy.spatial import cKDTree
from skimage.transform import AffineTransform
#from sklearn.decomposition import PCA
#from Training.delf import Delf
#import pickle
SCALES = [2, 1.4142, 1, 0.7071, 0.5, 0.3536, 0.25]
RF_VALUES = {"layer3": (267, 16, 133), "layer4": (427, 32, 213)}
KD_TREE_DISTANCE_THRESHOLD = 4.5 # has to be evaluated
RANSAC_MIN_SAMPLES = 3
RANSAC_NUM_TRAILS = 1000
RANSAC_RESIDUAL_THRESHOLD = 20
# TODO will be moved to experiment when model/ dataloaders are available
"""
......@@ -95,6 +106,44 @@ def extract_features(self, stage, pca=None):
print(l_pca.transform(test_sample))
# TODO tf repo says l2 normalize features after
"""
import time
def score_match(index_features, query_features, index_locations, query_locations):
index_tree = cKDTree(index_features)
distances, indices = index_tree.query(
query_features, distance_upper_bound=KD_TREE_DISTANCE_THRESHOLD, n_jobs=-1)
#print(distances)
#print(indices)
# Filter out features with no close neighbours
cleaned_query_locations = np.array([
query_locations[i, ] for i in range(query_locations.shape[0])
if indices[i] != index_locations.shape[0] # KD-Tree returns number of entries as index if no match is found
# within threshold
])
cleaned_index_locations = np.array([
index_locations[indices[i], ] for i in range(query_locations.shape[0])
if indices[i] != index_locations.shape[0]
])
#print(cleaned_index_locations)
#print(cleaned_query_locations)
#print(cleaned_query_locations.shape)
if cleaned_query_locations.shape[0] <= RANSAC_MIN_SAMPLES:
return 0
# Perform geometric verification using RANSAC.
# Ransac currently takes ~35 times longer than kd tree
model_robust, inliers = ransac(
(cleaned_index_locations, cleaned_query_locations),
AffineTransform,
min_samples=RANSAC_MIN_SAMPLES,
residual_threshold=RANSAC_RESIDUAL_THRESHOLD,
max_trials=RANSAC_NUM_TRAILS)
if model_robust is None:
return 0
#print(sum(inliers))
#print(model_robust)
return sum(inliers)
def get_receptive_boxes(height, width, scale, target_layer="layer3"):
"""
calculates the receptive boxes for a feature map, based on its height and width and the RF parameters of the
......@@ -112,6 +161,12 @@ def get_receptive_boxes(height, width, scale, target_layer="layer3"):
return boxes
def get_receptive_centers(boxes):
min_xy = boxes.index_select(dim=1, index=torch.LongTensor([0, 1]))
max_xy = boxes.index_select(dim=1, index=torch.LongTensor([2, 3]))
return (min_xy + max_xy) / 2
def get_coords(h, w):
"""
Builds a coordinate system based on height and width on the input, with values for x_min, y_min, x_max, y_max
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment