- Matching results are now getting sorted by score

- Implemented matching visualization

Todos: matching parameters (thresholds, trails etc. need estimation)
clean up
incorperate into experiment structure
process output information
parent 80b5805c
......@@ -2,7 +2,7 @@ from Training.delf import Delf
from Training.dataloader import get_data_loaders, get_path_data_loader
from sklearn.decomposition import PCA
from pathlib import Path
from Training.utils import score_match
from Training.utils import score_match, visualize_match
import torch
import numpy as np
import uuid
......@@ -388,7 +388,7 @@ class ExperimentManager:
# if we already have pca calculated we can apply it now and save space
features = pca.transform(features.numpy())
# create data entry for the image
index_extraction_list.append((Path(path[0]).stem, label, features, rf_centers.numpy()))
index_extraction_list.append((path[0], label, features, rf_centers.numpy()))
if use_index_for_pca:
# if we still have to calculate pca: accumulate features
if accumulated_features is None:
......@@ -414,12 +414,25 @@ class ExperimentManager:
features = torch.nn.functional.normalize(features, p=2, dim=1)
# apply pca
features = pca.transform(features.numpy())
# TODO DELF wants to normalize again after pca, other impl says no, same for index features
rf_centers = rf_centers.numpy()
now= time.time()
now = time.time()
# gather results
matching_results = []
for index_representation in index_extraction_list:
inliers = score_match(index_representation[2], features, index_representation[3], rf_centers)
print(f"query image {filename}, index image{index_representation[0]} inliers {inliers}")
results = \
score_match(index_representation[2], features, index_representation[3], rf_centers)
print(f"query image {filename}, index image{Path(index_representation[0]).stem} inliers {sum(results[0])}")
# TODO atm adding the index path to the result for visualization
# sort results by number of inliers in descending order
matching_results.sort(key=lambda x: sum(x[0]), reverse=True)
# plots the top 10 matches
for i in range(10):
visualize_match(matching_results[i][3],path[0],matching_results[i][1], matching_results[i][2], matching_results[i][0])
def calculate_pca(data, save_path=None):
pca = PCA(n_components=PCA_COMPONENTS, whiten=True)
......@@ -4,9 +4,10 @@ import numpy as np
from skimage.measure import ransac
from scipy.spatial import cKDTree
from skimage.transform import AffineTransform
#from sklearn.decomposition import PCA
#from Training.delf import Delf
#import pickle
from skimage.feature import plot_matches
from PIL import Image
from pathlib import Path
import matplotlib.pyplot as plt
SCALES = [2, 1.4142, 1, 0.7071, 0.5, 0.3536, 0.25]
RF_VALUES = {"layer3": (267, 16, 133), "layer4": (427, 32, 213)}
......@@ -16,98 +17,6 @@ RANSAC_NUM_TRAILS = 1000
# TODO will be moved to experiment when model/ dataloaders are available
def extract_features(self, stage, pca=None):
model = Delf(10, "retrieval", "../Experiments/variable target layer/keypoints/5db43e8d_dbb65c50.pth").cuda()
all_features = None
all_boxes = None
feature_list = []
with torch.no_grad():
for image in range(5): # here is where i put the dataloader
# TODO think about limiting the image size ranges. This works if all imgs are 2000X2000 but might use up
# more mem if size is variable, also much larger images need more than 8gb vram
# Could also restrict larger scales for big images
data = torch.rand((1,3,1000,1000)).cuda()# c x h x w
label = 0
filename = "blub"
all_scale_features = None
all_scale_scores = None
all_scale_boxes = None
for scale in SCALES:
# get features and scores from model
features, attention_scores = model.single_scale_retrieval(scale, data)
# we will not be using the features in cuda anymore so detach to save memory
features = features.detach().cpu()
# calculate the receptive boxes in original scale
receptive_boxes = get_receptive_boxes(features.size(2), features.size(3), scale)
# flatten h,w dimensions so we can append values from different scales
# spatial information can be inferred from the receptive boxes
features = features.view(features.size(1), -1).t()
attention_scores = attention_scores.view(-1)
receptive_boxes = receptive_boxes.view(-1, 4)
# accumulate the values of all scales
if all_scale_features is None:
all_scale_features = features
all_scale_features = torch.cat((all_scale_features, features), dim=0)
if all_scale_scores is None:
all_scale_scores = attention_scores
all_scale_scores = torch.cat((all_scale_scores, attention_scores), dim=0)
if all_scale_boxes is None:
all_scale_boxes = receptive_boxes
all_scale_boxes = torch.cat((all_scale_boxes, receptive_boxes), dim=0)
# perform nms based on scores and receptive boxes then keep the top 1000 scoring boxes remaining
keep = get_top_k_index(all_scale_scores, all_scale_boxes.cuda(), iou_threshold=0.8, k=1000)
# index select the best boxes
# we do not need the values in cuda anymore
all_scale_boxes = all_scale_boxes[keep].detach().cpu()
# could probably just ignore scores at this point
# all_scale_scores = all_scale_scores[keep].detach().cpu()
all_scale_features = all_scale_features[keep]
all_scale_features, all_scale_boxes = model.extract_features(data)
# for pca we can just stitch all features together for retrieval we need some kind of reference
# to which image they belong
extracted_data = (all_scale_features, all_scale_boxes, label, filename)
if all_features is None:
all_features = all_scale_features
all_features = torch.cat((all_features, all_scale_features), dim=0)
if all_boxes is None:
all_boxes = all_scale_boxes
all_boxes = torch.cat((all_boxes, all_scale_boxes), dim=0)
# TODO tf repo says l2 normalize features before
# convert features to numpy for pca
all_features = all_features.numpy()
# fit pca
delf_pca = PCA(n_components=40, whiten=True)
pickle.dump(delf_pca, open("pca.pkl", "wb"))
print(f"pca componenets {delf_pca.components_} explained var ratio {delf_pca.explained_variance_ratio_}")
# test application of pca
test_sample = torch.rand(size=(1,1024)).numpy()
l_pca = pickle.load(open("pca.pkl", "rb"))
# TODO tf repo says l2 normalize features after
import time
def score_match(index_features, query_features, index_locations, query_locations):
index_tree = cKDTree(index_features)
distances, indices = index_tree.query(
......@@ -128,7 +37,7 @@ def score_match(index_features, query_features, index_locations, query_locations
if cleaned_query_locations.shape[0] <= RANSAC_MIN_SAMPLES:
return 0
return [False], None, None
# Perform geometric verification using RANSAC.
# Ransac currently takes ~35 times longer than kd tree
model_robust, inliers = ransac(
......@@ -138,10 +47,10 @@ def score_match(index_features, query_features, index_locations, query_locations
if model_robust is None:
return 0
return [False], None, None
return sum(inliers)
return inliers, cleaned_index_locations, cleaned_query_locations
def get_receptive_boxes(height, width, scale, target_layer="layer3"):
......@@ -192,3 +101,38 @@ def get_top_k_index(scores, boxes, iou_threshold, k):
keep = keep[:k]
return keep
def visualize_match(index_image_path, query_image_path, index_points, query_points, matches):
creates a visualization of a match and saves it to the vis folder
:param index_image_path:
:param query_image_path:
:param index_points:
:param query_points:
:param matches:
# create vis directory if not existing
visualization_directory = Path("../Vis")
if not visualization_directory.is_dir():
# filename is index_name_query_name_number_matches
filename = f"{Path(index_image_path).stem}_{Path(query_image_path).stem}_{sum(matches)}.png"
filepath = Path(visualization_directory.joinpath(Path(filename)))
# create match pairs (shape: no_matches, 2)
# pair entries refer to index in index_points and query_points for each match
match_pairs = np.nonzero(matches)[0]
match_pairs = np.column_stack((match_pairs, match_pairs))
# load images as np array
index_image = Image.open(index_image_path)
query_image = Image.open(query_image_path)
index_image = np.asarray(index_image)
query_image = np.asarray(query_image)
# plot_matches expects coords as y,x but our format is x,y so flip
index_points = np.flip(index_points, 1)
query_points = np.flip(query_points, 1)
# create plot and save
fig, ax = plt.subplots()
plot_matches(ax, index_image, query_image, index_points, query_points, match_pairs)
plt.savefig(filepath, dpi=300)
Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment