Pikaliov/World-UAV-ds
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial import: World-UAV prepro

Browse Source 
Add dataloaders (v1/v2), analysis scripts, and documentation for working with UAV-GeoLoc (World-UAV).

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Pikaliov
2026-05-09 12:44:49 +03:00
commit 4ff36ce188
72 changed files with 13594 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

0
GeoLoc-UAV-main/models/__init__.py Normal file
View File

40
GeoLoc-UAV-main/models/aggregators/LPN.py Normal file
View File

@@ -0,0 +1,40 @@
import torch
import torch.nn as nn
import math
from torch.nn import functional as F
def get_part_pool(x, block=4, no_overlap=True):
result = []
H, W = x.size(2), x.size(3)
c_h, c_w = int(H/2), int(W/2)
per_h, per_w = H/(2*block),W/(2*block)
if per_h < 1 and per_w < 1:
new_H, new_W = H+(block-c_h)*2, W+(block-c_w)*2
x = nn.functional.interpolate(x, size=[new_H,new_W], mode='bilinear', align_corners=True)
H, W = x.size(2), x.size(3)
c_h, c_w = int(H/2), int(W/2)
per_h, per_w = H/(2*block),W/(2*block)
per_h, per_w = math.floor(per_h), math.floor(per_w)
for i in range(block):
i = i + 1
if i < block:
x_curr = x[:,:,(c_h-i*per_h):(c_h+i*per_h),(c_w-i*per_w):(c_w+i*per_w)]
if no_overlap and i > 1:
x_pre = x[:,:,(c_h-(i-1)*per_h):(c_h+(i-1)*per_h),(c_w-(i-1)*per_w):(c_w+(i-1)*per_w)]
x_pad = F.pad(x_pre,(per_h,per_h,per_w,per_w),"constant",0)
x_curr = x_curr - x_pad
result.append(x_curr)
else:
if no_overlap and i > 1:
x_pre = x[:,:,(c_h-(i-1)*per_h):(c_h+(i-1)*per_h),(c_w-(i-1)*per_w):(c_w+(i-1)*per_w)]
pad_h = c_h-(i-1)*per_h
pad_w = c_w-(i-1)*per_w
# x_pad = F.pad(x_pre,(pad_h,pad_h,pad_w,pad_w),"constant",0)
if x_pre.size(2)+2*pad_h == H:
x_pad = F.pad(x_pre,(pad_h,pad_h,pad_w,pad_w),"constant",0)
else:
ep = H - (x_pre.size(2)+2*pad_h)
x_pad = F.pad(x_pre,(pad_h+ep,pad_h,pad_w+ep,pad_w),"constant",0)
x = x - x_pad
result.append(x_curr)
return result

3
GeoLoc-UAV-main/models/aggregators/__init__.py Normal file
View File

@@ -0,0 +1,3 @@
from .gem import GeMPool
from .convap import ConvAP
from .multiconvap import MulConvAP

33
GeoLoc-UAV-main/models/aggregators/convap.py Normal file
View File

@@ -0,0 +1,33 @@
import torch
import torch.nn.functional as F
import torch.nn as nn
class ConvAP(nn.Module):
"""Implementation of ConvAP as of https://arxiv.org/pdf/2210.10239.pdf
Args:
in_channels (int): number of channels in the input of ConvAP
out_channels (int, optional): number of channels that ConvAP outputs. Defaults to 512.
s1 (int, optional): spatial height of the adaptive average pooling. Defaults to 2.
s2 (int, optional): spatial width of the adaptive average pooling. Defaults to 2.
"""
def __init__(self, in_channels, out_channels=512, s1=2, s2=2):
super(ConvAP, self).__init__()
self.channel_pool = nn.Conv2d(in_channels=in_channels, out_channels=out_channels, kernel_size=1, bias=True)
self.AAP = nn.AdaptiveAvgPool2d((s1, s2))
def forward(self, x):
#
x, t = x #dinov2专属
# x = self.channel_pool(x)
x = self.AAP(x)
x = F.normalize(x.flatten(1), p=2, dim=1)
return x
if __name__ == '__main__':
x = torch.randn(4, 2048, 10, 10)
m = ConvAP(2048, 512)
r = m(x)
print(r.shape)

17
GeoLoc-UAV-main/models/aggregators/gem.py Normal file
View File

@@ -0,0 +1,17 @@
import torch
import torch.nn.functional as F
import torch.nn as nn
class GeMPool(nn.Module):
"""Implementation of GeM as in https://github.com/filipradenovic/cnnimageretrieval-pytorch
we add flatten and norm so that we can use it as one aggregation layer.
"""
def __init__(self, p=3, eps=1e-6):
super().__init__()
self.p = nn.Parameter(torch.ones(1)*p)
self.eps = eps
def forward(self, x):
x = F.avg_pool2d(x.clamp(min=self.eps).pow(self.p), (x.size(-2), x.size(-1))).pow(1./self.p)
x = x.flatten(1)
return F.normalize(x, p=2, dim=1)

96
GeoLoc-UAV-main/models/aggregators/multiconvap.py Normal file
View File

@@ -0,0 +1,96 @@
import torch
import torch.nn.functional as F
import torch.nn as nn
from models.aggregators.LPN import get_part_pool
class L2Norm(nn.Module):
def __init__(self, dim=1):
super().__init__()
self.dim = dim
def forward(self, x):
return F.normalize(x, p=2, dim=self.dim)
class GeMPool(nn.Module):
"""Implementation of GeM as in https://github.com/filipradenovic/cnnimageretrieval-pytorch
we add flatten and norm so that we can use it as one aggregation layer.
"""
def __init__(self, p=3, eps=1e-6):
super().__init__()
self.p = nn.Parameter(torch.ones(1)*p)
self.eps = eps
def forward(self, x):
x = F.avg_pool2d(x.clamp(min=self.eps).pow(self.p), (x.size(-2), x.size(-1))).pow(1./self.p)
x = x.flatten(1)
return F.normalize(x, p=2, dim=1)
class MulConvAP(nn.Module):
"""Implementation of ConvAP as of https://arxiv.org/pdf/2210.10239.pdf
Args:
in_channels (int): number of channels in the input of ConvAP
out_channels (int, optional): number of channels that ConvAP outputs. Defaults to 512.
s1 (int, optional): spatial height of the adaptive average pooling. Defaults to 2.
s2 (int, optional): spatial width of the adaptive average pooling. Defaults to 2.
"""
def __init__(self, in_channels, out_channels=512, s1=2, s2=2, LPN=False):
super(MulConvAP, self).__init__()
self.out_channels = out_channels
self.channel_pool_1 = nn.Conv2d(in_channels=in_channels, out_channels=in_channels, kernel_size=1, bias=True)
self.channel_pool_3 = nn.Conv2d(in_channels=in_channels, out_channels=in_channels, kernel_size=3, padding=1,bias=True)
self.channel_pool_5 = nn.Conv2d(in_channels=in_channels, out_channels=in_channels, kernel_size=5, padding=2,bias=True)
# self.AAP = nn.AdaptiveAvgPool2d((s1, s2))
self.AAP = nn.Sequential(L2Norm(), GeMPool())
# using LPN
if LPN == True:
self.LPN = True
else:
self.LPN = False
def forward(self, x):
if self.LPN == False:
# x, t = x #dinov2专属
x1 = self.channel_pool_1(x)
x3 = self.channel_pool_3(x)
x5 = self.channel_pool_5(x)
x1 = self.AAP(x1)
x3 = self.AAP(x3)
x5 = self.AAP(x5)
x = [i for i in [x1, x3, x5]]
x = torch.cat(x,dim=1)
# x = self.AAP(x)
x = F.normalize(x.flatten(1), p=2, dim=1)
return x
else:
partition_feature = get_part_pool(x)
partition_feature_list = []
for one_feature in partition_feature:
x1 = self.channel_pool_1(one_feature)
x3 = self.channel_pool_3(one_feature)
x5 = self.channel_pool_5(one_feature)
x1 = self.AAP(x1)
x3 = self.AAP(x3)
x5 = self.AAP(x5)
x = [i for i in [x1, x3, x5]]
x = torch.cat(x,dim=1)
x = F.normalize(x.flatten(1), p=2, dim=1)
partition_feature_list.append(x)
# partition_feature_tensor = torch.stack(partition_feature_list, dim=2).reshape(x.shape[0], -1)
return partition_feature_list
if __name__ == '__main__':
x = torch.randn(4, 2048, 10, 10)
# m = ConvAP(2048, 512)
# r = m(x)
# print(r.shape)

46
GeoLoc-UAV-main/models/anyloc.py Normal file
View File

@@ -0,0 +1,46 @@
import torch
import torch.nn as nn
from models import aggregators
DINOV2_ARCHS = {
'dinov2_vits14': 384,
'dinov2_vitb14': 768,
'dinov2_vitl14': 1024,
'dinov2_vitg14': 1536,
}
class AnyModel(nn.Module):
def __init__(self,
model_name='dinov2_vitb14',
pretrained=True,
):
super(AnyModel, self).__init__()
assert model_name in DINOV2_ARCHS.keys(), f'Unknown model name {model_name}'
self.model = torch.hub.load('facebookresearch/dinov2', model_name)
self.num_channels = DINOV2_ARCHS[model_name]
self.gem = aggregators.GeMPool()
def forward(self, x):
B, C, H, W = x.shape
x = self.model.prepare_tokens_with_masks(x)
# First blocks are frozen
with torch.no_grad():
for blk in self.model.blocks:
x = blk(x)
x = x.detach()
t = x[:, 0]
f = x[:, 1:]
# Reshape to (B, C, H, W)
f = f.reshape((B, H // 14, W // 14, self.num_channels)).permute(0, 3, 1, 2)
g = self.gem(f)
return g

2
GeoLoc-UAV-main/models/backbone/__init__.py Normal file
View File

@@ -0,0 +1,2 @@
from .resnet import ResNet
from .dinov2 import DINOv2

94
GeoLoc-UAV-main/models/backbone/dinov2.py Normal file
View File

@@ -0,0 +1,94 @@
import torch
import torch.nn as nn
DINOV2_ARCHS = {
'dinov2_vits14': 384,
'dinov2_vitb14': 768,
'dinov2_vitl14': 1024,
'dinov2_vitg14': 1536,
}
class DINOv2(nn.Module):
"""
DINOv2 model
Args:
model_name (str): The name of the model architecture
should be one of ('dinov2_vits14', 'dinov2_vitb14', 'dinov2_vitl14', 'dinov2_vitg14')
num_trainable_blocks (int): The number of last blocks in the model that are trainable.
norm_layer (bool): If True, a normalization layer is applied in the forward pass.
return_token (bool): If True, the forward pass returns both the feature map and the token.
"""
def __init__(
self,
model_name='dinov2_vitb14',
num_trainable_blocks=2,
norm_layer=False,
return_token=False,
pretrain_flag=False
):
super().__init__()
assert model_name in DINOV2_ARCHS.keys(), f'Unknown model name {model_name}'
self.model = torch.hub.load('facebookresearch/dinov2', "dinov2_vits14")
# torch.hub.load('/home/Shen/.cache/torch/hub/facebookresearch_dinov2_main/',
# model_name,
# source='local')
# self.model = torch.hub.load('facebookresearch/dinov2', model_name)
self.num_channels = DINOV2_ARCHS[model_name]
self.num_trainable_blocks = num_trainable_blocks
self.norm_layer = norm_layer
self.return_token = return_token
self.flag = pretrain_flag
def forward(self, x):
"""
The forward method for the DINOv2 class
Parameters:
x (torch.Tensor): The input tensor [B, 3, H, W]. H and W should be divisible by 14.
Returns:
f (torch.Tensor): The feature map [B, C, H // 14, W // 14].
t (torch.Tensor): The token [B, C]. This is only returned if return_token is True.
"""
B, C, H, W = x.shape
x = self.model.prepare_tokens_with_masks(x)
if self.flag:
# When flag is True, freeze all parameters
for param in self.model.parameters():
param.requires_grad = False
with torch.no_grad():
for blk in self.model.blocks:
x = blk(x)
else:
# When flag is False, freeze part of the parameters (e.g., first blocks)
for param in self.model.parameters():
param.requires_grad = False # Freeze all layers initially
# Unfreeze the last few blocks (trainable)
for param in self.model.blocks[-self.num_trainable_blocks:].parameters():
param.requires_grad = True
with torch.no_grad():
for blk in self.model.blocks[:-self.num_trainable_blocks]: # Freeze these blocks
x = blk(x)
# Last blocks are trained
for blk in self.model.blocks[-self.num_trainable_blocks:]: # Train these blocks
x = blk(x)
if self.norm_layer:
x = self.model.norm(x)
t = x[:, 0]
f = x[:, 1:]
# Reshape to (B, C, H, W)
f = f.reshape((B, H // 14, W // 14, self.num_channels)).permute(0, 3, 1, 2)
if self.return_token:
return f, t
return f

107
GeoLoc-UAV-main/models/backbone/resnet.py Normal file
View File

@@ -0,0 +1,107 @@
import torch
import torch.nn as nn
import torchvision
import numpy as np
class ResNet(nn.Module):
def __init__(self,
model_name='resnet50',
pretrained=True,
layers_to_freeze=2,
layers_to_crop=[],
pretrain_flag = False
):
"""Class representing the resnet backbone used in the pipeline
we consider resnet network as a list of 5 blocks (from 0 to 4),
layer 0 is the first conv+bn and the other layers (1 to 4) are the rest of the residual blocks
we don't take into account the global pooling and the last fc
Args:
model_name (str, optional): The architecture of the resnet backbone to instanciate. Defaults to 'resnet50'.
pretrained (bool, optional): Whether pretrained or not. Defaults to True.
layers_to_freeze (int, optional): The number of residual blocks to freeze (starting from 0) . Defaults to 2.
layers_to_crop (list, optional): Which residual layers to crop, for example [3,4] will crop the third and fourth res blocks. Defaults to [].
Raises:
NotImplementedError: if the model_name corresponds to an unknown architecture.
"""
super().__init__()
self.model_name = model_name.lower()
self.layers_to_freeze = layers_to_freeze
self.flag = pretrain_flag
if pretrained:
# the new naming of pretrained weights, you can change to V2 if desired.
weights = 'IMAGENET1K_V1'
else:
weights = None
if 'swsl' in model_name or 'ssl' in model_name:
# These are the semi supervised and weakly semi supervised weights from Facebook
self.model = torch.hub.load(
'facebookresearch/semi-supervised-ImageNet1K-models', model_name)
else:
if 'resnext50' in model_name:
self.model = torchvision.models.resnext50_32x4d(
weights=weights)
elif 'resnet50' in model_name:
self.model = torchvision.models.resnet50(weights=weights)
elif '101' in model_name:
self.model = torchvision.models.resnet101(weights=weights)
elif '152' in model_name:
self.model = torchvision.models.resnet152(weights=weights)
elif '34' in model_name:
self.model = torchvision.models.resnet34(weights=weights)
elif '18' in model_name:
# self.model = torchvision.models.resnet18(pretrained=False)
self.model = torchvision.models.resnet18(weights=weights)
elif 'wide_resnet50_2' in model_name:
self.model = torchvision.models.wide_resnet50_2(
weights=weights)
else:
raise NotImplementedError(
'Backbone architecture not recognized!')
# freeze only if the model is pretrained
if pretrained and self.flag:
if layers_to_freeze >= 0:
self.model.conv1.requires_grad_(False)
self.model.bn1.requires_grad_(False)
if layers_to_freeze >= 1:
self.model.layer1.requires_grad_(False)
if layers_to_freeze >= 2:
self.model.layer2.requires_grad_(False)
if layers_to_freeze >= 3:
self.model.layer3.requires_grad_(False)
# remove the avgpool and most importantly the fc layer
self.model.avgpool = None
self.model.fc = None
if 4 in layers_to_crop:
self.model.layer4 = None
if 3 in layers_to_crop:
self.model.layer3 = None
out_channels = 2048
if '34' in model_name or '18' in model_name:
out_channels = 256
self.out_channels = out_channels // 2 if self.model.layer4 is None else out_channels
self.out_channels = self.out_channels // 2 if self.model.layer3 is None else self.out_channels
def forward(self, x):
x = self.model.conv1(x)
x = self.model.bn1(x)
x = self.model.relu(x)
x = self.model.maxpool(x)
x = self.model.layer1(x)
x = self.model.layer2(x)
if self.model.layer3 is not None:
x = self.model.layer3(x)
if self.model.layer4 is not None:
x = self.model.layer4(x)
return x

167
GeoLoc-UAV-main/models/game4loc.py Normal file
View File

@@ -0,0 +1,167 @@
import torch
import timm
import numpy as np
import torch.nn as nn
from PIL import Image
from urllib.request import urlopen
from thop import profile
class MLP(nn.Module):
def __init__(self, input_size=2048, hidden_size=512, output_size=2):
super(MLP, self).__init__()
self.fc1 = nn.Linear(input_size, hidden_size)
self.relu = nn.ReLU()
self.fc2 = nn.Linear(hidden_size, hidden_size // 2)
self.fc3 = nn.Linear(hidden_size // 2, output_size)
def forward(self, x):
x = self.fc1(x)
x = self.relu(x)
x = self.fc2(x)
x = self.relu(x)
x = self.fc3(x)
return x
class DesModel(nn.Module):
def __init__(self,
model_name='vit',
pretrained=True,
img_size=384,
share_weights=True,
train_with_recon=False,
train_with_offset=False,
model_hub='timm'):
super(DesModel, self).__init__()
self.share_weights = share_weights
self.model_name = model_name
self.img_size = img_size
if share_weights:
if "vit" in model_name or "swin" in model_name:
# automatically change interpolate pos-encoding to img_size
self.model = timm.create_model(model_name, pretrained=pretrained, num_classes=0, img_size=img_size)
else:
self.model = timm.create_model(model_name, pretrained=pretrained, num_classes=0)
else:
if "vit" in model_name or "swin" in model_name:
self.model1 = timm.create_model(model_name, pretrained=pretrained, num_classes=0, img_size=img_size)
self.model2 = timm.create_model(model_name, pretrained=pretrained, num_classes=0, img_size=img_size)
else:
self.model1 = timm.create_model(model_name, pretrained=pretrained, num_classes=0)
self.model2 = timm.create_model(model_name, pretrained=pretrained, num_classes=0)
if train_with_offset:
self.MLP = MLP()
self.logit_scale = torch.nn.Parameter(torch.ones([]) * np.log(1 / 0.07))
def get_config(self,):
if self.share_weights:
data_config = timm.data.resolve_model_data_config(self.model)
else:
data_config = timm.data.resolve_model_data_config(self.model1)
return data_config
def set_grad_checkpointing(self, enable=True):
if self.share_weights:
self.model.set_grad_checkpointing(enable)
else:
self.model1.set_grad_checkpointing(enable)
self.model2.set_grad_checkpointing(enable)
def freeze_layers(self, frozen_blocks=10, frozen_stages=[0,0,0,0]):
pass
def forward(self, img1=None, img2=None):
if self.share_weights:
if img1 is not None and img2 is not None:
image_features1 = self.model(img1)
image_features2 = self.model(img2)
return image_features1, image_features2
elif img1 is not None:
image_features = self.model(img1)
return image_features
else:
image_features = self.model(img2)
return image_features
else:
if img1 is not None and img2 is not None:
image_features1 = self.model1(img1)
image_features2 = self.model2(img2)
return image_features1, image_features2
elif img1 is not None:
image_features = self.model1(img1)
return image_features
else:
image_features = self.model2(img2)
return image_features
def offset_pred(self, img_feature1, img_feature2):
offset = self.MLP(torch.cat((img_feature1, img_feature2), dim=1))
return offset
if __name__ == '__main__':
# model = TimmModel(model_name='timm/vit_large_patch16_384.augreg_in21k_ft_in1k')
# # model = TimmModel(model_name='timm/vit_base_patch16_224.augreg_in1k')
# # from timm.models.vision_transformer import vit_base_patch16_224
# # model = vit_base_patch16_224(img_size=384, patch_size=16, embed_dim=768, depth=12, num_heads=12, mlp_ratio=4, num_classes=0)
# model = DesModel(model_name='timm/resnet101.tv_in1k', img_size=384)
# model = DesModel(model_name='convnext_base.fb_in22k_ft_in1k_384', img_size=384)
model = DesModel(model_name='timm/swin_base_patch4_window7_224.ms_in22k_ft_in1k', img_size=384)
# # model = TimmModel(model_name='vit_base_patch16_rope_reg1_gap_256.sbb_in1k')
# # model = TimmModel(model_name='timm/vit_medium_patch16_rope_reg1_gap_256.sbb_in1k')
# # model = TimmModel(model_name='timm/vit_medium_patch16_gap_256.sw_in12k_ft_in1k')
# # model = TimmModel(model_name='timm/resnet101.tv_in1k')
# # img = Image.open(urlopen(
# # 'https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/beignets-task-guide.png'
# # ))
x = torch.rand((1, 3, 384, 384))
x = x.cuda()
model.cuda()
x = model(x)
print(x.shape)
# flops, params = profile(model, inputs=(x,))
# # print(img.size)
# # img = transform(img)
# # print(img.size)
# # print(model1)
# print('flops(G)', flops/1e9, 'params(M)', params/1e6)
# from transformers import CLIPProcessor, CLIPModel
# model = CLIPModel.from_pretrained("/home/xmuairmud/jyx/clip-vit-base-patch16")
# vision_model = model.vision_model
# print(vision_model)
# dinov2_vitb14_reg = torch.hub.load('facebookresearch/dinov2', 'dinov2_vitb14_reg')
# print(dinov2_vitb14_reg.set_grad_checkpointing(True))
# from transformers import ViTModel, ViTImageProcessor, AutoModelForImageClassification, AutoConfig
# config = AutoConfig.from_pretrained('facebook/dino-vitb16')
# config.image_size = 384
# model = ViTModel.from_pretrained('facebook/dino-vitb16', config=config, ignore_mismatched_sizes=True)
# model = timm.create_model('vit_base_patch14_reg4_dinov2.lvd142m', pretrained=True, img_size=(384, 384))
# data_config = timm.data.resolve_model_data_config(model)
# print(data_config)
# processor = ViTImageProcessor.from_pretrained('facebook/dino-vitb16')
# x = torch.rand((1, 3, 384, 384))
# inputs = processor(images=x, return_tensors="pt")
# print(inputs['pixel_values'].shape)
# outputs = model(**inputs)
# print(outputs.pooler_output.shape)
# print(model(x).shape)
# flops, params = profile(dinov2_vitb14_reg, inputs=(x,))
# print('flops(G)', flops/1e9, 'params(M)', params/1e6)

2
GeoLoc-UAV-main/models/group/__init__.py Normal file
View File

@@ -0,0 +1,2 @@
from .groupnet import GroupNet
from .groupnet_dino import GroupDinoNet

182
GeoLoc-UAV-main/models/group/groupnet.py Normal file
View File

@@ -0,0 +1,182 @@
import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np
from utils.utils import dim_extend,interpolate_feats,l2_normalize
class GroupNetConfig:
def __init__(self):
self.sample_scale_begin = 0
self.sample_scale_inter = 0.5
self.sample_scale_num = 1
self.sample_rotate_begin = 0
self.sample_rotate_inter = 45
self.sample_rotate_num = 8
group_config = GroupNetConfig()
class VanillaLightCNN(nn.Module):
def __init__(self):
super(VanillaLightCNN, self).__init__()
self.conv0=nn.Sequential(
nn.Conv2d(3,16,5,1,2,bias=False),
nn.InstanceNorm2d(16),
nn.ReLU(inplace=True),
nn.Conv2d(16,32,5,1,2,bias=False),
nn.InstanceNorm2d(32),
nn.ReLU(inplace=True),
nn.AvgPool2d(2, 2),
# 修改
nn.Conv2d(32,64,5,1,2,bias=False),
nn.InstanceNorm2d(64),
nn.ReLU(inplace=True),
nn.AvgPool2d(2, 2),
)
# 原来 32
self.conv1=nn.Sequential(
nn.Conv2d(64,64,5,1,2,bias=False),
nn.InstanceNorm2d(64),
nn.ReLU(inplace=True),
nn.Conv2d(64,64,5,1,2,bias=False),
nn.InstanceNorm2d(64),
)
def forward(self, x):
x=self.conv1(self.conv0(x))
x=l2_normalize(x,axis=1) # [1,c,w//2, h//2]
return x
class ExtractorWrapper(nn.Module):
def __init__(self,scale_num, rotation_num):
super(ExtractorWrapper, self).__init__()
self.extractor=VanillaLightCNN()
self.sn, self.rn = scale_num, rotation_num
def forward(self,img_list,pts_list):
'''
:param img_list: list of [b,3,h,w]
:param pts_list: list of [b,n,2]
:return:gefeats [b,n,f,sn,rn]
'''
assert(len(img_list)==self.rn*self.sn)
gfeats_list,neg_gfeats_list=[],[]
# feature extraction
for img_index,img in enumerate(img_list):
# extract feature
feats=self.extractor(img)
gfeats_list.append(interpolate_feats(img,pts_list[img_index],feats)[:,:,:,None])
gfeats_list=torch.cat(gfeats_list,3) # b,n,f,sn*rn
b,n,f,_=gfeats_list.shape
gfeats_list=gfeats_list.reshape(b,n,f,self.sn,self.rn)
return gfeats_list
class BilinearGCNN(nn.Module):
def __init__(self, scale_num, rotation_num):
super(BilinearGCNN, self).__init__()
self.r, self.s = rotation_num, scale_num
self.network1_embed1 = nn.Sequential(
nn.Conv2d(64, 64, 3, 1, 1),
nn.ReLU(True),
nn.Conv2d(64, 64, 3, 1, 1),
)
self.network1_embed1_short = nn.Conv2d(64, 64, 1, 1)
self.network1_embed1_relu = nn.ReLU(True)
self.network1_embed2 = nn.Sequential(
nn.Conv2d(64, 64, 3, 1, 1),
nn.ReLU(True),
nn.Conv2d(64, 64, 3, 1, 1),
)
self.network1_embed2_short = nn.Conv2d(64, 64, 1, 1)
self.network1_embed2_relu = nn.ReLU(True)
self.network1_embed3 = nn.Sequential(
nn.Conv2d(64, 64, 3, 1, 1),
nn.ReLU(True),
nn.Conv2d(64, 16, 3, 1, 1),
)
###########################
self.network2_embed1 = nn.Sequential(
nn.Conv2d(64, 64, 3, 1, 1),
nn.ReLU(True),
nn.Conv2d(64, 64, 3, 1, 1),
)
self.network2_embed1_short = nn.Conv2d(64, 64, 1, 1)
self.network2_embed1_relu = nn.ReLU(True)
self.network2_embed2 = nn.Sequential(
nn.Conv2d(64, 64, 3, 1, 1),
nn.ReLU(True),
nn.Conv2d(64, 64, 3, 1, 1),
)
self.network2_embed2_short = nn.Conv2d(64, 64, 1, 1)
self.network2_embed2_relu = nn.ReLU(True)
self.network2_embed3 = nn.Sequential(
nn.Conv2d(64, 64, 3, 1, 1),
nn.ReLU(True),
nn.Conv2d(64, 16, 3, 1, 1),
)
def forward(self, x):
'''
:param x: b,n,f,ssn,srn
:return:
'''
b, n, f, ssn, srn = x.shape
assert (ssn == self.s and srn == self.r)
x = x.reshape(b * n, f, ssn, srn)
x1 = self.network1_embed1_relu(self.network1_embed1(x) + self.network1_embed1_short(x))
x1 = self.network1_embed2_relu(self.network1_embed2(x1) + self.network1_embed2_short(x1))
x1 = self.network1_embed3(x1)
x2 = self.network2_embed1_relu(self.network2_embed1(x) + self.network2_embed1_short(x))
x2 = self.network2_embed2_relu(self.network2_embed2(x2) + self.network2_embed2_short(x2))
x2 = self.network2_embed3(x2)
x1 = x1.reshape(b * n, 16, self.s * self.r)
x2 = x2.reshape(b * n, 16, self.s * self.r).permute(0, 2, 1) # b*n,25,16
x = torch.bmm(x1, x2).reshape(b * n, 256) # b*n,8,25
assert (x.shape[1] == 256)
x=x.reshape(b,n,256)
x=l2_normalize(x,axis=2)
return x
class EmbedderWrapper(nn.Module):
def __init__(self, scale_num, rotation_num):
super(EmbedderWrapper, self).__init__()
self.embedder=BilinearGCNN(scale_num, rotation_num)
def forward(self, gfeats):
# group cnns
gefeats=self.embedder(gfeats) # b,n,f
return gefeats
class GroupNet(nn.Module):
def __init__(self, config=group_config):
super(GroupNet, self).__init__()
self.scale_num = config.sample_scale_num
self.rotation_num = config.sample_rotate_num
self.extractor=ExtractorWrapper(self.scale_num, self.rotation_num).cuda()
self.embedder=EmbedderWrapper(self.scale_num, self.rotation_num).cuda()
def forward(self, img_list, pts_list):
gfeats=self.extractor(dim_extend(img_list),dim_extend(pts_list))
efeats=self.embedder(gfeats)
return efeats, gfeats

222
GeoLoc-UAV-main/models/group/groupnet_dino.py Normal file
View File

@@ -0,0 +1,222 @@
import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np
from utils.utils import dim_extend,interpolate_feats,l2_normalize
import json
json_path = "/media/guan/新加卷/Code/Code/configs/transform_config.json"
with open(json_path, 'r', encoding='utf-8') as file:
data = json.load(file)
group_config = data["transform_config"]
# class GroupNetConfig:
# def __init__(self):
# self.sample_scale_begin = 0
# self.sample_scale_inter = 0.5
# self.sample_scale_num = 3
# self.sample_rotate_begin = -45
# self.sample_rotate_inter = 45
# self.sample_rotate_num = 8
# class GroupNetConfig:
# def __init__(self):
# self.sample_scale_begin = 0
# self.sample_scale_inter = 1
# self.sample_scale_num = 1
# self.sample_rotate_begin = 0
# self.sample_rotate_inter = 0
# self.sample_rotate_num = 1
# group_config = GroupNetConfig()
class VanillaLightCNN(nn.Module):
def __init__(self):
super(VanillaLightCNN, self).__init__()
self.conv0 = nn.Sequential(
nn.Conv2d(384,384//2,1,1,bias=False),
nn.InstanceNorm2d(384//2),
nn.ReLU(inplace=True),
nn.Conv2d(384//2,384//4,1,1,bias=False),
nn.InstanceNorm2d(384//4),
nn.ReLU(inplace=True),
nn.Conv2d(384//4,64,1,1,bias=False),
nn.InstanceNorm2d(64),
)
self.conv1 = nn.Sequential(
nn.Conv2d(3,16,5,1,2,bias=False),
nn.InstanceNorm2d(16),
nn.ReLU(inplace=True),
nn.Conv2d(16,32,5,1,2,bias=False),
nn.InstanceNorm2d(32),
nn.ReLU(inplace=True),
nn.AvgPool2d(2, 2))
self.proj = nn.Conv2d(96, 64, 1, 1, bias=False)
def forward(self, x, img):
x_dino=self.conv0(x)
x_resized = F.interpolate(img, size=(32, 32), mode='bilinear', align_corners=False)
x_cnn = self.conv1(x_resized)
x_cat = torch.concat((x_dino, x_cnn), dim=1)
x_proj = self.proj(x_cat)
x=l2_normalize(x_proj,axis=1) # [1,c,w//2, h//2]
return x
class ExtractorWrapper(nn.Module):
def __init__(self,scale_num, rotation_num):
super(ExtractorWrapper, self).__init__()
self.extractor=VanillaLightCNN()
self.sn, self.rn = scale_num, rotation_num
dinov2_weights = torch.hub.load('facebookresearch/dinov2', "dinov2_vits14")
# torch.load("/media/Shen/Data/RingoData/WorldLoc/Code/dinov2_vits14_pretrain.pth")
from models.transformer import vit_small
vit_kwargs = dict(
patch_size= 14,
img_size=518,
init_values = 1.0,
ffn_layer = "mlp",
block_chunks = 0,
)
self.dinov2_vits14 = vit_small(**vit_kwargs).eval()
# self.dinov2_vits14.load_state_dict(dinov2_weights)
def forward(self,img_list,pts_list):
'''
:param img_list: list of [b,3,h,w]
:param pts_list: list of [b,n,2]
:return:gefeats [b,n,f,sn,rn]
'''
assert(len(img_list)==self.rn*self.sn)
gfeats_list = []
# feature extraction
for img_index,img in enumerate(img_list):
# extract feature
with torch.no_grad():
dinov2_features_16 = self.dinov2_vits14.forward_features(img)
B, _, H, W = img.shape
features_16 = dinov2_features_16['x_norm_patchtokens'].permute(0,2,1).reshape(B,-1,H//14, W//14)
feats=self.extractor(features_16, img)
gfeats_list.append(interpolate_feats(img, pts_list[img_index], feats)[:,:,:,None])
gfeats_list=torch.cat(gfeats_list,3) # b,n,f,sn*rn
b,n,f,_=gfeats_list.shape
gfeats_list=gfeats_list.reshape(b,n,f,self.sn,self.rn)
return gfeats_list
class BilinearGCNN(nn.Module):
def __init__(self, scale_num, rotation_num):
super(BilinearGCNN, self).__init__()
self.r, self.s = rotation_num, scale_num
self.network1_embed1 = nn.Sequential(
nn.Conv2d(64, 64, 3, 1, 1),
nn.ReLU(True),
nn.Conv2d(64, 64, 3, 1, 1),
)
self.network1_embed1_short = nn.Conv2d(64, 64, 1, 1)
self.network1_embed1_relu = nn.ReLU(True)
self.network1_embed2 = nn.Sequential(
nn.Conv2d(64, 64, 3, 1, 1),
nn.ReLU(True),
nn.Conv2d(64, 64, 3, 1, 1),
)
self.network1_embed2_short = nn.Conv2d(64, 64, 1, 1)
self.network1_embed2_relu = nn.ReLU(True)
self.network1_embed3 = nn.Sequential(
nn.Conv2d(64, 64, 3, 1, 1),
nn.ReLU(True),
nn.Conv2d(64, 16, 3, 1, 1),
)
###########################
self.network2_embed1 = nn.Sequential(
nn.Conv2d(64, 64, 3, 1, 1),
nn.ReLU(True),
nn.Conv2d(64, 64, 3, 1, 1),
)
self.network2_embed1_short = nn.Conv2d(64, 64, 1, 1)
self.network2_embed1_relu = nn.ReLU(True)
self.network2_embed2 = nn.Sequential(
nn.Conv2d(64, 64, 3, 1, 1),
nn.ReLU(True),
nn.Conv2d(64, 64, 3, 1, 1),
)
self.network2_embed2_short = nn.Conv2d(64, 64, 1, 1)
self.network2_embed2_relu = nn.ReLU(True)
self.network2_embed3 = nn.Sequential(
nn.Conv2d(64, 64, 3, 1, 1),
nn.ReLU(True),
nn.Conv2d(64, 16, 3, 1, 1),
)
def forward(self, x):
'''
:param x: b,n,f,ssn,srn
:return:
'''
b, n, f, ssn, srn = x.shape
# equal = x.reshape(b, n, f, ssn*srn)
# equ_features=torch.max(equal,dim=-1,keepdim=False)[0]
# x = l2_normalize(equ_features, axis=1)
assert (ssn == self.s and srn == self.r)
x = x.reshape(b * n, f, ssn, srn)
x1 = self.network1_embed1_relu(self.network1_embed1(x) + self.network1_embed1_short(x))
x1 = self.network1_embed2_relu(self.network1_embed2(x1) + self.network1_embed2_short(x1))
x1 = self.network1_embed3(x1)
x2 = self.network2_embed1_relu(self.network2_embed1(x) + self.network2_embed1_short(x))
x2 = self.network2_embed2_relu(self.network2_embed2(x2) + self.network2_embed2_short(x2))
x2 = self.network2_embed3(x2)
x1 = x1.reshape(b * n, 16, self.s * self.r)
x2 = x2.reshape(b * n, 16, self.s * self.r).permute(0, 2, 1) # b*n,25,16
x = torch.bmm(x1, x2).reshape(b * n, 256) # b*n,8,25
assert (x.shape[1] == 256)
x=x.reshape(b,n,256)
x=l2_normalize(x,axis=2)
return x
class EmbedderWrapper(nn.Module):
def __init__(self, scale_num, rotation_num):
super(EmbedderWrapper, self).__init__()
self.embedder=BilinearGCNN(scale_num, rotation_num)
def forward(self, gfeats):
# group cnns
gefeats=self.embedder(gfeats) # b,n,f
return gefeats
class GroupDinoNet(nn.Module):
def __init__(self, config=group_config):
super(GroupDinoNet, self).__init__()
self.scale_num = config["sample_scale_num"]
self.rotation_num = config["sample_rotate_num"]
self.extractor=ExtractorWrapper(self.scale_num, self.rotation_num).cuda()
self.embedder=EmbedderWrapper(self.scale_num, self.rotation_num).cuda()
def forward(self, img_list, pts_list):
gfeats=self.extractor(dim_extend(img_list),dim_extend(pts_list))
efeats=self.embedder(gfeats)
return efeats, gfeats

90
GeoLoc-UAV-main/models/helper.py Normal file
View File

@@ -0,0 +1,90 @@
from models import group
from models import aggregators
from models import backbone
def get_groupnet(groupnet_arch='groupnet', group_config={}):
if "groupnet" in groupnet_arch.lower():
return group.GroupNet(**group_config)
def get_groupdinonet(groupnet_arch='groupdinonet', group_config={}):
if "groupdinonet" in groupnet_arch.lower():
return group.GroupDinoNet(**group_config)
def get_aggregator(agg_arch='ConvAP', agg_config={}):
"""Helper function that returns the aggregation layer given its name.
If you happen to make your own aggregator, you might need to add a call
to this helper function.
Args:
agg_arch (str, optional): the name of the aggregator. Defaults to 'ConvAP'.
agg_config (dict, optional): this must contain all the arguments needed to instantiate the aggregator class. Defaults to {}.
Returns:
nn.Module: the aggregation layer
"""
if 'cosplace' in agg_arch.lower():
assert 'in_dim' in agg_config
assert 'out_dim' in agg_config
return aggregators.CosPlace(**agg_config)
elif 'gem' in agg_arch.lower():
if agg_config == {}:
agg_config['p'] = 3
else:
assert 'p' in agg_config
return aggregators.GeMPool(**agg_config)
elif 'multiconvap' in agg_arch.lower():
assert 'in_channels' in agg_config
return aggregators.MulConvAP(**agg_config)
elif 'convap' in agg_arch.lower():
assert 'in_channels' in agg_config
return aggregators.ConvAP(**agg_config)
elif 'mixvpr' in agg_arch.lower():
assert 'in_channels' in agg_config
assert 'out_channels' in agg_config
assert 'in_h' in agg_config
assert 'in_w' in agg_config
assert 'mix_depth' in agg_config
return aggregators.MixVPR(**agg_config)
elif 'salad' in agg_arch.lower():
assert 'num_channels' in agg_config
assert 'num_clusters' in agg_config
assert 'cluster_dim' in agg_config
assert 'token_dim' in agg_config
return aggregators.SALAD(**agg_config)
elif 'netvlad' in agg_arch.lower():
return aggregators.NetVLAD()
def get_backbone(backbone_arch='resnet50',
pretrained=True,
layers_to_freeze=2,
layers_to_crop=[],
pretrain_flag=False):
"""Helper function that returns the backbone given its name
Args:
backbone_arch (str, optional): . Defaults to 'resnet50'.
pretrained (bool, optional): . Defaults to True.
layers_to_freeze (int, optional): . Defaults to 2.
layers_to_crop (list, optional): This is mostly used with ResNet where we sometimes need to crop the last residual block (ex. [4]). Defaults to [].
Returns:
model: the backbone as a nn.Model object
"""
if 'resnet' in backbone_arch.lower():
return backbone.ResNet(backbone_arch, pretrained, layers_to_freeze, layers_to_crop, pretrain_flag)
elif 'dinov2' in backbone_arch.lower():
return backbone.DINOv2(model_name=backbone_arch, num_trainable_blocks=4,
norm_layer=True,
return_token=True,
pretrain_flag=pretrain_flag)

122
GeoLoc-UAV-main/models/model.py Normal file
View File

@@ -0,0 +1,122 @@
import numpy as np
import torch.nn as nn
import torch
from models import helper
class GrounpDinoGlobal(nn.Module):
def __init__(self,
groupnet_arch,
agg_arch,
agg_config ):
super(GrounpDinoGlobal, self).__init__()
self.groupnet = helper.get_groupdinonet(groupnet_arch)
self.aggregator = helper.get_aggregator(agg_arch, agg_config)
self.logit_scale = torch.nn.Parameter(torch.ones([]) * np.log(1 / 0.07))
def forward(self, x, pts_list):
local_feature, gfeats_lists = self.groupnet(x, pts_list)
local_feature = local_feature.permute(0,2,1).unsqueeze(-1)
global_feature = self.aggregator(local_feature)
# img_num = len(x)
# bs = x[0][0].shape[0]
# global_feature = torch.zeros(bs*len(x), 256, device='cuda')
# for i in range(img_num):
# imgs, pts = x[i], pts_list[i]
# local_feature = self.groupnet(imgs, pts)
# local_feature = local_feature.permute(0,2,1).unsqueeze(-1)
# des = self.aggregator(local_feature)
# for j in range(len(des)):
# global_feature[j*img_num+i,:] = des[j,:]
return global_feature, gfeats_lists
class GrounpGlobal(nn.Module):
def __init__(self,
groupnet_arch,
agg_arch,
agg_config ):
super(GrounpGlobal, self).__init__()
self.groupnet = helper.get_groupnet(groupnet_arch)
self.aggregator = helper.get_aggregator(agg_arch, agg_config)
self.logit_scale = torch.nn.Parameter(torch.ones([]) * np.log(1 / 0.07))
def forward(self, x, pts_list):
local_feature, gfeats_lists = self.groupnet(x, pts_list)
local_feature = local_feature.permute(0,2,1).unsqueeze(-1)
global_feature = self.aggregator(local_feature)
# img_num = len(x)
# bs = x[0][0].shape[0]
# global_feature = torch.zeros(bs*len(x), 256, device='cuda')
# for i in range(img_num):
# imgs, pts = x[i], pts_list[i]
# local_feature = self.groupnet(imgs, pts)
# local_feature = local_feature.permute(0,2,1).unsqueeze(-1)
# des = self.aggregator(local_feature)
# for j in range(len(des)):
# global_feature[j*img_num+i,:] = des[j,:]
return global_feature, gfeats_lists
class BackboneGlobal(nn.Module):
def __init__(self,
backbone_arch,
pretrain_flag,
agg_arch,
agg_config ):
super(BackboneGlobal, self).__init__()
self.backbone = helper.get_backbone(backbone_arch, pretrain_flag)
self.aggregator = helper.get_aggregator(agg_arch, agg_config)
self.logit_scale = torch.nn.Parameter(torch.ones([]) * np.log(1 / 0.07))
if 'dinov2' in backbone_arch.lower():
self.FLAG = True
else:
self.FLAG = False
def forward(self, x):
local_feature = self.backbone(x)
# dinov2
if self.FLAG:
global_feature = self.aggregator(local_feature[0])
else:
global_feature = self.aggregator(local_feature)
# img_num = len(x)
# bs = x[0][0].shape[0]
# global_feature = torch.zeros(bs*len(x), 256, device='cuda')
# for i in range(img_num):
# imgs, pts = x[i], pts_list[i]
# local_feature = self.groupnet(imgs, pts)
# local_feature = local_feature.permute(0,2,1).unsqueeze(-1)
# des = self.aggregator(local_feature)
# for j in range(len(des)):
# global_feature[j*img_num+i,:] = des[j,:]
return global_feature

231
GeoLoc-UAV-main/models/trainer.py Normal file
View File

@@ -0,0 +1,231 @@
import time
import torch
from tqdm import tqdm
from utils import setting
from torch.cuda.amp import autocast
import torch.nn.functional as F
def train(train_config, model, dataloader, loss_function, optimizer,scheduler=None, scaler=None, writer=None):
# set model train mode
model.train()
losses = setting.AverageMeter()
# wait before starting progress bar
time.sleep(0.1)
# Zero gradients for first step
optimizer.zero_grad(set_to_none=True)
step = 1
if train_config.verbose:
bar = tqdm(dataloader, total=len(dataloader))
else:
bar = dataloader
# for loop over one epoch
# 修改代码为带weight
# for query,query_pt, reference, reference_pt, ids, weight in bar:
for query,query_pt, reference, reference_pt, ids in bar:
if scaler:
with autocast():
# data (batches) to device
query = query
reference = reference
query_pt = query_pt
reference_pt = reference_pt
# Forward pass
features1, _ = model(query, query_pt)
features2, _ = model(reference, reference_pt)
if torch.cuda.device_count() > 1 and len(train_config.gpu_ids) > 1:
loss = loss_function(features1, features2, model.module.logit_scale.exp())
# loss = loss_function(features1, features2, model.module.logit_scale.exp(), weight)
else:
# InfoNCE Loss
loss = loss_function(features1, features2, model.logit_scale.exp())
# loss = loss_function(features1, features2, model.logit_scale.exp(), weight)
# SupCon Loss
# feature = torch.cat((features1, features2), dim=0)
# labels = torch.cat((ids, ids), dim=0)
# loss = loss_function(feature, labels)
losses.update(loss.item())
scaler.scale(loss).backward()
# Gradient clipping
if train_config.clip_grad:
scaler.unscale_(optimizer)
torch.nn.utils.clip_grad_value_(model.parameters(), train_config.clip_grad)
# Update model parameters (weights)
scaler.step(optimizer)
scaler.update()
# Zero gradients for next step
optimizer.zero_grad()
# Scheduler
if train_config.scheduler == "polynomial" or train_config.scheduler == "cosine" or train_config.scheduler == "constant":
scheduler.step()
else:
# data (batches) to device
query = query.to(train_config.device)
reference = reference.to(train_config.device)
# Forward pass
features1, features2 = model(query, reference)
if torch.cuda.device_count() > 1 and len(train_config.gpu_ids) > 1:
# loss = loss_function(features1, features2, model.module.logit_scale.exp(), weight)
loss = loss_function(features1, features2, model.module.logit_scale.exp())
else:
loss = loss_function(features1, features2, model.logit_scale.exp())
# loss = loss_function(features1, features2, model.logit_scale.exp(), weight)
losses.update(loss.item())
# Calculate gradient using backward pass
loss.backward()
# Gradient clipping
if train_config.clip_grad:
torch.nn.utils.clip_grad_value_(model.parameters(), train_config.clip_grad)
# Update model parameters (weights)
optimizer.step()
# Zero gradients for next step
optimizer.zero_grad()
# Scheduler
if train_config.scheduler == "polynomial" or train_config.scheduler == "cosine" or train_config.scheduler == "constant":
scheduler.step()
if train_config.verbose:
monitor = {"loss": "{:.4f}".format(loss.item()),
"loss_avg": "{:.4f}".format(losses.avg),
"lr" : "{:.6f}".format(optimizer.param_groups[0]['lr'])}
bar.set_postfix(ordered_dict=monitor)
writer.add_scalar('Loss/train', loss.item(), step)
writer.add_scalar('Loss/avg_loss', losses.avg, step)
writer.add_scalar('lr', optimizer.param_groups[0]['lr'], step)
step += 1
if train_config.verbose:
bar.close()
return losses.avg
def train_backbone(train_config, model, dataloader, loss_function, optimizer, scheduler=None, scaler=None, writer=None, LPN=False):
# set model train mode
model.train()
losses = setting.AverageMeter()
# wait before starting progress bar
time.sleep(0.1)
# Zero gradients for first step
optimizer.zero_grad(set_to_none=True)
step = 1
if train_config.verbose:
bar = tqdm(dataloader, total=len(dataloader))
else:
bar = dataloader
# for loop over one epoch
for query, reference, ids in bar:
loss = 0.0
query = query.to(train_config.device)
reference = reference.to(train_config.device)
# Forward pass
features1 = model(query)
features2 = model(reference)
if LPN == False:
if torch.cuda.device_count() > 1 and len(train_config.gpu_ids) > 1:
loss = loss_function(features1, features2, model.module.logit_scale.exp())
else:
loss = loss_function(features1, features2, model.logit_scale.exp())
else:
for index in range(len(features1)):
feature1_one = features1[index]
feature2_one = features2[index]
if torch.cuda.device_count() > 1 and len(train_config.gpu_ids) > 1:
temp_loss = loss_function(feature1_one, feature2_one, model.module.logit_scale.exp())
else:
temp_loss = loss_function(feature1_one, feature2_one, model.logit_scale.exp())
loss += temp_loss
losses.update(loss.item())
# Zero gradients for next step
optimizer.zero_grad()
# Scheduler
if train_config.scheduler == "polynomial" or train_config.scheduler == "cosine" or train_config.scheduler == "constant":
scheduler.step()
losses.update(loss.item())
# Calculate gradient using backward pass
loss.backward(retain_graph=True)
# Update model parameters (weights)
optimizer.step()
# Zero gradients for next step
optimizer.zero_grad()
# Scheduler
if train_config.scheduler == "polynomial" or train_config.scheduler == "cosine" or train_config.scheduler == "constant":
scheduler.step()
if train_config.verbose:
monitor = {"loss": "{:.4f}".format(loss.item()),
"loss_avg": "{:.4f}".format(losses.avg),
"lr" : "{:.6f}".format(optimizer.param_groups[0]['lr'])}
bar.set_postfix(ordered_dict=monitor)
writer.add_scalar('Loss/train', loss.item(), step)
writer.add_scalar('Loss/avg_loss', losses.avg, step)
writer.add_scalar('lr', optimizer.param_groups[0]['lr'], step)
step += 1
if train_config.verbose:
bar.close()
return losses.avg

48
GeoLoc-UAV-main/models/transformer/__init__.py Normal file
View File

@@ -0,0 +1,48 @@
import torch
import torch.nn as nn
import torch.nn.functional as F
from romatch.utils.utils import get_grid, get_autocast_params
from .layers.block import Block
from .layers.attention import MemEffAttention
from .dinov2 import vit_large, vit_small
class TransformerDecoder(nn.Module):
def __init__(self, blocks, hidden_dim, out_dim, is_classifier = False, *args,
amp = False, pos_enc = True, learned_embeddings = False, embedding_dim = None, amp_dtype = torch.float16, **kwargs) -> None:
super().__init__(*args, **kwargs)
self.blocks = blocks
self.to_out = nn.Linear(hidden_dim, out_dim)
self.hidden_dim = hidden_dim
self.out_dim = out_dim
self._scales = [16]
self.is_classifier = is_classifier
self.amp = amp
self.amp_dtype = amp_dtype
self.pos_enc = pos_enc
self.learned_embeddings = learned_embeddings
if self.learned_embeddings:
self.learned_pos_embeddings = nn.Parameter(nn.init.kaiming_normal_(torch.empty((1, hidden_dim, embedding_dim, embedding_dim))))
def scales(self):
return self._scales.copy()
def forward(self, gp_posterior, features, old_stuff, new_scale):
autocast_device, autocast_enabled, autocast_dtype = get_autocast_params(gp_posterior.device, enabled=self.amp, dtype=self.amp_dtype)
with torch.autocast(autocast_device, enabled=autocast_enabled, dtype = autocast_dtype):
B,C,H,W = gp_posterior.shape
x = torch.cat((gp_posterior, features), dim = 1)
B,C,H,W = x.shape
grid = get_grid(B, H, W, x.device).reshape(B,H*W,2)
if self.learned_embeddings:
pos_enc = F.interpolate(self.learned_pos_embeddings, size = (H,W), mode = 'bilinear', align_corners = False).permute(0,2,3,1).reshape(1,H*W,C)
else:
pos_enc = 0
tokens = x.reshape(B,C,H*W).permute(0,2,1) + pos_enc
z = self.blocks(tokens)
out = self.to_out(z)
out = out.permute(0,2,1).reshape(B, self.out_dim, H, W)
warp, certainty = out[:, :-1], out[:, -1:]
return warp, certainty, None

360
GeoLoc-UAV-main/models/transformer/dinov2.py Normal file
View File

@@ -0,0 +1,360 @@
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under the license found in the
# LICENSE file in the root directory of this source tree.
# References:
# https://github.com/facebookresearch/dino/blob/main/vision_transformer.py
# https://github.com/rwightman/pytorch-image-models/tree/master/timm/models/vision_transformer.py
from functools import partial
import math
import logging
from typing import Sequence, Tuple, Union, Callable
import torch
import torch.nn as nn
import torch.utils.checkpoint
from torch.nn.init import trunc_normal_
from .layers import Mlp, PatchEmbed, SwiGLUFFNFused, MemEffAttention, NestedTensorBlock as Block
def named_apply(fn: Callable, module: nn.Module, name="", depth_first=True, include_root=False) -> nn.Module:
if not depth_first and include_root:
fn(module=module, name=name)
for child_name, child_module in module.named_children():
child_name = ".".join((name, child_name)) if name else child_name
named_apply(fn=fn, module=child_module, name=child_name, depth_first=depth_first, include_root=True)
if depth_first and include_root:
fn(module=module, name=name)
return module
class BlockChunk(nn.ModuleList):
def forward(self, x):
for b in self:
x = b(x)
return x
class DinoVisionTransformer(nn.Module):
def __init__(
self,
img_size=224,
patch_size=16,
in_chans=3,
embed_dim=768,
depth=12,
num_heads=12,
mlp_ratio=4.0,
qkv_bias=True,
ffn_bias=True,
proj_bias=True,
drop_path_rate=0.0,
drop_path_uniform=False,
init_values=None, # for layerscale: None or 0 => no layerscale
embed_layer=PatchEmbed,
act_layer=nn.GELU,
block_fn=Block,
ffn_layer="mlp",
block_chunks=1,
):
"""
Args:
img_size (int, tuple): input image size
patch_size (int, tuple): patch size
in_chans (int): number of input channels
embed_dim (int): embedding dimension
depth (int): depth of transformer
num_heads (int): number of attention heads
mlp_ratio (int): ratio of mlp hidden dim to embedding dim
qkv_bias (bool): enable bias for qkv if True
proj_bias (bool): enable bias for proj in attn if True
ffn_bias (bool): enable bias for ffn if True
drop_path_rate (float): stochastic depth rate
drop_path_uniform (bool): apply uniform drop rate across blocks
weight_init (str): weight init scheme
init_values (float): layer-scale init values
embed_layer (nn.Module): patch embedding layer
act_layer (nn.Module): MLP activation layer
block_fn (nn.Module): transformer block class
ffn_layer (str): "mlp", "swiglu", "swiglufused" or "identity"
block_chunks: (int) split block sequence into block_chunks units for FSDP wrap
"""
super().__init__()
norm_layer = partial(nn.LayerNorm, eps=1e-6)
self.num_features = self.embed_dim = embed_dim # num_features for consistency with other models
self.num_tokens = 1
self.n_blocks = depth
self.num_heads = num_heads
self.patch_size = patch_size
self.patch_embed = embed_layer(img_size=img_size, patch_size=patch_size, in_chans=in_chans, embed_dim=embed_dim)
num_patches = self.patch_embed.num_patches
self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim))
self.pos_embed = nn.Parameter(torch.zeros(1, num_patches + self.num_tokens, embed_dim))
if drop_path_uniform is True:
dpr = [drop_path_rate] * depth
else:
dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)] # stochastic depth decay rule
if ffn_layer == "mlp":
ffn_layer = Mlp
elif ffn_layer == "swiglufused" or ffn_layer == "swiglu":
ffn_layer = SwiGLUFFNFused
elif ffn_layer == "identity":
def f(*args, **kwargs):
return nn.Identity()
ffn_layer = f
else:
raise NotImplementedError
blocks_list = [
block_fn(
dim=embed_dim,
num_heads=num_heads,
mlp_ratio=mlp_ratio,
qkv_bias=qkv_bias,
proj_bias=proj_bias,
ffn_bias=ffn_bias,
drop_path=dpr[i],
norm_layer=norm_layer,
act_layer=act_layer,
ffn_layer=ffn_layer,
init_values=init_values,
)
for i in range(depth)
]
if block_chunks > 0:
self.chunked_blocks = True
chunked_blocks = []
chunksize = depth // block_chunks
for i in range(0, depth, chunksize):
# this is to keep the block index consistent if we chunk the block list
chunked_blocks.append([nn.Identity()] * i + blocks_list[i : i + chunksize])
self.blocks = nn.ModuleList([BlockChunk(p) for p in chunked_blocks])
else:
self.chunked_blocks = False
self.blocks = nn.ModuleList(blocks_list)
self.norm = norm_layer(embed_dim)
self.head = nn.Identity()
self.mask_token = nn.Parameter(torch.zeros(1, embed_dim))
self.init_weights()
for param in self.parameters():
param.requires_grad = False
@property
def device(self):
return self.cls_token.device
def init_weights(self):
trunc_normal_(self.pos_embed, std=0.02)
nn.init.normal_(self.cls_token, std=1e-6)
named_apply(init_weights_vit_timm, self)
def interpolate_pos_encoding(self, x, w, h):
previous_dtype = x.dtype
npatch = x.shape[1] - 1
N = self.pos_embed.shape[1] - 1
if npatch == N and w == h:
return self.pos_embed
pos_embed = self.pos_embed.float()
class_pos_embed = pos_embed[:, 0]
patch_pos_embed = pos_embed[:, 1:]
dim = x.shape[-1]
w0 = w // self.patch_size
h0 = h // self.patch_size
# we add a small number to avoid floating point error in the interpolation
# see discussion at https://github.com/facebookresearch/dino/issues/8
w0, h0 = w0 + 0.1, h0 + 0.1
patch_pos_embed = nn.functional.interpolate(
patch_pos_embed.reshape(1, int(math.sqrt(N)), int(math.sqrt(N)), dim).permute(0, 3, 1, 2),
scale_factor=(w0 / math.sqrt(N), h0 / math.sqrt(N)),
mode="bicubic",
)
assert int(w0) == patch_pos_embed.shape[-2] and int(h0) == patch_pos_embed.shape[-1]
patch_pos_embed = patch_pos_embed.permute(0, 2, 3, 1).view(1, -1, dim)
return torch.cat((class_pos_embed.unsqueeze(0), patch_pos_embed), dim=1).to(previous_dtype)
def prepare_tokens_with_masks(self, x, masks=None):
B, nc, w, h = x.shape
x = self.patch_embed(x)
if masks is not None:
x = torch.where(masks.unsqueeze(-1), self.mask_token.to(x.dtype).unsqueeze(0), x)
x = torch.cat((self.cls_token.expand(x.shape[0], -1, -1), x), dim=1)
x = x + self.interpolate_pos_encoding(x, w, h)
return x
def forward_features_list(self, x_list, masks_list):
x = [self.prepare_tokens_with_masks(x, masks) for x, masks in zip(x_list, masks_list)]
for blk in self.blocks:
x = blk(x)
all_x = x
output = []
for x, masks in zip(all_x, masks_list):
x_norm = self.norm(x)
output.append(
{
"x_norm_clstoken": x_norm[:, 0],
"x_norm_patchtokens": x_norm[:, 1:],
"x_prenorm": x,
"masks": masks,
}
)
return output
def forward_features(self, x, masks=None):
if isinstance(x, list):
return self.forward_features_list(x, masks)
x = self.prepare_tokens_with_masks(x, masks)
for blk in self.blocks:
x = blk(x)
x_norm = self.norm(x)
# import pdb;pdb.set_trace()
return {
"x_norm_clstoken": x_norm[:, 0],
"x_norm_patchtokens": x_norm[:, 1:],
"x_prenorm": x,
"masks": masks,
}
def _get_intermediate_layers_not_chunked(self, x, n=1):
x = self.prepare_tokens_with_masks(x)
# If n is an int, take the n last blocks. If it's a list, take them
output, total_block_len = [], len(self.blocks)
blocks_to_take = range(total_block_len - n, total_block_len) if isinstance(n, int) else n
for i, blk in enumerate(self.blocks):
x = blk(x)
if i in blocks_to_take:
output.append(x)
assert len(output) == len(blocks_to_take), f"only {len(output)} / {len(blocks_to_take)} blocks found"
return output
def _get_intermediate_layers_chunked(self, x, n=1):
x = self.prepare_tokens_with_masks(x)
output, i, total_block_len = [], 0, len(self.blocks[-1])
# If n is an int, take the n last blocks. If it's a list, take them
blocks_to_take = range(total_block_len - n, total_block_len) if isinstance(n, int) else n
for block_chunk in self.blocks:
for blk in block_chunk[i:]: # Passing the nn.Identity()
x = blk(x)
if i in blocks_to_take:
output.append(x)
i += 1
assert len(output) == len(blocks_to_take), f"only {len(output)} / {len(blocks_to_take)} blocks found"
return output
def get_intermediate_layers(
self,
x: torch.Tensor,
n: Union[int, Sequence] = 1, # Layers or n last layers to take
reshape: bool = False,
return_class_token: bool = False,
norm=True,
) -> Tuple[Union[torch.Tensor, Tuple[torch.Tensor]]]:
if self.chunked_blocks:
outputs = self._get_intermediate_layers_chunked(x, n)
else:
outputs = self._get_intermediate_layers_not_chunked(x, n)
if norm:
outputs = [self.norm(out) for out in outputs]
class_tokens = [out[:, 0] for out in outputs]
outputs = [out[:, 1:] for out in outputs]
if reshape:
B, _, w, h = x.shape
outputs = [
out.reshape(B, w // self.patch_size, h // self.patch_size, -1).permute(0, 3, 1, 2).contiguous()
for out in outputs
]
if return_class_token:
return tuple(zip(outputs, class_tokens))
return tuple(outputs)
def forward(self, *args, is_training=False, **kwargs):
ret = self.forward_features(*args, **kwargs)
if is_training:
return ret
else:
return self.head(ret["x_norm_clstoken"])
def init_weights_vit_timm(module: nn.Module, name: str = ""):
"""ViT weight initialization, original timm impl (for reproducibility)"""
if isinstance(module, nn.Linear):
trunc_normal_(module.weight, std=0.02)
if module.bias is not None:
nn.init.zeros_(module.bias)
def vit_small(patch_size=16, **kwargs):
model = DinoVisionTransformer(
patch_size=patch_size,
embed_dim=384,
depth=12,
num_heads=6,
mlp_ratio=4,
block_fn=partial(Block, attn_class=MemEffAttention),
**kwargs,
)
return model
def vit_base(patch_size=16, **kwargs):
model = DinoVisionTransformer(
patch_size=patch_size,
embed_dim=768,
depth=12,
num_heads=12,
mlp_ratio=4,
block_fn=partial(Block, attn_class=MemEffAttention),
**kwargs,
)
return model
def vit_large(patch_size=16, **kwargs):
model = DinoVisionTransformer(
patch_size=patch_size,
embed_dim=1024,
depth=24,
num_heads=16,
mlp_ratio=4,
block_fn=partial(Block, attn_class=MemEffAttention),
**kwargs,
)
return model
def vit_giant2(patch_size=16, **kwargs):
"""
Close to ViT-giant, with embed-dim 1536 and 24 heads => embed-dim per head 64
"""
model = DinoVisionTransformer(
patch_size=patch_size,
embed_dim=1536,
depth=40,
num_heads=24,
mlp_ratio=4,
block_fn=partial(Block, attn_class=MemEffAttention),
**kwargs,
)
return model

12
GeoLoc-UAV-main/models/transformer/layers/__init__.py Normal file
View File

@@ -0,0 +1,12 @@
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under the license found in the
# LICENSE file in the root directory of this source tree.
from .dino_head import DINOHead
from .mlp import Mlp
from .patch_embed import PatchEmbed
from .swiglu_ffn import SwiGLUFFN, SwiGLUFFNFused
from .block import NestedTensorBlock
from .attention import MemEffAttention

81
GeoLoc-UAV-main/models/transformer/layers/attention.py Normal file
View File

@@ -0,0 +1,81 @@
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under the license found in the
# LICENSE file in the root directory of this source tree.
# References:
# https://github.com/facebookresearch/dino/blob/master/vision_transformer.py
# https://github.com/rwightman/pytorch-image-models/tree/master/timm/models/vision_transformer.py
import logging
from torch import Tensor
from torch import nn
logger = logging.getLogger("dinov2")
try:
from xformers.ops import memory_efficient_attention, unbind, fmha
XFORMERS_AVAILABLE = True
except ImportError:
logger.warning("xFormers not available")
XFORMERS_AVAILABLE = False
class Attention(nn.Module):
def __init__(
self,
dim: int,
num_heads: int = 8,
qkv_bias: bool = False,
proj_bias: bool = True,
attn_drop: float = 0.0,
proj_drop: float = 0.0,
) -> None:
super().__init__()
self.num_heads = num_heads
head_dim = dim // num_heads
self.scale = head_dim**-0.5
self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
self.attn_drop = nn.Dropout(attn_drop)
self.proj = nn.Linear(dim, dim, bias=proj_bias)
self.proj_drop = nn.Dropout(proj_drop)
def forward(self, x: Tensor) -> Tensor:
B, N, C = x.shape
qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4)
q, k, v = qkv[0] * self.scale, qkv[1], qkv[2]
attn = q @ k.transpose(-2, -1)
attn = attn.softmax(dim=-1)
attn = self.attn_drop(attn)
x = (attn @ v).transpose(1, 2).reshape(B, N, C)
x = self.proj(x)
x = self.proj_drop(x)
return x
class MemEffAttention(Attention):
def forward(self, x: Tensor, attn_bias=None) -> Tensor:
if not XFORMERS_AVAILABLE:
assert attn_bias is None, "xFormers is required for nested tensors usage"
return super().forward(x)
B, N, C = x.shape
qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads)
q, k, v = unbind(qkv, 2)
x = memory_efficient_attention(q, k, v, attn_bias=attn_bias)
x = x.reshape([B, N, C])
x = self.proj(x)
x = self.proj_drop(x)
return x

252
GeoLoc-UAV-main/models/transformer/layers/block.py Normal file
View File

@@ -0,0 +1,252 @@
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under the license found in the
# LICENSE file in the root directory of this source tree.
# References:
# https://github.com/facebookresearch/dino/blob/master/vision_transformer.py
# https://github.com/rwightman/pytorch-image-models/tree/master/timm/layers/patch_embed.py
import logging
from typing import Callable, List, Any, Tuple, Dict
import torch
from torch import nn, Tensor
from .attention import Attention, MemEffAttention
from .drop_path import DropPath
from .layer_scale import LayerScale
from .mlp import Mlp
logger = logging.getLogger("dinov2")
try:
from xformers.ops import fmha
from xformers.ops import scaled_index_add, index_select_cat
XFORMERS_AVAILABLE = True
except ImportError:
logger.warning("xFormers not available")
XFORMERS_AVAILABLE = False
class Block(nn.Module):
def __init__(
self,
dim: int,
num_heads: int,
mlp_ratio: float = 4.0,
qkv_bias: bool = False,
proj_bias: bool = True,
ffn_bias: bool = True,
drop: float = 0.0,
attn_drop: float = 0.0,
init_values=None,
drop_path: float = 0.0,
act_layer: Callable[..., nn.Module] = nn.GELU,
norm_layer: Callable[..., nn.Module] = nn.LayerNorm,
attn_class: Callable[..., nn.Module] = Attention,
ffn_layer: Callable[..., nn.Module] = Mlp,
) -> None:
super().__init__()
# print(f"biases: qkv: {qkv_bias}, proj: {proj_bias}, ffn: {ffn_bias}")
self.norm1 = norm_layer(dim)
self.attn = attn_class(
dim,
num_heads=num_heads,
qkv_bias=qkv_bias,
proj_bias=proj_bias,
attn_drop=attn_drop,
proj_drop=drop,
)
self.ls1 = LayerScale(dim, init_values=init_values) if init_values else nn.Identity()
self.drop_path1 = DropPath(drop_path) if drop_path > 0.0 else nn.Identity()
self.norm2 = norm_layer(dim)
mlp_hidden_dim = int(dim * mlp_ratio)
self.mlp = ffn_layer(
in_features=dim,
hidden_features=mlp_hidden_dim,
act_layer=act_layer,
drop=drop,
bias=ffn_bias,
)
self.ls2 = LayerScale(dim, init_values=init_values) if init_values else nn.Identity()
self.drop_path2 = DropPath(drop_path) if drop_path > 0.0 else nn.Identity()
self.sample_drop_ratio = drop_path
def forward(self, x: Tensor) -> Tensor:
def attn_residual_func(x: Tensor) -> Tensor:
return self.ls1(self.attn(self.norm1(x)))
def ffn_residual_func(x: Tensor) -> Tensor:
return self.ls2(self.mlp(self.norm2(x)))
if self.training and self.sample_drop_ratio > 0.1:
# the overhead is compensated only for a drop path rate larger than 0.1
x = drop_add_residual_stochastic_depth(
x,
residual_func=attn_residual_func,
sample_drop_ratio=self.sample_drop_ratio,
)
x = drop_add_residual_stochastic_depth(
x,
residual_func=ffn_residual_func,
sample_drop_ratio=self.sample_drop_ratio,
)
elif self.training and self.sample_drop_ratio > 0.0:
x = x + self.drop_path1(attn_residual_func(x))
x = x + self.drop_path1(ffn_residual_func(x)) # FIXME: drop_path2
else:
x = x + attn_residual_func(x)
x = x + ffn_residual_func(x)
return x
def drop_add_residual_stochastic_depth(
x: Tensor,
residual_func: Callable[[Tensor], Tensor],
sample_drop_ratio: float = 0.0,
) -> Tensor:
# 1) extract subset using permutation
b, n, d = x.shape
sample_subset_size = max(int(b * (1 - sample_drop_ratio)), 1)
brange = (torch.randperm(b, device=x.device))[:sample_subset_size]
x_subset = x[brange]
# 2) apply residual_func to get residual
residual = residual_func(x_subset)
x_flat = x.flatten(1)
residual = residual.flatten(1)
residual_scale_factor = b / sample_subset_size
# 3) add the residual
x_plus_residual = torch.index_add(x_flat, 0, brange, residual.to(dtype=x.dtype), alpha=residual_scale_factor)
return x_plus_residual.view_as(x)
def get_branges_scales(x, sample_drop_ratio=0.0):
b, n, d = x.shape
sample_subset_size = max(int(b * (1 - sample_drop_ratio)), 1)
brange = (torch.randperm(b, device=x.device))[:sample_subset_size]
residual_scale_factor = b / sample_subset_size
return brange, residual_scale_factor
def add_residual(x, brange, residual, residual_scale_factor, scaling_vector=None):
if scaling_vector is None:
x_flat = x.flatten(1)
residual = residual.flatten(1)
x_plus_residual = torch.index_add(x_flat, 0, brange, residual.to(dtype=x.dtype), alpha=residual_scale_factor)
else:
x_plus_residual = scaled_index_add(
x, brange, residual.to(dtype=x.dtype), scaling=scaling_vector, alpha=residual_scale_factor
)
return x_plus_residual
attn_bias_cache: Dict[Tuple, Any] = {}
def get_attn_bias_and_cat(x_list, branges=None):
"""
this will perform the index select, cat the tensors, and provide the attn_bias from cache
"""
batch_sizes = [b.shape[0] for b in branges] if branges is not None else [x.shape[0] for x in x_list]
all_shapes = tuple((b, x.shape[1]) for b, x in zip(batch_sizes, x_list))
if all_shapes not in attn_bias_cache.keys():
seqlens = []
for b, x in zip(batch_sizes, x_list):
for _ in range(b):
seqlens.append(x.shape[1])
attn_bias = fmha.BlockDiagonalMask.from_seqlens(seqlens)
attn_bias._batch_sizes = batch_sizes
attn_bias_cache[all_shapes] = attn_bias
if branges is not None:
cat_tensors = index_select_cat([x.flatten(1) for x in x_list], branges).view(1, -1, x_list[0].shape[-1])
else:
tensors_bs1 = tuple(x.reshape([1, -1, *x.shape[2:]]) for x in x_list)
cat_tensors = torch.cat(tensors_bs1, dim=1)
return attn_bias_cache[all_shapes], cat_tensors
def drop_add_residual_stochastic_depth_list(
x_list: List[Tensor],
residual_func: Callable[[Tensor, Any], Tensor],
sample_drop_ratio: float = 0.0,
scaling_vector=None,
) -> Tensor:
# 1) generate random set of indices for dropping samples in the batch
branges_scales = [get_branges_scales(x, sample_drop_ratio=sample_drop_ratio) for x in x_list]
branges = [s[0] for s in branges_scales]
residual_scale_factors = [s[1] for s in branges_scales]
# 2) get attention bias and index+concat the tensors
attn_bias, x_cat = get_attn_bias_and_cat(x_list, branges)
# 3) apply residual_func to get residual, and split the result
residual_list = attn_bias.split(residual_func(x_cat, attn_bias=attn_bias)) # type: ignore
outputs = []
for x, brange, residual, residual_scale_factor in zip(x_list, branges, residual_list, residual_scale_factors):
outputs.append(add_residual(x, brange, residual, residual_scale_factor, scaling_vector).view_as(x))
return outputs
class NestedTensorBlock(Block):
def forward_nested(self, x_list: List[Tensor]) -> List[Tensor]:
"""
x_list contains a list of tensors to nest together and run
"""
assert isinstance(self.attn, MemEffAttention)
if self.training and self.sample_drop_ratio > 0.0:
def attn_residual_func(x: Tensor, attn_bias=None) -> Tensor:
return self.attn(self.norm1(x), attn_bias=attn_bias)
def ffn_residual_func(x: Tensor, attn_bias=None) -> Tensor:
return self.mlp(self.norm2(x))
x_list = drop_add_residual_stochastic_depth_list(
x_list,
residual_func=attn_residual_func,
sample_drop_ratio=self.sample_drop_ratio,
scaling_vector=self.ls1.gamma if isinstance(self.ls1, LayerScale) else None,
)
x_list = drop_add_residual_stochastic_depth_list(
x_list,
residual_func=ffn_residual_func,
sample_drop_ratio=self.sample_drop_ratio,
scaling_vector=self.ls2.gamma if isinstance(self.ls1, LayerScale) else None,
)
return x_list
else:
def attn_residual_func(x: Tensor, attn_bias=None) -> Tensor:
return self.ls1(self.attn(self.norm1(x), attn_bias=attn_bias))
def ffn_residual_func(x: Tensor, attn_bias=None) -> Tensor:
return self.ls2(self.mlp(self.norm2(x)))
attn_bias, x = get_attn_bias_and_cat(x_list)
x = x + attn_residual_func(x, attn_bias=attn_bias)
x = x + ffn_residual_func(x)
return attn_bias.split(x)
def forward(self, x_or_x_list):
if isinstance(x_or_x_list, Tensor):
return super().forward(x_or_x_list)
elif isinstance(x_or_x_list, list):
assert XFORMERS_AVAILABLE, "Please install xFormers for nested tensors usage"
return self.forward_nested(x_or_x_list)
else:
raise AssertionError

59
GeoLoc-UAV-main/models/transformer/layers/dino_head.py Normal file
View File

@@ -0,0 +1,59 @@
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under the license found in the
# LICENSE file in the root directory of this source tree.
import torch
import torch.nn as nn
from torch.nn.init import trunc_normal_
from torch.nn.utils import weight_norm
class DINOHead(nn.Module):
def __init__(
self,
in_dim,
out_dim,
use_bn=False,
nlayers=3,
hidden_dim=2048,
bottleneck_dim=256,
mlp_bias=True,
):
super().__init__()
nlayers = max(nlayers, 1)
self.mlp = _build_mlp(nlayers, in_dim, bottleneck_dim, hidden_dim=hidden_dim, use_bn=use_bn, bias=mlp_bias)
self.apply(self._init_weights)
self.last_layer = weight_norm(nn.Linear(bottleneck_dim, out_dim, bias=False))
self.last_layer.weight_g.data.fill_(1)
def _init_weights(self, m):
if isinstance(m, nn.Linear):
trunc_normal_(m.weight, std=0.02)
if isinstance(m, nn.Linear) and m.bias is not None:
nn.init.constant_(m.bias, 0)
def forward(self, x):
x = self.mlp(x)
eps = 1e-6 if x.dtype == torch.float16 else 1e-12
x = nn.functional.normalize(x, dim=-1, p=2, eps=eps)
x = self.last_layer(x)
return x
def _build_mlp(nlayers, in_dim, bottleneck_dim, hidden_dim=None, use_bn=False, bias=True):
if nlayers == 1:
return nn.Linear(in_dim, bottleneck_dim, bias=bias)
else:
layers = [nn.Linear(in_dim, hidden_dim, bias=bias)]
if use_bn:
layers.append(nn.BatchNorm1d(hidden_dim))
layers.append(nn.GELU())
for _ in range(nlayers - 2):
layers.append(nn.Linear(hidden_dim, hidden_dim, bias=bias))
if use_bn:
layers.append(nn.BatchNorm1d(hidden_dim))
layers.append(nn.GELU())
layers.append(nn.Linear(hidden_dim, bottleneck_dim, bias=bias))
return nn.Sequential(*layers)

35
GeoLoc-UAV-main/models/transformer/layers/drop_path.py Normal file
View File

@@ -0,0 +1,35 @@
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under the license found in the
# LICENSE file in the root directory of this source tree.
# References:
# https://github.com/facebookresearch/dino/blob/master/vision_transformer.py
# https://github.com/rwightman/pytorch-image-models/tree/master/timm/layers/drop.py
from torch import nn
def drop_path(x, drop_prob: float = 0.0, training: bool = False):
if drop_prob == 0.0 or not training:
return x
keep_prob = 1 - drop_prob
shape = (x.shape[0],) + (1,) * (x.ndim - 1) # work with diff dim tensors, not just 2D ConvNets
random_tensor = x.new_empty(shape).bernoulli_(keep_prob)
if keep_prob > 0.0:
random_tensor.div_(keep_prob)
output = x * random_tensor
return output
class DropPath(nn.Module):
"""Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks)."""
def __init__(self, drop_prob=None):
super(DropPath, self).__init__()
self.drop_prob = drop_prob
def forward(self, x):
return drop_path(x, self.drop_prob, self.training)

28
GeoLoc-UAV-main/models/transformer/layers/layer_scale.py Normal file
View File

@@ -0,0 +1,28 @@
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under the license found in the
# LICENSE file in the root directory of this source tree.
# Modified from: https://github.com/huggingface/pytorch-image-models/blob/main/timm/models/vision_transformer.py#L103-L110
from typing import Union
import torch
from torch import Tensor
from torch import nn
class LayerScale(nn.Module):
def __init__(
self,
dim: int,
init_values: Union[float, Tensor] = 1e-5,
inplace: bool = False,
) -> None:
super().__init__()
self.inplace = inplace
self.gamma = nn.Parameter(init_values * torch.ones(dim))
def forward(self, x: Tensor) -> Tensor:
return x.mul_(self.gamma) if self.inplace else x * self.gamma

41
GeoLoc-UAV-main/models/transformer/layers/mlp.py Normal file
View File

@@ -0,0 +1,41 @@
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under the license found in the
# LICENSE file in the root directory of this source tree.
# References:
# https://github.com/facebookresearch/dino/blob/master/vision_transformer.py
# https://github.com/rwightman/pytorch-image-models/tree/master/timm/layers/mlp.py
from typing import Callable, Optional
from torch import Tensor, nn
class Mlp(nn.Module):
def __init__(
self,
in_features: int,
hidden_features: Optional[int] = None,
out_features: Optional[int] = None,
act_layer: Callable[..., nn.Module] = nn.GELU,
drop: float = 0.0,
bias: bool = True,
) -> None:
super().__init__()
out_features = out_features or in_features
hidden_features = hidden_features or in_features
self.fc1 = nn.Linear(in_features, hidden_features, bias=bias)
self.act = act_layer()
self.fc2 = nn.Linear(hidden_features, out_features, bias=bias)
self.drop = nn.Dropout(drop)
def forward(self, x: Tensor) -> Tensor:
x = self.fc1(x)
x = self.act(x)
x = self.drop(x)
x = self.fc2(x)
x = self.drop(x)
return x

89
GeoLoc-UAV-main/models/transformer/layers/patch_embed.py Normal file
View File

@@ -0,0 +1,89 @@
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under the license found in the
# LICENSE file in the root directory of this source tree.
# References:
# https://github.com/facebookresearch/dino/blob/master/vision_transformer.py
# https://github.com/rwightman/pytorch-image-models/tree/master/timm/layers/patch_embed.py
from typing import Callable, Optional, Tuple, Union
from torch import Tensor
import torch.nn as nn
def make_2tuple(x):
if isinstance(x, tuple):
assert len(x) == 2
return x
assert isinstance(x, int)
return (x, x)
class PatchEmbed(nn.Module):
"""
2D image to patch embedding: (B,C,H,W) -> (B,N,D)
Args:
img_size: Image size.
patch_size: Patch token size.
in_chans: Number of input image channels.
embed_dim: Number of linear projection output channels.
norm_layer: Normalization layer.
"""
def __init__(
self,
img_size: Union[int, Tuple[int, int]] = 224,
patch_size: Union[int, Tuple[int, int]] = 16,
in_chans: int = 3,
embed_dim: int = 768,
norm_layer: Optional[Callable] = None,
flatten_embedding: bool = True,
) -> None:
super().__init__()
image_HW = make_2tuple(img_size)
patch_HW = make_2tuple(patch_size)
patch_grid_size = (
image_HW[0] // patch_HW[0],
image_HW[1] // patch_HW[1],
)
self.img_size = image_HW
self.patch_size = patch_HW
self.patches_resolution = patch_grid_size
self.num_patches = patch_grid_size[0] * patch_grid_size[1]
self.in_chans = in_chans
self.embed_dim = embed_dim
self.flatten_embedding = flatten_embedding
self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_HW, stride=patch_HW)
self.norm = norm_layer(embed_dim) if norm_layer else nn.Identity()
def forward(self, x: Tensor) -> Tensor:
_, _, H, W = x.shape
patch_H, patch_W = self.patch_size
assert H % patch_H == 0, f"Input image height {H} is not a multiple of patch height {patch_H}"
assert W % patch_W == 0, f"Input image width {W} is not a multiple of patch width: {patch_W}"
x = self.proj(x) # B C H W
H, W = x.size(2), x.size(3)
x = x.flatten(2).transpose(1, 2) # B HW C
x = self.norm(x)
if not self.flatten_embedding:
x = x.reshape(-1, H, W, self.embed_dim) # B H W C
return x
def flops(self) -> float:
Ho, Wo = self.patches_resolution
flops = Ho * Wo * self.embed_dim * self.in_chans * (self.patch_size[0] * self.patch_size[1])
if self.norm is not None:
flops += Ho * Wo * self.embed_dim
return flops

63
GeoLoc-UAV-main/models/transformer/layers/swiglu_ffn.py Normal file
View File

@@ -0,0 +1,63 @@
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under the license found in the
# LICENSE file in the root directory of this source tree.
from typing import Callable, Optional
from torch import Tensor, nn
import torch.nn.functional as F
class SwiGLUFFN(nn.Module):
def __init__(
self,
in_features: int,
hidden_features: Optional[int] = None,
out_features: Optional[int] = None,
act_layer: Callable[..., nn.Module] = None,
drop: float = 0.0,
bias: bool = True,
) -> None:
super().__init__()
out_features = out_features or in_features
hidden_features = hidden_features or in_features
self.w12 = nn.Linear(in_features, 2 * hidden_features, bias=bias)
self.w3 = nn.Linear(hidden_features, out_features, bias=bias)
def forward(self, x: Tensor) -> Tensor:
x12 = self.w12(x)
x1, x2 = x12.chunk(2, dim=-1)
hidden = F.silu(x1) * x2
return self.w3(hidden)
try:
from xformers.ops import SwiGLU
XFORMERS_AVAILABLE = True
except ImportError:
SwiGLU = SwiGLUFFN
XFORMERS_AVAILABLE = False
class SwiGLUFFNFused(SwiGLU):
def __init__(
self,
in_features: int,
hidden_features: Optional[int] = None,
out_features: Optional[int] = None,
act_layer: Callable[..., nn.Module] = None,
drop: float = 0.0,
bias: bool = True,
) -> None:
out_features = out_features or in_features
hidden_features = hidden_features or in_features
hidden_features = (int(hidden_features * 2 / 3) + 7) // 8 * 8
super().__init__(
in_features=in_features,
hidden_features=hidden_features,
out_features=out_features,
bias=bias,
)