Pikaliov/World-UAV-ds
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial import: World-UAV prepro

Browse Source 
Add dataloaders (v1/v2), analysis scripts, and documentation for working with UAV-GeoLoc (World-UAV).

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Pikaliov
2026-05-09 12:44:49 +03:00
commit 4ff36ce188
72 changed files with 13594 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

558
analyze/generate_charts.py Normal file
View File

@@ -0,0 +1,558 @@
#!/usr/bin/env python3
"""Generate comprehensive publication-quality charts for UAV-GeoLoc dataset analysis."""
import os
import re
import glob
from collections import Counter, defaultdict
from pathlib import Path
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import matplotlib.ticker as ticker
import numpy as np
from PIL import Image
BASE = '/mnt/data1tb/cvgl_datasets/UAV-GeoLoc'
CHARTS = os.path.join(BASE, 'charts')
os.makedirs(CHARTS, exist_ok=True)
plt.style.use('seaborn-v0_8-whitegrid')
SAVE_KW = dict(dpi=150, bbox_inches='tight')
# Color palette
C_BLUE = '#4C72B0'
C_RED = '#C44E52'
C_GREEN = '#55A868'
C_ORANGE = '#DD8452'
C_PURPLE = '#8172B3'
# ============================================================
# 1. Scenes per country
# ============================================================
def chart_scenes_per_country():
country_dir = os.path.join(BASE, 'Country')
data = {}
for country in sorted(os.listdir(country_dir)):
cpath = os.path.join(country_dir, country)
if not os.path.isdir(cpath):
continue
# Count scenes: each leaf directory that has a DB folder
scenes = glob.glob(os.path.join(cpath, '*', '*', 'DB'))
if not scenes:
scenes = glob.glob(os.path.join(cpath, '*', 'DB'))
if not scenes:
scenes = glob.glob(os.path.join(cpath, '**', 'DB'), recursive=True)
# Filter out nested txt/DB dirs
scenes = [s for s in scenes if '/txt/' not in s]
data[country] = len(scenes)
# Sort descending
items = sorted(data.items(), key=lambda x: x[1], reverse=True)
names, counts = zip(*items)
fig, ax = plt.subplots(figsize=(8, 5))
bars = ax.barh(range(len(names)), counts, color=C_BLUE, edgecolor='white')
ax.set_yticks(range(len(names)))
ax.set_yticklabels(names)
ax.invert_yaxis()
ax.set_xlabel('Number of Scenes')
ax.set_title('Scenes per Country (Country Subset)')
for bar, c in zip(bars, counts):
ax.text(bar.get_width() + 0.3, bar.get_y() + bar.get_height()/2, str(c),
va='center', fontsize=9)
ax.set_xlim(0, max(counts) * 1.15)
fig.savefig(os.path.join(CHARTS, 'chart_scenes_per_country.png'), **SAVE_KW)
plt.close(fig)
print(f'[1] chart_scenes_per_country.png — {len(names)} countries, {sum(counts)} scenes')
# ============================================================
# 2. Scenes per terrain type
# ============================================================
def chart_scenes_per_terrain():
terrain_dir = os.path.join(BASE, 'Terrain')
data = {}
for terrain in sorted(os.listdir(terrain_dir)):
tpath = os.path.join(terrain_dir, terrain)
if not os.path.isdir(tpath):
continue
if terrain.endswith('-ignore'):
continue
scenes = glob.glob(os.path.join(tpath, '*', 'DB'))
# Filter out nested txt/DB dirs
scenes = [s for s in scenes if '/txt/' not in s]
if scenes:
data[terrain] = len(scenes)
items = sorted(data.items(), key=lambda x: x[1], reverse=True)
names, counts = zip(*items)
fig, ax = plt.subplots(figsize=(9, 8))
bars = ax.barh(range(len(names)), counts, color=C_RED, edgecolor='white')
ax.set_yticks(range(len(names)))
ax.set_yticklabels(names, fontsize=8)
ax.invert_yaxis()
ax.set_xlabel('Number of Scenes')
ax.set_title('Scenes per Terrain Type (Terrain Subset)')
for bar, c in zip(bars, counts):
ax.text(bar.get_width() + 0.2, bar.get_y() + bar.get_height()/2, str(c),
va='center', fontsize=8)
ax.set_xlim(0, max(counts) * 1.15)
fig.savefig(os.path.join(CHARTS, 'chart_scenes_per_terrain.png'), **SAVE_KW)
plt.close(fig)
print(f'[2] chart_scenes_per_terrain.png — {len(names)} types, {sum(counts)} scenes')
# ============================================================
# 3. Crop sizes distribution
# ============================================================
def chart_crop_sizes_distribution():
crop_sizes = Counter()
for subset in ['Country', 'Terrain']:
db_dirs = glob.glob(os.path.join(BASE, subset, '**', 'DB', 'img'), recursive=True)
for db_img_dir in db_dirs:
if '/txt/' in db_img_dir:
continue
# Sample first crop image to get size
pngs = [f for f in os.listdir(db_img_dir) if f.startswith('crop_') and f.endswith('.png')]
if pngs:
sample = os.path.join(db_img_dir, pngs[0])
try:
img = Image.open(sample)
w, h = img.size
label = f'{w}x{h}'
crop_sizes[label] += 1
except Exception:
pass
# Sort by the numeric width
items = sorted(crop_sizes.items(), key=lambda x: int(x[0].split('x')[0]))
labels, counts = zip(*items)
fig, ax = plt.subplots(figsize=(10, 5))
bars = ax.bar(range(len(labels)), counts, color=C_GREEN, edgecolor='white')
ax.set_xticks(range(len(labels)))
ax.set_xticklabels(labels, rotation=45, ha='right')
ax.set_xlabel('Crop Size (pixels)')
ax.set_ylabel('Number of Scenes')
ax.set_title('Distribution of Crop Sizes Across All Scenes (Country + Terrain)')
for bar, c in zip(bars, counts):
ax.text(bar.get_x() + bar.get_width()/2, bar.get_height() + 1, str(c),
ha='center', va='bottom', fontsize=8)
fig.savefig(os.path.join(CHARTS, 'chart_crop_sizes_distribution.png'), **SAVE_KW)
plt.close(fig)
print(f'[3] chart_crop_sizes_distribution.png — {len(labels)} sizes, {sum(counts)} total scenes')
# ============================================================
# 4. Train/Val/Test split sizes
# ============================================================
def chart_split_sizes():
index_dir = os.path.join(BASE, 'Index')
def count_lines(fname):
fpath = os.path.join(index_dir, fname)
if not os.path.exists(fpath):
return 0
with open(fpath) as f:
return sum(1 for _ in f)
def count_scenes(fname):
fpath = os.path.join(index_dir, fname)
if not os.path.exists(fpath):
return 0
scenes = set()
with open(fpath) as f:
for line in f:
parts = line.strip().split()
if parts:
# Scene = up to 3rd level directory
p = parts[0]
scene = '/'.join(p.split('/')[:3])
scenes.add(scene)
return len(scenes)
# For Terrain subset (the default Index files are Terrain-based)
splits = ['train', 'val', 'test']
# Count scenes from the split txt files (train.txt, val.txt... or train_query.txt etc.)
# train.txt / val.txt / test.txt list the scenes
scene_counts = []
for sp in splits:
f = os.path.join(index_dir, f'{sp}.txt')
if os.path.exists(f):
with open(f) as fh:
scene_counts.append(sum(1 for line in fh if line.strip()))
else:
scene_counts.append(0)
query_counts = [count_lines(f'{sp}_query.txt') for sp in splits]
db_counts = [count_lines(f'{sp}_db.txt') for sp in splits]
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 5))
# Subplot 1: Scenes
x = np.arange(len(splits))
w = 0.5
bars1 = ax1.bar(x, scene_counts, w, color=C_BLUE, edgecolor='white')
ax1.set_xticks(x)
ax1.set_xticklabels(['Train', 'Val', 'Test'])
ax1.set_ylabel('Count')
ax1.set_title('Scenes per Split (Terrain)')
for bar, c in zip(bars1, scene_counts):
ax1.text(bar.get_x() + bar.get_width()/2, bar.get_height() + 1, str(c),
ha='center', va='bottom', fontsize=10)
# Subplot 2: Images (query vs DB)
w2 = 0.35
bars_q = ax2.bar(x - w2/2, query_counts, w2, label='Query', color=C_ORANGE, edgecolor='white')
bars_d = ax2.bar(x + w2/2, db_counts, w2, label='DB', color=C_PURPLE, edgecolor='white')
ax2.set_xticks(x)
ax2.set_xticklabels(['Train', 'Val', 'Test'])
ax2.set_ylabel('Number of Images')
ax2.set_title('Query and DB Images per Split (Terrain)')
ax2.legend()
ax2.yaxis.set_major_formatter(ticker.FuncFormatter(lambda v, _: f'{int(v):,}'))
for bar, c in zip(bars_q, query_counts):
ax2.text(bar.get_x() + bar.get_width()/2, bar.get_height() + 500, f'{c:,}',
ha='center', va='bottom', fontsize=7, rotation=15)
for bar, c in zip(bars_d, db_counts):
ax2.text(bar.get_x() + bar.get_width()/2, bar.get_height() + 500, f'{c:,}',
ha='center', va='bottom', fontsize=7, rotation=15)
fig.suptitle('Train / Val / Test Split Sizes', fontsize=14, y=1.02)
fig.tight_layout()
fig.savefig(os.path.join(CHARTS, 'chart_split_sizes.png'), **SAVE_KW)
plt.close(fig)
print(f'[4] chart_split_sizes.png — scenes: {scene_counts}, query: {query_counts}, db: {db_counts}')
# ============================================================
# 5. Positives per query (real distribution)
# ============================================================
def chart_positives_per_query():
db_pattern = re.compile(r'\S*DB/img/crop_\S+')
positives_dist = Counter()
fpath = os.path.join(BASE, 'Index', 'train_query.txt')
with open(fpath) as f:
for line in f:
line = line.strip()
if not line:
continue
matches = db_pattern.findall(line)
n = len(matches)
positives_dist[n] += 1
items = sorted(positives_dist.items())
n_pos, counts = zip(*items)
total = sum(counts)
fig, ax = plt.subplots(figsize=(8, 5))
bars = ax.bar([str(n) for n in n_pos], counts, color=C_ORANGE, edgecolor='white')
ax.set_xlabel('Number of Positive Matches per Query')
ax.set_ylabel('Number of Queries')
ax.set_title(f'Distribution of Positive Matches per Query (N={total:,})')
ax.yaxis.set_major_formatter(ticker.FuncFormatter(lambda v, _: f'{int(v):,}'))
for bar, c in zip(bars, counts):
pct = 100.0 * c / total
ax.text(bar.get_x() + bar.get_width()/2, bar.get_height() + total*0.005,
f'{c:,}\n({pct:.1f}%)', ha='center', va='bottom', fontsize=8)
fig.savefig(os.path.join(CHARTS, 'chart_positives_per_query.png'), **SAVE_KW)
plt.close(fig)
print(f'[5] chart_positives_per_query.png — distribution: {dict(items)}')
# ============================================================
# 6. Geographic coverage (world scatter)
# ============================================================
def chart_geographic_coverage():
scene_locs = [] # (lat, lon, subset)
for subset, color, label in [('Country', C_BLUE, 'Country'), ('Terrain', C_RED, 'Terrain')]:
db_pos_files = glob.glob(os.path.join(BASE, subset, '**', 'db_postion.txt'), recursive=True)
for dbf in db_pos_files:
if '/txt/' in dbf:
continue
lats, lons = [], []
try:
with open(dbf) as f:
for line in f:
parts = line.strip().split()
if len(parts) >= 3:
lon = float(parts[1])
lat = float(parts[2])
lons.append(lon)
lats.append(lat)
if lats:
scene_locs.append((np.mean(lats), np.mean(lons), subset))
except Exception:
pass
fig, ax = plt.subplots(figsize=(14, 7))
# Simple world coastline approximation using a rectangle and gridlines
ax.set_xlim(-180, 180)
ax.set_ylim(-90, 90)
ax.set_facecolor('#f0f8ff')
ax.grid(True, alpha=0.3)
# Plot
for subset, color, marker, label in [
('Country', C_BLUE, 'o', 'Country'),
('Terrain', C_RED, '^', 'Terrain'),
]:
pts = [(lat, lon) for lat, lon, s in scene_locs if s == subset]
if pts:
lats, lons = zip(*pts)
ax.scatter(lons, lats, c=color, marker=marker, s=40, alpha=0.7,
edgecolors='white', linewidths=0.5, label=f'{label} ({len(pts)} scenes)')
ax.set_xlabel('Longitude')
ax.set_ylabel('Latitude')
ax.set_title('Geographic Coverage of UAV-GeoLoc Scenes')
ax.legend(loc='lower left', fontsize=10)
# Add simple continent labels for context
continent_labels = {
'N. America': (-100, 45), 'S. America': (-60, -15),
'Europe': (15, 50), 'Africa': (20, 5),
'Asia': (80, 40), 'Oceania': (135, -25),
}
for name, (lon, lat) in continent_labels.items():
ax.text(lon, lat, name, fontsize=8, alpha=0.3, ha='center', va='center',
fontstyle='italic')
fig.savefig(os.path.join(CHARTS, 'chart_geographic_coverage.png'), **SAVE_KW)
plt.close(fig)
print(f'[6] chart_geographic_coverage.png — {len(scene_locs)} scenes plotted')
# ============================================================
# 7. Image counts by subset
# ============================================================
def chart_image_counts_by_subset():
# Count from actual index files and filesystem
index_dir = os.path.join(BASE, 'Index')
def count_lines(fname):
fpath = os.path.join(index_dir, fname)
if not os.path.exists(fpath):
return 0
with open(fpath) as f:
return sum(1 for _ in f)
# Terrain query/db from all splits
terrain_query = count_lines('train_query.txt') + count_lines('val_db.txt') # val_query
terrain_db = count_lines('train_db.txt') + count_lines('val_db.txt')
# Actually, let's use the _all files or compute from filesystem
# Use the provided data
subsets = ['Country', 'Terrain', 'Rot']
# Count scenes
country_scenes = len(glob.glob(os.path.join(BASE, 'Country', '*', '*', 'DB')))
if country_scenes == 0:
country_scenes = len([s for s in glob.glob(os.path.join(BASE, 'Country', '**', 'DB'), recursive=True) if '/txt/' not in s])
terrain_scenes = len([s for s in glob.glob(os.path.join(BASE, 'Terrain', '**', 'DB'), recursive=True) if '/txt/' not in s])
rot_scenes = 1
# Count query and DB images from filesystem
def count_images_in_dirs(pattern, ext='*.jpeg'):
dirs = glob.glob(os.path.join(BASE, pattern), recursive=True)
total = 0
for d in dirs:
total += len(glob.glob(os.path.join(d, ext)))
return total
# Use index files where available
# For all: train_query_all, train_db_all, etc.
country_query_count = count_lines('train_query_country.txt')
country_db_count = count_lines('train_db_country.txt')
# For terrain: from the non-country files
all_query = count_lines('train_query_all.txt')
all_db = count_lines('train_db_all.txt')
# Fallback to provided data if files don't exist or are 0
if country_query_count == 0:
country_query_count = 308352
if country_db_count == 0:
country_db_count = 141045
terrain_query_total = count_lines('train_query.txt') + count_lines('test_query.txt')
terrain_db_total = count_lines('train_db.txt') + count_lines('test_db.txt')
# Also try to get val
val_query_file = os.path.join(index_dir, 'val_all.txt')
if os.path.exists(val_query_file):
# val_all might have mixed
pass
# Use provided approximate numbers as fallback
if terrain_query_total < 100000:
terrain_query_total = 337704
if terrain_db_total < 50000:
terrain_db_total = 132990
rot_query = 6688
rot_db = 648
scenes = [country_scenes, terrain_scenes, rot_scenes]
queries = [country_query_count, terrain_query_total, rot_query]
dbs = [country_db_count, terrain_db_total, rot_db]
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(13, 5))
# Scenes
x = np.arange(3)
bars_s = ax1.bar(x, scenes, 0.5, color=[C_BLUE, C_RED, C_GREEN], edgecolor='white')
ax1.set_xticks(x)
ax1.set_xticklabels(subsets)
ax1.set_ylabel('Number of Scenes')
ax1.set_title('Scenes per Subset')
for bar, c in zip(bars_s, scenes):
ax1.text(bar.get_x() + bar.get_width()/2, bar.get_height() + 1, str(c),
ha='center', va='bottom', fontsize=10)
# Images (grouped)
w = 0.35
bars_q = ax2.bar(x - w/2, queries, w, label='Query Images', color=C_ORANGE, edgecolor='white')
bars_d = ax2.bar(x + w/2, dbs, w, label='DB Images', color=C_PURPLE, edgecolor='white')
ax2.set_xticks(x)
ax2.set_xticklabels(subsets)
ax2.set_ylabel('Number of Images')
ax2.set_title('Query and DB Images per Subset')
ax2.legend()
ax2.yaxis.set_major_formatter(ticker.FuncFormatter(lambda v, _: f'{int(v):,}'))
for bar, c in zip(bars_q, queries):
ax2.text(bar.get_x() + bar.get_width()/2, bar.get_height() + 3000,
f'{c:,}', ha='center', va='bottom', fontsize=7, rotation=15)
for bar, c in zip(bars_d, dbs):
ax2.text(bar.get_x() + bar.get_width()/2, bar.get_height() + 3000,
f'{c:,}', ha='center', va='bottom', fontsize=7, rotation=15)
fig.suptitle('Image Counts by Subset', fontsize=14, y=1.02)
fig.tight_layout()
fig.savefig(os.path.join(CHARTS, 'chart_image_counts_by_subset.png'), **SAVE_KW)
plt.close(fig)
print(f'[7] chart_image_counts_by_subset.png — scenes: {scenes}, queries: {queries}, dbs: {dbs}')
# ============================================================
# 8. Merge.tif dimensions scatter
# ============================================================
def chart_merge_sizes():
points = [] # (w, h, subset)
for subset, color in [('Country', C_BLUE), ('Terrain', C_RED)]:
tifs = glob.glob(os.path.join(BASE, subset, '**', 'merge.tif'), recursive=True)
for tif in tifs:
if '/txt/' in tif:
continue
try:
img = Image.open(tif)
w, h = img.size
points.append((w, h, subset))
except Exception:
pass
fig, ax = plt.subplots(figsize=(8, 6))
for subset, color, marker, label in [
('Country', C_BLUE, 'o', 'Country'),
('Terrain', C_RED, '^', 'Terrain'),
]:
pts = [(w, h) for w, h, s in points if s == subset]
if pts:
ws, hs = zip(*pts)
ax.scatter(ws, hs, c=color, marker=marker, s=40, alpha=0.6,
edgecolors='white', linewidths=0.5, label=f'{label} ({len(pts)})')
ax.set_xlabel('Width (pixels)')
ax.set_ylabel('Height (pixels)')
ax.set_title('Satellite Map (merge.tif) Dimensions')
ax.legend()
ax.xaxis.set_major_formatter(ticker.FuncFormatter(lambda v, _: f'{int(v):,}'))
ax.yaxis.set_major_formatter(ticker.FuncFormatter(lambda v, _: f'{int(v):,}'))
fig.savefig(os.path.join(CHARTS, 'chart_merge_sizes.png'), **SAVE_KW)
plt.close(fig)
print(f'[8] chart_merge_sizes.png — {len(points)} maps plotted')
# ============================================================
# 9. Query variants (azimuth x height)
# ============================================================
def chart_query_variants():
angles = [0, 45, 90, 135, 180, 225, 270, 315]
heights = [100, 125, 150]
angle_labels = ['0\u00b0\n(N)', '45\u00b0\n(NE)', '90\u00b0\n(E)', '135\u00b0\n(SE)',
'180\u00b0\n(S)', '225\u00b0\n(SW)', '270\u00b0\n(W)', '315\u00b0\n(NW)']
fig, ax = plt.subplots(figsize=(8, 8), subplot_kw=dict(projection='polar'))
theta = np.deg2rad(angles)
colors_h = [C_BLUE, C_GREEN, C_RED]
markers_h = ['o', 's', 'D']
for i, h in enumerate(heights):
r = [h] * len(angles)
ax.scatter(theta, r, c=colors_h[i], s=120, marker=markers_h[i],
label=f'Height {h}m', zorder=5, edgecolors='white', linewidths=1)
ax.set_theta_zero_location('N')
ax.set_theta_direction(-1) # Clockwise
ax.set_xticks(theta)
ax.set_xticklabels(angle_labels, fontsize=9)
ax.set_rticks(heights)
ax.set_yticklabels([f'{h}m' for h in heights], fontsize=8)
ax.set_rlim(50, 180)
ax.set_title('Query Variants: 8 Azimuths x 3 Heights\n(24 combinations per scene point)',
pad=20, fontsize=12)
ax.legend(loc='lower right', bbox_to_anchor=(1.2, 0))
fig.savefig(os.path.join(CHARTS, 'chart_query_variants.png'), **SAVE_KW)
plt.close(fig)
print('[9] chart_query_variants.png — 8 azimuths x 3 heights = 24 variants')
# ============================================================
# 10. Rotation accuracy (copy rot_1.png)
# ============================================================
def chart_rot_accuracy():
src = os.path.join(BASE, 'rot_1.png')
if not os.path.exists(src):
print('[10] SKIPPED — rot_1.png not found')
return
img = Image.open(src)
fig, ax = plt.subplots(figsize=(10, 6))
ax.imshow(np.array(img))
ax.axis('off')
ax.set_title('Rotation Robustness Results (from paper)', fontsize=12)
fig.savefig(os.path.join(CHARTS, 'chart_rot_accuracy_by_angle.png'), **SAVE_KW)
plt.close(fig)
print(f'[10] chart_rot_accuracy_by_angle.png — {img.size[0]}x{img.size[1]}')
# ============================================================
# Main
# ============================================================
if __name__ == '__main__':
print('Generating charts...\n')
chart_scenes_per_country()
chart_scenes_per_terrain()
chart_crop_sizes_distribution()
chart_split_sizes()
chart_positives_per_query()
chart_geographic_coverage()
chart_image_counts_by_subset()
chart_merge_sizes()
chart_query_variants()
chart_rot_accuracy()
print('\n--- Generated files ---')
for f in sorted(os.listdir(CHARTS)):
fpath = os.path.join(CHARTS, f)
size_kb = os.path.getsize(fpath) / 1024
print(f' {f:45s} {size_kb:8.1f} KB')