From 7d59305b5f10e1a67729cefde580ccdbd2811b6a Mon Sep 17 00:00:00 2001 From: Yuwen Xiong Date: Tue, 16 Jan 2024 00:22:22 +0800 Subject: [PATCH] birth --- .gitignore | 25 + DCNv4_op/DCNv4/functions/__init__.py | 11 + DCNv4_op/DCNv4/functions/dcnv4_func.py | 129 ++ .../DCNv4/functions/flash_deform_attn_func.py | 114 ++ DCNv4_op/DCNv4/functions/table.py | 1355 +++++++++++++++++ DCNv4_op/DCNv4/modules/__init__.py | 10 + DCNv4_op/DCNv4/modules/dcnv4.py | 152 ++ DCNv4_op/DCNv4/modules/flash_deform_attn.py | 141 ++ DCNv4_op/MANIFEST.in | 2 + DCNv4_op/__init__.py | 2 + DCNv4_op/make.sh | 10 + DCNv4_op/scripts/find_best.py | 61 + DCNv4_op/scripts/search_bwd.sh | 2 + DCNv4_op/scripts/search_dcnv4.py | 131 ++ DCNv4_op/scripts/search_dcnv4_bwd.py | 200 +++ DCNv4_op/scripts/search_dcnv4_bwd_engine.py | 24 + DCNv4_op/scripts/search_dcnv4_engine.py | 24 + DCNv4_op/scripts/search_fwd.sh | 2 + DCNv4_op/scripts/test_dcnv4.py | 144 ++ DCNv4_op/scripts/test_dcnv4_bwd.py | 221 +++ DCNv4_op/scripts/test_flash_deform_attn.py | 174 +++ .../test_flash_deform_attn_backward.py | 194 +++ DCNv4_op/setup.py | 72 + DCNv4_op/src/cuda/common.h | 212 +++ DCNv4_op/src/cuda/dcnv4_col2im_cuda.cuh | 561 +++++++ DCNv4_op/src/cuda/dcnv4_cuda.cu | 175 +++ DCNv4_op/src/cuda/dcnv4_cuda.h | 33 + DCNv4_op/src/cuda/dcnv4_im2col_cuda.cuh | 416 +++++ DCNv4_op/src/cuda/flash_deform_attn_cuda.cu | 162 ++ DCNv4_op/src/cuda/flash_deform_attn_cuda.h | 25 + .../src/cuda/flash_deform_col2im_cuda.cuh | 580 +++++++ .../src/cuda/flash_deform_im2col_cuda.cuh | 451 ++++++ DCNv4_op/src/dcnv4.h | 107 ++ DCNv4_op/src/vision.cpp | 21 + LICENSE | 21 + README.md | 158 ++ classification/README.md | 190 +++ classification/config.py | 302 ++++ .../configs/flash_intern_image_b_1k_224.yaml | 20 + .../flash_intern_image_l_22kto1k_384.yaml | 40 + .../configs/flash_intern_image_s_1k_224.yaml | 20 + .../configs/flash_intern_image_t_1k_224.yaml | 17 + classification/dataset/__init__.py | 7 + classification/dataset/build.py | 286 ++++ classification/dataset/cached_image_folder.py | 538 +++++++ classification/dataset/samplers.py | 114 ++ classification/dataset/zipreader.py | 102 ++ classification/ddp_hooks.py | 183 +++ classification/ema_deepspeed.py | 99 ++ classification/eval.sh | 2 + classification/export.py | 122 ++ classification/extract_feature.py | 128 ++ classification/logger.py | 44 + classification/lr_scheduler.py | 112 ++ classification/main.py | 671 ++++++++ classification/main_accelerate.py | 380 +++++ classification/main_deepspeed.py | 531 +++++++ .../meta_data/22k_class_to_idx.json | 1 + classification/meta_data/map22kto1k.txt | 1000 ++++++++++++ classification/meta_data/meta | 1 + classification/models/__init__.py | 7 + classification/models/build.py | 58 + classification/models/flash_intern_image.py | 795 ++++++++++ classification/models/intern_image.py | 759 +++++++++ .../ops_dcnv3/functions/__init__.py | 7 + .../ops_dcnv3/functions/dcnv3_func.py | 220 +++ classification/ops_dcnv3/make.sh | 8 + classification/ops_dcnv3/modules/__init__.py | 7 + classification/ops_dcnv3/modules/dcnv3.py | 357 +++++ classification/ops_dcnv3/setup.py | 75 + .../ops_dcnv3/src/cpu/dcnv3_cpu.cpp | 37 + classification/ops_dcnv3/src/cpu/dcnv3_cpu.h | 31 + .../ops_dcnv3/src/cuda/dcnv3_cuda.cu | 174 +++ .../ops_dcnv3/src/cuda/dcnv3_cuda.h | 31 + .../ops_dcnv3/src/cuda/dcnv3_im2col_cuda.cuh | 1094 +++++++++++++ classification/ops_dcnv3/src/dcnv3.h | 59 + classification/ops_dcnv3/src/vision.cpp | 17 + classification/ops_dcnv3/test.py | 264 ++++ classification/optimizer.py | 159 ++ classification/train_in1k.sh | 31 + classification/train_in1k_deepspeed.sh | 27 + classification/utils.py | 423 +++++ detection/README.md | 93 ++ .../configs/_base_/datasets/coco_detection.py | 49 + .../configs/_base_/datasets/coco_instance.py | 49 + .../configs/_base_/datasets/crowd_human.py | 54 + detection/configs/_base_/default_runtime.py | 16 + .../models/cascade_mask_rcnn_convnext_fpn.py | 208 +++ .../models/cascade_mask_rcnn_r50_fpn.py | 196 +++ .../cascade_mask_rcnn_r50_fpn_crowdhuman.py | 183 +++ .../_base_/models/cascade_rcnn_r50_fpn.py | 179 +++ .../_base_/models/fast_rcnn_r50_fpn.py | 62 + .../_base_/models/faster_rcnn_r50_caffe_c4.py | 114 ++ .../models/faster_rcnn_r50_caffe_dc5.py | 105 ++ .../_base_/models/faster_rcnn_r50_fpn.py | 108 ++ .../_base_/models/mask_rcnn_convnext_fpn.py | 128 ++ .../_base_/models/mask_rcnn_r50_caffe_c4.py | 125 ++ .../_base_/models/mask_rcnn_r50_fpn.py | 120 ++ .../_base_/models/retinanet_r50_fpn.py | 60 + .../configs/_base_/models/rpn_r50_caffe_c4.py | 58 + .../configs/_base_/models/rpn_r50_fpn.py | 58 + detection/configs/_base_/models/ssd300.py | 56 + .../configs/_base_/schedules/schedule_1x.py | 11 + .../configs/_base_/schedules/schedule_3x.py | 11 + detection/configs/coco/README.md | 51 + ...ascade_flash_intern_image_l_fpn_1x_coco.py | 148 ++ ...ascade_flash_intern_image_l_fpn_3x_coco.py | 196 +++ ...dino_4scale_flash_internimage_b_1x_coco.py | 180 +++ ...dino_4scale_flash_internimage_l_1x_coco.py | 184 +++ ...dino_4scale_flash_internimage_s_1x_coco.py | 180 +++ ...dino_4scale_flash_internimage_t_1x_coco.py | 179 +++ ...k_rcnn_flash_intern_image_b_fpn_1x_coco.py | 83 + ...k_rcnn_flash_intern_image_b_fpn_3x_coco.py | 125 ++ ...k_rcnn_flash_intern_image_s_fpn_1x_coco.py | 84 + ...k_rcnn_flash_intern_image_s_fpn_3x_coco.py | 126 ++ ...k_rcnn_flash_intern_image_t_fpn_1x_coco.py | 81 + ...k_rcnn_flash_intern_image_t_fpn_3x_coco.py | 124 ++ detection/dist_test.sh | 9 + detection/dist_train.sh | 8 + detection/get_flops.py | 120 ++ detection/image_demo.py | 61 + detection/mmcv_custom/__init__.py | 9 + ...ustom_layer_decay_optimizer_constructor.py | 161 ++ detection/mmdet_custom/__init__.py | 8 + detection/mmdet_custom/datasets/__init__.py | 7 + .../mmdet_custom/datasets/crowd_human.py | 529 +++++++ detection/mmdet_custom/models/__init__.py | 11 + .../mmdet_custom/models/backbones/__init__.py | 8 + .../models/backbones/flash_intern_image.py | 763 ++++++++++ .../models/dense_heads/__init__.py | 13 + .../models/dense_heads/bbox_head.py | 222 +++ .../dense_heads/deformable_detr_head.py | 332 ++++ .../models/dense_heads/detr_head.py | 954 ++++++++++++ .../models/dense_heads/dino_head.py | 365 +++++ .../models/dense_heads/mask_rcnn.py | 27 + .../mmdet_custom/models/dense_heads/msda.py | 369 +++++ .../models/dense_heads/two_stage.py | 225 +++ .../mmdet_custom/models/detectors/__init__.py | 9 + .../mmdet_custom/models/detectors/dino.py | 10 + detection/mmdet_custom/models/necks/fpn.py | 207 +++ .../mmdet_custom/models/utils/__init__.py | 6 + .../models/utils/convModule_norm.py | 34 + .../models/utils/query_denoising.py | 234 +++ .../mmdet_custom/models/utils/transformer.py | 278 ++++ detection/ops_dcnv3/functions/__init__.py | 7 + detection/ops_dcnv3/functions/dcnv3_func.py | 188 +++ detection/ops_dcnv3/make.sh | 8 + detection/ops_dcnv3/modules/__init__.py | 7 + detection/ops_dcnv3/modules/dcnv3.py | 346 +++++ detection/ops_dcnv3/setup.py | 75 + detection/ops_dcnv3/src/cpu/dcnv3_cpu.cpp | 37 + detection/ops_dcnv3/src/cpu/dcnv3_cpu.h | 31 + detection/ops_dcnv3/src/cuda/dcnv3_cuda.cu | 174 +++ detection/ops_dcnv3/src/cuda/dcnv3_cuda.h | 31 + .../ops_dcnv3/src/cuda/dcnv3_im2col_cuda.cuh | 1045 +++++++++++++ detection/ops_dcnv3/src/dcnv3.h | 59 + detection/ops_dcnv3/src/vision.cpp | 17 + detection/ops_dcnv3/test.py | 263 ++++ detection/slurm_test.sh | 25 + detection/slurm_train.sh | 25 + detection/test.py | 265 ++++ detection/tools/create_crowd_anno.py | 95 ++ detection/tools/evaluate/__init__.py | 2 + detection/train.py | 249 +++ segmentation/README.md | 96 ++ .../configs/_base_/datasets/ade20k.py | 54 + .../configs/_base_/datasets/ade20k_640x640.py | 54 + .../configs/_base_/datasets/chase_db1.py | 59 + .../configs/_base_/datasets/cityscapes.py | 54 + .../_base_/datasets/cityscapes_1024x1024.py | 35 + .../_base_/datasets/cityscapes_extra.py | 54 + .../configs/_base_/datasets/coco-stuff10k.py | 57 + .../configs/_base_/datasets/coco-stuff164k.py | 54 + segmentation/configs/_base_/datasets/drive.py | 59 + segmentation/configs/_base_/datasets/hrf.py | 59 + .../configs/_base_/datasets/loveda.py | 54 + .../configs/_base_/datasets/mapillary.py | 55 + .../_base_/datasets/mapillary_1024x1024.py | 55 + .../configs/_base_/datasets/nyu_depth_v2.py | 59 + .../configs/_base_/datasets/pascal_context.py | 60 + .../_base_/datasets/pascal_context_59.py | 60 + .../configs/_base_/datasets/pascal_voc12.py | 57 + .../_base_/datasets/pascal_voc12_aug.py | 9 + .../configs/_base_/datasets/potsdam.py | 0 segmentation/configs/_base_/datasets/stare.py | 59 + .../configs/_base_/default_runtime.py | 14 + .../configs/_base_/models/mask2former_beit.py | 138 ++ .../configs/_base_/models/segformer_mit-b0.py | 34 + .../configs/_base_/models/upernet_convnext.py | 46 + .../configs/_base_/models/upernet_r50.py | 44 + .../configs/_base_/models/upernet_swin.py | 54 + .../configs/_base_/schedules/schedule_160k.py | 9 + .../configs/_base_/schedules/schedule_20k.py | 9 + .../configs/_base_/schedules/schedule_320k.py | 9 + .../configs/_base_/schedules/schedule_40k.py | 9 + .../configs/_base_/schedules/schedule_80k.py | 9 + segmentation/configs/ade20k/README.md | 33 + ..._flash_internimage_b_640_160k_ade20k_ss.py | 159 ++ ..._flash_internimage_l_640_160k_ade20k_ss.py | 160 ++ ..._flash_internimage_s_640_160k_ade20k_ss.py | 159 ++ ...h_internimage_s_640_160k_ade20k_ss_nsmx.py | 160 ++ ..._flash_internimage_t_512_160k_ade20k_ss.py | 157 ++ ...net_flash_internimage_b_512_160k_ade20k.py | 68 + ...net_flash_internimage_l_640_160k_ade20k.py | 84 + ...net_flash_internimage_s_512_160k_ade20k.py | 68 + ...net_flash_internimage_t_512_160k_ade20k.py | 68 + .../configs/_base_/backends/tensorrt.py | 2 + .../deploy/configs/_base_/onnx_config.py | 10 + .../configs/mmseg/segmentation_static.py | 2 + .../segmentation_tensorrt_static-512x512.py | 13 + segmentation/deploy/demo.png | Bin 0 -> 307861 bytes segmentation/dist_test.sh | 9 + segmentation/dist_train.sh | 9 + segmentation/get_flops.py | 119 ++ segmentation/image_demo.py | 70 + segmentation/mmcv_custom/__init__.py | 11 + ...ustom_layer_decay_optimizer_constructor.py | 157 ++ segmentation/mmcv_custom/layer_decay.py | 123 ++ segmentation/mmcv_custom/layer_decay_vit.py | 105 ++ segmentation/mmseg_custom/__init__.py | 9 + segmentation/mmseg_custom/core/__init__.py | 9 + .../mmseg_custom/core/anchor/__init__.py | 2 + .../mmseg_custom/core/anchor/builder.py | 19 + .../core/anchor/point_generator.py | 260 ++++ .../mmseg_custom/core/box/__init__.py | 3 + segmentation/mmseg_custom/core/box/builder.py | 15 + .../core/box/samplers/__init__.py | 2 + .../core/box/samplers/base_sampler.py | 105 ++ .../core/box/samplers/mask_pseudo_sampler.py | 43 + .../core/box/samplers/mask_sampling_result.py | 59 + .../core/box/samplers/sampling_result.py | 150 ++ .../mmseg_custom/core/evaluation/__init__.py | 2 + .../core/evaluation/panoptic_utils.py | 6 + .../mmseg_custom/core/mask/__init__.py | 2 + segmentation/mmseg_custom/core/mask/utils.py | 89 ++ .../mmseg_custom/core/utils/__init__.py | 9 + .../mmseg_custom/core/utils/dist_utils.py | 148 ++ segmentation/mmseg_custom/core/utils/misc.py | 40 + .../mmseg_custom/datasets/__init__.py | 10 + .../mmseg_custom/datasets/dataset_wrappers.py | 155 ++ .../mmseg_custom/datasets/mapillary.py | 48 + .../mmseg_custom/datasets/nyu_depth_v2.py | 43 + .../datasets/pipelines/__init__.py | 8 + .../datasets/pipelines/formatting.py | 82 + .../datasets/pipelines/transform.py | 350 +++++ segmentation/mmseg_custom/models/__init__.py | 12 + .../mmseg_custom/models/backbones/__init__.py | 9 + .../models/backbones/flash_intern_image.py | 763 ++++++++++ segmentation/mmseg_custom/models/builder.py | 23 + .../models/decode_heads/__init__.py | 8 + .../models/decode_heads/mask2former_head.py | 579 +++++++ .../models/decode_heads/maskformer_head.py | 519 +++++++ .../mmseg_custom/models/decode_heads/msda.py | 355 +++++ .../mmseg_custom/models/losses/__init__.py | 13 + .../models/losses/cross_entropy_loss.py | 291 ++++ .../mmseg_custom/models/losses/dice_loss.py | 179 +++ .../mmseg_custom/models/losses/focal_loss.py | 180 +++ .../mmseg_custom/models/losses/match_costs.py | 233 +++ .../mmseg_custom/models/losses/match_loss.py | 179 +++ .../mmseg_custom/models/plugins/__init__.py | 8 + .../plugins/msdeformattn_pixel_decoder.py | 268 ++++ .../models/plugins/pixel_decoder.py | 237 +++ .../models/segmentors/__init__.py | 5 + .../segmentors/encoder_decoder_mask2former.py | 285 ++++ .../encoder_decoder_mask2former_aug.py | 289 ++++ .../mmseg_custom/models/utils/__init__.py | 13 + .../mmseg_custom/models/utils/assigner.py | 165 ++ .../mmseg_custom/models/utils/point_sample.py | 87 ++ .../models/utils/positional_encoding.py | 161 ++ .../mmseg_custom/models/utils/transformer.py | 1083 +++++++++++++ segmentation/ops_dcnv3/functions/__init__.py | 7 + .../ops_dcnv3/functions/dcnv3_func.py | 188 +++ segmentation/ops_dcnv3/make.sh | 8 + segmentation/ops_dcnv3/modules/__init__.py | 7 + segmentation/ops_dcnv3/modules/dcnv3.py | 345 +++++ segmentation/ops_dcnv3/setup.py | 75 + segmentation/ops_dcnv3/src/cpu/dcnv3_cpu.cpp | 37 + segmentation/ops_dcnv3/src/cpu/dcnv3_cpu.h | 31 + segmentation/ops_dcnv3/src/cuda/dcnv3_cuda.cu | 174 +++ segmentation/ops_dcnv3/src/cuda/dcnv3_cuda.h | 31 + .../ops_dcnv3/src/cuda/dcnv3_im2col_cuda.cuh | 1045 +++++++++++++ segmentation/ops_dcnv3/src/dcnv3.h | 59 + segmentation/ops_dcnv3/src/vision.cpp | 17 + segmentation/ops_dcnv3/test.py | 263 ++++ segmentation/slurm_test.sh | 25 + segmentation/slurm_train.sh | 24 + segmentation/test.py | 294 ++++ segmentation/train.py | 251 +++ 288 files changed, 41101 insertions(+) create mode 100644 .gitignore create mode 100644 DCNv4_op/DCNv4/functions/__init__.py create mode 100644 DCNv4_op/DCNv4/functions/dcnv4_func.py create mode 100644 DCNv4_op/DCNv4/functions/flash_deform_attn_func.py create mode 100644 DCNv4_op/DCNv4/functions/table.py create mode 100644 DCNv4_op/DCNv4/modules/__init__.py create mode 100644 DCNv4_op/DCNv4/modules/dcnv4.py create mode 100644 DCNv4_op/DCNv4/modules/flash_deform_attn.py create mode 100644 DCNv4_op/MANIFEST.in create mode 100644 DCNv4_op/__init__.py create mode 100755 DCNv4_op/make.sh create mode 100644 DCNv4_op/scripts/find_best.py create mode 100644 DCNv4_op/scripts/search_bwd.sh create mode 100644 DCNv4_op/scripts/search_dcnv4.py create mode 100644 DCNv4_op/scripts/search_dcnv4_bwd.py create mode 100644 DCNv4_op/scripts/search_dcnv4_bwd_engine.py create mode 100644 DCNv4_op/scripts/search_dcnv4_engine.py create mode 100644 DCNv4_op/scripts/search_fwd.sh create mode 100644 DCNv4_op/scripts/test_dcnv4.py create mode 100644 DCNv4_op/scripts/test_dcnv4_bwd.py create mode 100644 DCNv4_op/scripts/test_flash_deform_attn.py create mode 100644 DCNv4_op/scripts/test_flash_deform_attn_backward.py create mode 100644 DCNv4_op/setup.py create mode 100644 DCNv4_op/src/cuda/common.h create mode 100644 DCNv4_op/src/cuda/dcnv4_col2im_cuda.cuh create mode 100644 DCNv4_op/src/cuda/dcnv4_cuda.cu create mode 100644 DCNv4_op/src/cuda/dcnv4_cuda.h create mode 100644 DCNv4_op/src/cuda/dcnv4_im2col_cuda.cuh create mode 100644 DCNv4_op/src/cuda/flash_deform_attn_cuda.cu create mode 100644 DCNv4_op/src/cuda/flash_deform_attn_cuda.h create mode 100644 DCNv4_op/src/cuda/flash_deform_col2im_cuda.cuh create mode 100644 DCNv4_op/src/cuda/flash_deform_im2col_cuda.cuh create mode 100644 DCNv4_op/src/dcnv4.h create mode 100644 DCNv4_op/src/vision.cpp create mode 100644 LICENSE create mode 100644 README.md create mode 100644 classification/README.md create mode 100644 classification/config.py create mode 100755 classification/configs/flash_intern_image_b_1k_224.yaml create mode 100755 classification/configs/flash_intern_image_l_22kto1k_384.yaml create mode 100755 classification/configs/flash_intern_image_s_1k_224.yaml create mode 100755 classification/configs/flash_intern_image_t_1k_224.yaml create mode 100644 classification/dataset/__init__.py create mode 100644 classification/dataset/build.py create mode 100644 classification/dataset/cached_image_folder.py create mode 100644 classification/dataset/samplers.py create mode 100644 classification/dataset/zipreader.py create mode 100644 classification/ddp_hooks.py create mode 100644 classification/ema_deepspeed.py create mode 100644 classification/eval.sh create mode 100644 classification/export.py create mode 100644 classification/extract_feature.py create mode 100644 classification/logger.py create mode 100644 classification/lr_scheduler.py create mode 100644 classification/main.py create mode 100644 classification/main_accelerate.py create mode 100644 classification/main_deepspeed.py create mode 100644 classification/meta_data/22k_class_to_idx.json create mode 100644 classification/meta_data/map22kto1k.txt create mode 120000 classification/meta_data/meta create mode 100644 classification/models/__init__.py create mode 100644 classification/models/build.py create mode 100644 classification/models/flash_intern_image.py create mode 100644 classification/models/intern_image.py create mode 100644 classification/ops_dcnv3/functions/__init__.py create mode 100644 classification/ops_dcnv3/functions/dcnv3_func.py create mode 100755 classification/ops_dcnv3/make.sh create mode 100644 classification/ops_dcnv3/modules/__init__.py create mode 100644 classification/ops_dcnv3/modules/dcnv3.py create mode 100644 classification/ops_dcnv3/setup.py create mode 100644 classification/ops_dcnv3/src/cpu/dcnv3_cpu.cpp create mode 100644 classification/ops_dcnv3/src/cpu/dcnv3_cpu.h create mode 100644 classification/ops_dcnv3/src/cuda/dcnv3_cuda.cu create mode 100644 classification/ops_dcnv3/src/cuda/dcnv3_cuda.h create mode 100644 classification/ops_dcnv3/src/cuda/dcnv3_im2col_cuda.cuh create mode 100644 classification/ops_dcnv3/src/dcnv3.h create mode 100644 classification/ops_dcnv3/src/vision.cpp create mode 100644 classification/ops_dcnv3/test.py create mode 100644 classification/optimizer.py create mode 100755 classification/train_in1k.sh create mode 100644 classification/train_in1k_deepspeed.sh create mode 100644 classification/utils.py create mode 100644 detection/README.md create mode 100644 detection/configs/_base_/datasets/coco_detection.py create mode 100644 detection/configs/_base_/datasets/coco_instance.py create mode 100644 detection/configs/_base_/datasets/crowd_human.py create mode 100644 detection/configs/_base_/default_runtime.py create mode 100644 detection/configs/_base_/models/cascade_mask_rcnn_convnext_fpn.py create mode 100644 detection/configs/_base_/models/cascade_mask_rcnn_r50_fpn.py create mode 100644 detection/configs/_base_/models/cascade_mask_rcnn_r50_fpn_crowdhuman.py create mode 100644 detection/configs/_base_/models/cascade_rcnn_r50_fpn.py create mode 100644 detection/configs/_base_/models/fast_rcnn_r50_fpn.py create mode 100644 detection/configs/_base_/models/faster_rcnn_r50_caffe_c4.py create mode 100644 detection/configs/_base_/models/faster_rcnn_r50_caffe_dc5.py create mode 100644 detection/configs/_base_/models/faster_rcnn_r50_fpn.py create mode 100644 detection/configs/_base_/models/mask_rcnn_convnext_fpn.py create mode 100644 detection/configs/_base_/models/mask_rcnn_r50_caffe_c4.py create mode 100644 detection/configs/_base_/models/mask_rcnn_r50_fpn.py create mode 100644 detection/configs/_base_/models/retinanet_r50_fpn.py create mode 100644 detection/configs/_base_/models/rpn_r50_caffe_c4.py create mode 100644 detection/configs/_base_/models/rpn_r50_fpn.py create mode 100644 detection/configs/_base_/models/ssd300.py create mode 100644 detection/configs/_base_/schedules/schedule_1x.py create mode 100644 detection/configs/_base_/schedules/schedule_3x.py create mode 100644 detection/configs/coco/README.md create mode 100644 detection/configs/coco/cascade_flash_intern_image_l_fpn_1x_coco.py create mode 100644 detection/configs/coco/cascade_flash_intern_image_l_fpn_3x_coco.py create mode 100644 detection/configs/coco/dino_4scale_flash_internimage_b_1x_coco.py create mode 100644 detection/configs/coco/dino_4scale_flash_internimage_l_1x_coco.py create mode 100644 detection/configs/coco/dino_4scale_flash_internimage_s_1x_coco.py create mode 100644 detection/configs/coco/dino_4scale_flash_internimage_t_1x_coco.py create mode 100644 detection/configs/coco/mask_rcnn_flash_intern_image_b_fpn_1x_coco.py create mode 100644 detection/configs/coco/mask_rcnn_flash_intern_image_b_fpn_3x_coco.py create mode 100644 detection/configs/coco/mask_rcnn_flash_intern_image_s_fpn_1x_coco.py create mode 100644 detection/configs/coco/mask_rcnn_flash_intern_image_s_fpn_3x_coco.py create mode 100644 detection/configs/coco/mask_rcnn_flash_intern_image_t_fpn_1x_coco.py create mode 100644 detection/configs/coco/mask_rcnn_flash_intern_image_t_fpn_3x_coco.py create mode 100755 detection/dist_test.sh create mode 100755 detection/dist_train.sh create mode 100644 detection/get_flops.py create mode 100644 detection/image_demo.py create mode 100644 detection/mmcv_custom/__init__.py create mode 100644 detection/mmcv_custom/custom_layer_decay_optimizer_constructor.py create mode 100644 detection/mmdet_custom/__init__.py create mode 100644 detection/mmdet_custom/datasets/__init__.py create mode 100644 detection/mmdet_custom/datasets/crowd_human.py create mode 100644 detection/mmdet_custom/models/__init__.py create mode 100644 detection/mmdet_custom/models/backbones/__init__.py create mode 100644 detection/mmdet_custom/models/backbones/flash_intern_image.py create mode 100644 detection/mmdet_custom/models/dense_heads/__init__.py create mode 100644 detection/mmdet_custom/models/dense_heads/bbox_head.py create mode 100644 detection/mmdet_custom/models/dense_heads/deformable_detr_head.py create mode 100644 detection/mmdet_custom/models/dense_heads/detr_head.py create mode 100644 detection/mmdet_custom/models/dense_heads/dino_head.py create mode 100644 detection/mmdet_custom/models/dense_heads/mask_rcnn.py create mode 100644 detection/mmdet_custom/models/dense_heads/msda.py create mode 100644 detection/mmdet_custom/models/dense_heads/two_stage.py create mode 100644 detection/mmdet_custom/models/detectors/__init__.py create mode 100644 detection/mmdet_custom/models/detectors/dino.py create mode 100644 detection/mmdet_custom/models/necks/fpn.py create mode 100644 detection/mmdet_custom/models/utils/__init__.py create mode 100644 detection/mmdet_custom/models/utils/convModule_norm.py create mode 100644 detection/mmdet_custom/models/utils/query_denoising.py create mode 100644 detection/mmdet_custom/models/utils/transformer.py create mode 100644 detection/ops_dcnv3/functions/__init__.py create mode 100644 detection/ops_dcnv3/functions/dcnv3_func.py create mode 100755 detection/ops_dcnv3/make.sh create mode 100644 detection/ops_dcnv3/modules/__init__.py create mode 100644 detection/ops_dcnv3/modules/dcnv3.py create mode 100644 detection/ops_dcnv3/setup.py create mode 100644 detection/ops_dcnv3/src/cpu/dcnv3_cpu.cpp create mode 100644 detection/ops_dcnv3/src/cpu/dcnv3_cpu.h create mode 100644 detection/ops_dcnv3/src/cuda/dcnv3_cuda.cu create mode 100644 detection/ops_dcnv3/src/cuda/dcnv3_cuda.h create mode 100644 detection/ops_dcnv3/src/cuda/dcnv3_im2col_cuda.cuh create mode 100644 detection/ops_dcnv3/src/dcnv3.h create mode 100644 detection/ops_dcnv3/src/vision.cpp create mode 100644 detection/ops_dcnv3/test.py create mode 100644 detection/slurm_test.sh create mode 100755 detection/slurm_train.sh create mode 100644 detection/test.py create mode 100644 detection/tools/create_crowd_anno.py create mode 100644 detection/tools/evaluate/__init__.py create mode 100644 detection/train.py create mode 100644 segmentation/README.md create mode 100644 segmentation/configs/_base_/datasets/ade20k.py create mode 100644 segmentation/configs/_base_/datasets/ade20k_640x640.py create mode 100644 segmentation/configs/_base_/datasets/chase_db1.py create mode 100644 segmentation/configs/_base_/datasets/cityscapes.py create mode 100644 segmentation/configs/_base_/datasets/cityscapes_1024x1024.py create mode 100644 segmentation/configs/_base_/datasets/cityscapes_extra.py create mode 100644 segmentation/configs/_base_/datasets/coco-stuff10k.py create mode 100644 segmentation/configs/_base_/datasets/coco-stuff164k.py create mode 100644 segmentation/configs/_base_/datasets/drive.py create mode 100644 segmentation/configs/_base_/datasets/hrf.py create mode 100644 segmentation/configs/_base_/datasets/loveda.py create mode 100644 segmentation/configs/_base_/datasets/mapillary.py create mode 100644 segmentation/configs/_base_/datasets/mapillary_1024x1024.py create mode 100644 segmentation/configs/_base_/datasets/nyu_depth_v2.py create mode 100644 segmentation/configs/_base_/datasets/pascal_context.py create mode 100644 segmentation/configs/_base_/datasets/pascal_context_59.py create mode 100644 segmentation/configs/_base_/datasets/pascal_voc12.py create mode 100644 segmentation/configs/_base_/datasets/pascal_voc12_aug.py create mode 100644 segmentation/configs/_base_/datasets/potsdam.py create mode 100644 segmentation/configs/_base_/datasets/stare.py create mode 100644 segmentation/configs/_base_/default_runtime.py create mode 100644 segmentation/configs/_base_/models/mask2former_beit.py create mode 100644 segmentation/configs/_base_/models/segformer_mit-b0.py create mode 100644 segmentation/configs/_base_/models/upernet_convnext.py create mode 100644 segmentation/configs/_base_/models/upernet_r50.py create mode 100644 segmentation/configs/_base_/models/upernet_swin.py create mode 100644 segmentation/configs/_base_/schedules/schedule_160k.py create mode 100644 segmentation/configs/_base_/schedules/schedule_20k.py create mode 100644 segmentation/configs/_base_/schedules/schedule_320k.py create mode 100644 segmentation/configs/_base_/schedules/schedule_40k.py create mode 100644 segmentation/configs/_base_/schedules/schedule_80k.py create mode 100644 segmentation/configs/ade20k/README.md create mode 100644 segmentation/configs/ade20k/mask2former_flash_internimage_b_640_160k_ade20k_ss.py create mode 100644 segmentation/configs/ade20k/mask2former_flash_internimage_l_640_160k_ade20k_ss.py create mode 100644 segmentation/configs/ade20k/mask2former_flash_internimage_s_640_160k_ade20k_ss.py create mode 100644 segmentation/configs/ade20k/mask2former_flash_internimage_s_640_160k_ade20k_ss_nsmx.py create mode 100644 segmentation/configs/ade20k/mask2former_flash_internimage_t_512_160k_ade20k_ss.py create mode 100644 segmentation/configs/ade20k/upernet_flash_internimage_b_512_160k_ade20k.py create mode 100644 segmentation/configs/ade20k/upernet_flash_internimage_l_640_160k_ade20k.py create mode 100644 segmentation/configs/ade20k/upernet_flash_internimage_s_512_160k_ade20k.py create mode 100644 segmentation/configs/ade20k/upernet_flash_internimage_t_512_160k_ade20k.py create mode 100644 segmentation/deploy/configs/_base_/backends/tensorrt.py create mode 100644 segmentation/deploy/configs/_base_/onnx_config.py create mode 100644 segmentation/deploy/configs/mmseg/segmentation_static.py create mode 100644 segmentation/deploy/configs/mmseg/segmentation_tensorrt_static-512x512.py create mode 100644 segmentation/deploy/demo.png create mode 100755 segmentation/dist_test.sh create mode 100755 segmentation/dist_train.sh create mode 100644 segmentation/get_flops.py create mode 100644 segmentation/image_demo.py create mode 100644 segmentation/mmcv_custom/__init__.py create mode 100644 segmentation/mmcv_custom/custom_layer_decay_optimizer_constructor.py create mode 100644 segmentation/mmcv_custom/layer_decay.py create mode 100644 segmentation/mmcv_custom/layer_decay_vit.py create mode 100644 segmentation/mmseg_custom/__init__.py create mode 100644 segmentation/mmseg_custom/core/__init__.py create mode 100644 segmentation/mmseg_custom/core/anchor/__init__.py create mode 100644 segmentation/mmseg_custom/core/anchor/builder.py create mode 100644 segmentation/mmseg_custom/core/anchor/point_generator.py create mode 100644 segmentation/mmseg_custom/core/box/__init__.py create mode 100644 segmentation/mmseg_custom/core/box/builder.py create mode 100644 segmentation/mmseg_custom/core/box/samplers/__init__.py create mode 100644 segmentation/mmseg_custom/core/box/samplers/base_sampler.py create mode 100644 segmentation/mmseg_custom/core/box/samplers/mask_pseudo_sampler.py create mode 100644 segmentation/mmseg_custom/core/box/samplers/mask_sampling_result.py create mode 100644 segmentation/mmseg_custom/core/box/samplers/sampling_result.py create mode 100644 segmentation/mmseg_custom/core/evaluation/__init__.py create mode 100644 segmentation/mmseg_custom/core/evaluation/panoptic_utils.py create mode 100644 segmentation/mmseg_custom/core/mask/__init__.py create mode 100644 segmentation/mmseg_custom/core/mask/utils.py create mode 100644 segmentation/mmseg_custom/core/utils/__init__.py create mode 100644 segmentation/mmseg_custom/core/utils/dist_utils.py create mode 100644 segmentation/mmseg_custom/core/utils/misc.py create mode 100644 segmentation/mmseg_custom/datasets/__init__.py create mode 100644 segmentation/mmseg_custom/datasets/dataset_wrappers.py create mode 100644 segmentation/mmseg_custom/datasets/mapillary.py create mode 100644 segmentation/mmseg_custom/datasets/nyu_depth_v2.py create mode 100644 segmentation/mmseg_custom/datasets/pipelines/__init__.py create mode 100644 segmentation/mmseg_custom/datasets/pipelines/formatting.py create mode 100644 segmentation/mmseg_custom/datasets/pipelines/transform.py create mode 100644 segmentation/mmseg_custom/models/__init__.py create mode 100644 segmentation/mmseg_custom/models/backbones/__init__.py create mode 100644 segmentation/mmseg_custom/models/backbones/flash_intern_image.py create mode 100644 segmentation/mmseg_custom/models/builder.py create mode 100644 segmentation/mmseg_custom/models/decode_heads/__init__.py create mode 100644 segmentation/mmseg_custom/models/decode_heads/mask2former_head.py create mode 100644 segmentation/mmseg_custom/models/decode_heads/maskformer_head.py create mode 100644 segmentation/mmseg_custom/models/decode_heads/msda.py create mode 100644 segmentation/mmseg_custom/models/losses/__init__.py create mode 100644 segmentation/mmseg_custom/models/losses/cross_entropy_loss.py create mode 100644 segmentation/mmseg_custom/models/losses/dice_loss.py create mode 100644 segmentation/mmseg_custom/models/losses/focal_loss.py create mode 100644 segmentation/mmseg_custom/models/losses/match_costs.py create mode 100644 segmentation/mmseg_custom/models/losses/match_loss.py create mode 100644 segmentation/mmseg_custom/models/plugins/__init__.py create mode 100644 segmentation/mmseg_custom/models/plugins/msdeformattn_pixel_decoder.py create mode 100644 segmentation/mmseg_custom/models/plugins/pixel_decoder.py create mode 100644 segmentation/mmseg_custom/models/segmentors/__init__.py create mode 100644 segmentation/mmseg_custom/models/segmentors/encoder_decoder_mask2former.py create mode 100644 segmentation/mmseg_custom/models/segmentors/encoder_decoder_mask2former_aug.py create mode 100644 segmentation/mmseg_custom/models/utils/__init__.py create mode 100644 segmentation/mmseg_custom/models/utils/assigner.py create mode 100644 segmentation/mmseg_custom/models/utils/point_sample.py create mode 100644 segmentation/mmseg_custom/models/utils/positional_encoding.py create mode 100644 segmentation/mmseg_custom/models/utils/transformer.py create mode 100644 segmentation/ops_dcnv3/functions/__init__.py create mode 100644 segmentation/ops_dcnv3/functions/dcnv3_func.py create mode 100755 segmentation/ops_dcnv3/make.sh create mode 100644 segmentation/ops_dcnv3/modules/__init__.py create mode 100644 segmentation/ops_dcnv3/modules/dcnv3.py create mode 100644 segmentation/ops_dcnv3/setup.py create mode 100644 segmentation/ops_dcnv3/src/cpu/dcnv3_cpu.cpp create mode 100644 segmentation/ops_dcnv3/src/cpu/dcnv3_cpu.h create mode 100644 segmentation/ops_dcnv3/src/cuda/dcnv3_cuda.cu create mode 100644 segmentation/ops_dcnv3/src/cuda/dcnv3_cuda.h create mode 100644 segmentation/ops_dcnv3/src/cuda/dcnv3_im2col_cuda.cuh create mode 100644 segmentation/ops_dcnv3/src/dcnv3.h create mode 100644 segmentation/ops_dcnv3/src/vision.cpp create mode 100644 segmentation/ops_dcnv3/test.py create mode 100644 segmentation/slurm_test.sh create mode 100755 segmentation/slurm_train.sh create mode 100644 segmentation/test.py create mode 100644 segmentation/train.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9c57ef3 --- /dev/null +++ b/.gitignore @@ -0,0 +1,25 @@ + +.idea/ +.DS_Store +__pycache__/ +classification/convertor/ +segmentation/convertor/ +checkpoint_dir/ +demo/ +detection/work_dirs +*.pth +ops_dcnv3/build +ops_dcnv3/dist +ops_dcnv3/DCNv3.egg-info +build/ +dist/ +ckpts/ +ckpts +data +data/ +detection/data +detection/ckpts +segmentation/data +segmentation/ckpts +work_dirs/ +output \ No newline at end of file diff --git a/DCNv4_op/DCNv4/functions/__init__.py b/DCNv4_op/DCNv4/functions/__init__.py new file mode 100644 index 0000000..0338e58 --- /dev/null +++ b/DCNv4_op/DCNv4/functions/__init__.py @@ -0,0 +1,11 @@ +# ------------------------------------------------------------------------------------------------ +# Deformable DETR +# Copyright (c) 2020 SenseTime. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------------------------------ +# Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +# ------------------------------------------------------------------------------------------------ + +# from .ms_flash_deform_attn_func import FlashMSDeformAttnFunction +from .flash_deform_attn_func import FlashDeformAttnFunction +from .dcnv4_func import DCNv4Function \ No newline at end of file diff --git a/DCNv4_op/DCNv4/functions/dcnv4_func.py b/DCNv4_op/DCNv4/functions/dcnv4_func.py new file mode 100644 index 0000000..1f95c3e --- /dev/null +++ b/DCNv4_op/DCNv4/functions/dcnv4_func.py @@ -0,0 +1,129 @@ +# -------------------------------------------------------- +# InternImage +# Copyright (c) 2022 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + +from __future__ import absolute_import +from __future__ import print_function +from __future__ import division + +import torch +import math +import torch.nn.functional as F +from torch.autograd import Function +from torch.autograd.function import once_differentiable +from torch.cuda.amp import custom_bwd, custom_fwd +from .table import TABLE, BWDTABLE + +from DCNv4 import ext + +def factors(N): + res = [] + for i in range(1, N+1): + if N % i == 0: + res.append(i) + return res + +def findspec(B, H, W, G, C): + key = f"{B}x{H}x{W}x{G}x{C}" + if key in TABLE: + return TABLE[key][0], TABLE[key][1] + + d_stride = 8 + ms = factors(B*H*W) + multiplier = 1 + for m in ms: + if m <= 64 and (m * G * C // d_stride) <= 512: + multiplier = m + n_thread = multiplier * G * C // d_stride + key = f"{B}x{H}x{W}x{G}x{C}" + TABLE[key] = (d_stride, n_thread) + return d_stride, n_thread + +def find_spec_bwd(B, H, W, G, C): + key = f"{B}x{H}x{W}x{G}x{C}" + if key in BWDTABLE: + return BWDTABLE[key][0], BWDTABLE[key][1] + + if C >= 64: + d_stride = 2 + else: + d_stride = 1 + + ms = factors(B*H*W) + multiplier = 1 + for m in ms: + if m <= 64 and (m * G * C // d_stride) <= 256: + multiplier = m + n_thread = multiplier * G * C // d_stride + return d_stride, n_thread + +class DCNv4Function(Function): + @staticmethod + @custom_fwd + def forward( + ctx, input, offset_mask, + kernel_h, kernel_w, stride_h, stride_w, + pad_h, pad_w, dilation_h, dilation_w, + group, group_channels, offset_scale, + im2col_step, remove_center): + + forward_d_stride, forward_block_thread = findspec(input.shape[0], input.shape[1], input.shape[2], group, group_channels) + backward_d_stride, backward_block_thread = find_spec_bwd(input.shape[0], input.shape[1], input.shape[2], group, group_channels) + + ctx.kernel_h = kernel_h + ctx.kernel_w = kernel_w + ctx.stride_h = stride_h + ctx.stride_w = stride_w + ctx.pad_h = pad_h + ctx.pad_w = pad_w + ctx.dilation_h = dilation_h + ctx.dilation_w = dilation_w + ctx.group = group + ctx.group_channels = group_channels + ctx.offset_scale = offset_scale + ctx.im2col_step = im2col_step + ctx.remove_center = remove_center + ctx.backward_d_stride = backward_d_stride + ctx.backward_block_thread = backward_block_thread + + args = [ + input, offset_mask, kernel_h, + kernel_w, stride_h, stride_w, pad_h, + pad_w, dilation_h, dilation_w, group, + group_channels, offset_scale, + ctx.im2col_step, + remove_center, + forward_d_stride, + forward_block_thread, + False, + ] + + output = ext.dcnv4_forward(*args) + ctx.save_for_backward(input, offset_mask) + + return output + + @staticmethod + @once_differentiable + @custom_bwd + def backward(ctx, grad_output): + input, offset_mask = ctx.saved_tensors + + args = [ + input, offset_mask, ctx.kernel_h, + ctx.kernel_w, ctx.stride_h, ctx.stride_w, ctx.pad_h, + ctx.pad_w, ctx.dilation_h, ctx.dilation_w, ctx.group, + ctx.group_channels, ctx.offset_scale, ctx.im2col_step, + grad_output.contiguous(), ctx.remove_center, + ctx.backward_d_stride, ctx.backward_block_thread, + False + ] + + grad_input, grad_offset_mask = \ + ext.dcnv4_backward(*args) + + return grad_input, grad_offset_mask, \ + None, None, None, None, None, None, None,\ + None, None, None, None, None, None diff --git a/DCNv4_op/DCNv4/functions/flash_deform_attn_func.py b/DCNv4_op/DCNv4/functions/flash_deform_attn_func.py new file mode 100644 index 0000000..08c44cc --- /dev/null +++ b/DCNv4_op/DCNv4/functions/flash_deform_attn_func.py @@ -0,0 +1,114 @@ +# ------------------------------------------------------------------------------------------------ +# Deformable DETR +# Copyright (c) 2020 SenseTime. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------------------------------ +# Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +# ------------------------------------------------------------------------------------------------ + +from __future__ import absolute_import +from __future__ import print_function +from __future__ import division + +import torch +import torch.nn.functional as F +from torch.autograd import Function +from torch.autograd.function import once_differentiable +import numpy as np + +from DCNv4 import ext + +shm_size_dict = { + "8.0": 163000, + "8.6": 99000, + "8.7": 163000, + "8.9": 99000, + "9.0": 227000, + "7.5": 64000, + "7.0": 96000, +} + +cuda_capability = f"{torch.cuda.get_device_properties(0).major}.{torch.cuda.get_device_properties(0).minor}" + +if cuda_capability not in shm_size_dict: + raise NotImplementedError + +shm_size_cap = shm_size_dict[cuda_capability] + +def factors(N): + res = [] + for i in range(1, N+1): + if N % i == 0: + res.append(i) + return res + +def findspec(B, Q, G, C): + d_stride = 8 + ms = factors(B*Q) + multiplier = 1 + for m in ms: + if m <= 64 and (m * G * C // d_stride) <= 512: + multiplier = m + n_thread = multiplier * G * C // d_stride + return d_stride, n_thread + +def findspec_bwd(B, Q, G, C): + if C >= 64: + d_stride = 2 + else: + d_stride = 1 + + ms = factors(B*Q) + multiplier = 1 + for m in ms: + if m <= 64 and (m * G * C // d_stride) <= 256: + multiplier = m + n_thread = multiplier * G * C // d_stride + return d_stride, n_thread + +class FlashDeformAttnFunction(Function): + @staticmethod + @torch.autocast("cuda", enabled=True, dtype=torch.float16) + def forward( + ctx, value, value_spatial_shapes, value_level_start_index, + sampling_loc_attn, im2col_step, K=8 + ): + + ctx.im2col_step = im2col_step + ctx.K = K + d_stride, blockthread = findspec(value.shape[0], sampling_loc_attn.shape[1], value.shape[2], value.shape[3]) + d_stride_backward, blockthread_backward = findspec_bwd(value.shape[0], sampling_loc_attn.shape[1], value.shape[2], value.shape[3]) + + ctx.d_stride_backward = d_stride_backward + ctx.blockthread_backward = blockthread_backward + + output = ext.flash_deform_attn_forward( + value, + value_spatial_shapes, + value_level_start_index, + sampling_loc_attn, + ctx.im2col_step, + K, + d_stride, + blockthread, + ) + ctx.save_for_backward(value, value_spatial_shapes, value_level_start_index, sampling_loc_attn) + return output + + @staticmethod + @once_differentiable + def backward(ctx, grad_output): + value, value_spatial_shapes, value_level_start_index, sampling_loc_attn = ctx.saved_tensors + grad_value, grad_sampling_loc_attn = ext.flash_deform_attn_backward( + value, + value_spatial_shapes, + value_level_start_index, + sampling_loc_attn, + grad_output.contiguous(), + ctx.im2col_step, + ctx.K, + ctx.d_stride_backward, + ctx.blockthread_backward, + ) + + return grad_value, None, None, grad_sampling_loc_attn, None, None diff --git a/DCNv4_op/DCNv4/functions/table.py b/DCNv4_op/DCNv4/functions/table.py new file mode 100644 index 0000000..d726371 --- /dev/null +++ b/DCNv4_op/DCNv4/functions/table.py @@ -0,0 +1,1355 @@ +TABLE = { + "64x56x56x4x16": [ + 8, + 448, + 56 + ], + "64x28x28x4x16": [ + 8, + 448, + 56 + ], + "64x14x14x4x16": [ + 8, + 32, + 4 + ], + "64x7x7x4x16": [ + 8, + 56, + 7 + ], + "1x200x320x4x16": [ + 8, + 32, + 4 + ], + "1x100x160x4x16": [ + 8, + 32, + 4 + ], + "1x50x80x4x16": [ + 4, + 512, + 32 + ], + "1x25x40x4x16": [ + 4, + 320, + 20 + ], + "1x64x64x4x16": [ + 8, + 512, + 64 + ], + "64x56x56x5x16": [ + 8, + 490, + 49 + ], + "64x28x28x5x16": [ + 8, + 490, + 49 + ], + "64x14x14x5x16": [ + 8, + 280, + 28 + ], + "64x7x7x5x16": [ + 4, + 140, + 7 + ], + "1x200x320x5x16": [ + 4, + 400, + 20 + ], + "1x100x160x5x16": [ + 4, + 400, + 20 + ], + "1x50x80x5x16": [ + 8, + 500, + 50 + ], + "1x25x40x5x16": [ + 8, + 20, + 2 + ], + "1x64x64x5x16": [ + 8, + 320, + 32 + ], + "64x56x56x6x16": [ + 8, + 768, + 64 + ], + "64x28x28x6x16": [ + 8, + 672, + 56 + ], + "64x14x14x6x16": [ + 8, + 336, + 28 + ], + "64x7x7x6x16": [ + 8, + 84, + 7 + ], + "1x200x320x6x16": [ + 4, + 600, + 25 + ], + "1x100x160x6x16": [ + 4, + 600, + 25 + ], + "1x50x80x6x16": [ + 8, + 600, + 50 + ], + "1x25x40x6x16": [ + 2, + 240, + 5 + ], + "1x64x64x6x16": [ + 8, + 384, + 32 + ], + "64x56x56x7x16": [ + 8, + 896, + 64 + ], + "64x28x28x7x16": [ + 8, + 686, + 49 + ], + "64x14x14x7x16": [ + 8, + 392, + 28 + ], + "64x7x7x7x16": [ + 8, + 686, + 49 + ], + "1x200x320x7x16": [ + 8, + 700, + 50 + ], + "1x100x160x7x16": [ + 8, + 700, + 50 + ], + "1x50x80x7x16": [ + 8, + 700, + 50 + ], + "1x25x40x7x16": [ + 8, + 70, + 5 + ], + "1x64x64x7x16": [ + 8, + 448, + 32 + ], + "64x56x56x8x16": [ + 8, + 448, + 28 + ], + "64x28x28x8x16": [ + 8, + 448, + 28 + ], + "64x14x14x8x16": [ + 8, + 448, + 28 + ], + "64x7x7x8x16": [ + 8, + 784, + 49 + ], + "1x200x320x8x16": [ + 8, + 800, + 50 + ], + "1x100x160x8x16": [ + 4, + 640, + 20 + ], + "1x50x80x8x16": [ + 8, + 800, + 50 + ], + "1x25x40x8x16": [ + 4, + 64, + 2 + ], + "1x64x64x8x16": [ + 8, + 256, + 16 + ], + "64x56x56x4x32": [ + 8, + 448, + 28 + ], + "64x28x28x4x32": [ + 8, + 448, + 28 + ], + "64x14x14x4x32": [ + 8, + 448, + 28 + ], + "64x7x7x4x32": [ + 8, + 112, + 7 + ], + "1x200x320x4x32": [ + 8, + 512, + 32 + ], + "1x100x160x4x32": [ + 8, + 800, + 50 + ], + "1x50x80x4x32": [ + 8, + 800, + 50 + ], + "1x25x40x4x32": [ + 4, + 128, + 4 + ], + "1x64x64x4x32": [ + 8, + 128, + 8 + ], + "64x56x56x5x32": [ + 8, + 560, + 28 + ], + "64x28x28x5x32": [ + 8, + 560, + 28 + ], + "64x14x14x5x32": [ + 8, + 560, + 28 + ], + "64x7x7x5x32": [ + 8, + 980, + 49 + ], + "1x200x320x5x32": [ + 8, + 500, + 25 + ], + "1x100x160x5x32": [ + 8, + 800, + 40 + ], + "1x50x80x5x32": [ + 8, + 1000, + 50 + ], + "1x25x40x5x32": [ + 4, + 200, + 5 + ], + "1x64x64x5x32": [ + 8, + 640, + 32 + ], + "64x56x56x6x32": [ + 8, + 336, + 14 + ], + "64x28x28x6x32": [ + 8, + 336, + 14 + ], + "64x14x14x6x32": [ + 8, + 336, + 14 + ], + "64x7x7x6x32": [ + 16, + 588, + 49 + ], + "1x200x320x6x32": [ + 8, + 480, + 20 + ], + "1x100x160x6x32": [ + 8, + 480, + 20 + ], + "1x50x80x6x32": [ + 16, + 600, + 50 + ], + "1x25x40x6x32": [ + 8, + 96, + 4 + ], + "1x64x64x6x32": [ + 8, + 768, + 32 + ], + "64x56x56x7x32": [ + 8, + 448, + 16 + ], + "64x28x28x7x32": [ + 8, + 448, + 16 + ], + "64x14x14x7x32": [ + 8, + 196, + 7 + ], + "64x7x7x7x32": [ + 8, + 28, + 1 + ], + "1x200x320x7x32": [ + 8, + 448, + 16 + ], + "1x100x160x7x32": [ + 8, + 448, + 16 + ], + "1x50x80x7x32": [ + 8, + 700, + 25 + ], + "1x25x40x7x32": [ + 8, + 56, + 2 + ], + "1x64x64x7x32": [ + 8, + 896, + 32 + ], + "64x56x56x8x32": [ + 8, + 448, + 14 + ], + "64x28x28x8x32": [ + 8, + 448, + 14 + ], + "64x14x14x8x32": [ + 8, + 448, + 14 + ], + "64x7x7x8x32": [ + 8, + 32, + 1 + ], + "1x200x320x8x32": [ + 8, + 512, + 16 + ], + "1x100x160x8x32": [ + 8, + 800, + 25 + ], + "1x50x80x8x32": [ + 8, + 800, + 25 + ], + "1x25x40x8x32": [ + 4, + 512, + 8 + ], + "1x64x64x8x32": [ + 8, + 32, + 1 + ], + "64x56x56x4x64": [ + 8, + 448, + 14 + ], + "64x28x28x4x64": [ + 8, + 448, + 14 + ], + "64x14x14x4x64": [ + 8, + 448, + 14 + ], + "64x7x7x4x64": [ + 8, + 32, + 1 + ], + "1x200x320x4x64": [ + 8, + 512, + 16 + ], + "1x100x160x4x64": [ + 8, + 512, + 16 + ], + "1x50x80x4x64": [ + 8, + 800, + 25 + ], + "1x25x40x4x64": [ + 8, + 640, + 20 + ], + "1x64x64x4x64": [ + 8, + 512, + 16 + ], + "64x56x56x5x64": [ + 8, + 560, + 14 + ], + "64x28x28x5x64": [ + 8, + 560, + 14 + ], + "64x14x14x5x64": [ + 8, + 560, + 14 + ], + "64x7x7x5x64": [ + 8, + 280, + 7 + ], + "1x200x320x5x64": [ + 8, + 800, + 20 + ], + "1x100x160x5x64": [ + 8, + 800, + 20 + ], + "1x50x80x5x64": [ + 8, + 1000, + 25 + ], + "1x25x40x5x64": [ + 8, + 80, + 2 + ], + "1x64x64x5x64": [ + 8, + 320, + 8 + ], + "64x56x56x6x64": [ + 8, + 768, + 16 + ], + "64x28x28x6x64": [ + 8, + 768, + 16 + ], + "64x14x14x6x64": [ + 8, + 336, + 7 + ], + "64x7x7x6x64": [ + 8, + 336, + 7 + ], + "1x200x320x6x64": [ + 8, + 768, + 16 + ], + "1x100x160x6x64": [ + 8, + 480, + 10 + ], + "1x50x80x6x64": [ + 16, + 240, + 10 + ], + "1x25x40x6x64": [ + 8, + 240, + 5 + ], + "1x64x64x6x64": [ + 8, + 768, + 16 + ], + "64x56x56x7x64": [ + 8, + 896, + 16 + ], + "64x28x28x7x64": [ + 8, + 448, + 8 + ], + "64x14x14x7x64": [ + 8, + 392, + 7 + ], + "64x7x7x7x64": [ + 8, + 56, + 1 + ], + "1x200x320x7x64": [ + 8, + 896, + 16 + ], + "1x100x160x7x64": [ + 8, + 448, + 8 + ], + "1x50x80x7x64": [ + 8, + 448, + 8 + ], + "1x25x40x7x64": [ + 8, + 448, + 8 + ], + "1x64x64x7x64": [ + 8, + 448, + 8 + ], + "64x56x56x8x64": [ + 8, + 896, + 14 + ], + "64x28x28x8x64": [ + 8, + 896, + 14 + ], + "64x14x14x8x64": [ + 8, + 448, + 7 + ], + "64x7x7x8x64": [ + 8, + 64, + 1 + ], + "1x200x320x8x64": [ + 8, + 512, + 8 + ], + "1x100x160x8x64": [ + 8, + 512, + 8 + ], + "1x50x80x8x64": [ + 8, + 512, + 8 + ], + "1x25x40x8x64": [ + 8, + 512, + 8 + ], + "1x64x64x8x64": [ + 8, + 512, + 8 + ] +} + +BWDTABLE = { + "64x56x56x4x16": [ + 1, + 256, + 4 + ], + "64x56x56x5x16": [ + 1, + 320, + 4 + ], + "64x56x56x6x16": [ + 1, + 192, + 2 + ], + "64x56x56x7x16": [ + 1, + 224, + 2 + ], + "64x56x56x8x16": [ + 1, + 256, + 2 + ], + "64x56x56x4x32": [ + 1, + 256, + 2 + ], + "64x56x56x5x32": [ + 1, + 160, + 1 + ], + "64x56x56x6x32": [ + 1, + 192, + 1 + ], + "64x56x56x7x32": [ + 1, + 224, + 1 + ], + "64x56x56x8x32": [ + 1, + 256, + 1 + ], + "64x56x56x4x64": [ + 2, + 512, + 4 + ], + "64x56x56x5x64": [ + 2, + 640, + 4 + ], + "64x56x56x6x64": [ + 2, + 384, + 2 + ], + "64x56x56x7x64": [ + 2, + 224, + 1 + ], + "64x56x56x8x64": [ + 2, + 1024, + 4 + ], + "64x28x28x4x16": [ + 1, + 128, + 2 + ], + "64x28x28x5x16": [ + 1, + 320, + 4 + ], + "64x28x28x6x16": [ + 1, + 96, + 1 + ], + "64x28x28x7x16": [ + 1, + 224, + 2 + ], + "64x28x28x8x16": [ + 1, + 128, + 1 + ], + "64x28x28x4x32": [ + 1, + 128, + 1 + ], + "64x28x28x5x32": [ + 1, + 320, + 2 + ], + "64x28x28x6x32": [ + 1, + 192, + 1 + ], + "64x28x28x7x32": [ + 1, + 224, + 1 + ], + "64x28x28x8x32": [ + 1, + 256, + 1 + ], + "64x28x28x4x64": [ + 2, + 512, + 4 + ], + "64x28x28x5x64": [ + 2, + 640, + 4 + ], + "64x28x28x6x64": [ + 2, + 384, + 2 + ], + "64x28x28x7x64": [ + 2, + 224, + 1 + ], + "64x28x28x8x64": [ + 2, + 512, + 2 + ], + "64x14x14x4x16": [ + 1, + 128, + 2 + ], + "64x14x14x5x16": [ + 1, + 320, + 4 + ], + "64x14x14x6x16": [ + 1, + 192, + 2 + ], + "64x14x14x7x16": [ + 1, + 224, + 2 + ], + "64x14x14x8x16": [ + 1, + 128, + 1 + ], + "64x14x14x4x32": [ + 1, + 256, + 2 + ], + "64x14x14x5x32": [ + 1, + 160, + 1 + ], + "64x14x14x6x32": [ + 1, + 192, + 1 + ], + "64x14x14x7x32": [ + 1, + 224, + 1 + ], + "64x14x14x8x32": [ + 1, + 256, + 1 + ], + "64x14x14x4x64": [ + 2, + 128, + 1 + ], + "64x14x14x5x64": [ + 2, + 160, + 1 + ], + "64x14x14x6x64": [ + 2, + 384, + 2 + ], + "64x14x14x7x64": [ + 2, + 224, + 1 + ], + "64x14x14x8x64": [ + 2, + 256, + 1 + ], + "64x7x7x4x16": [ + 4, + 784, + 49 + ], + "64x7x7x5x16": [ + 2, + 280, + 7 + ], + "64x7x7x6x16": [ + 2, + 48, + 1 + ], + "64x7x7x7x16": [ + 2, + 392, + 7 + ], + "64x7x7x8x16": [ + 1, + 128, + 1 + ], + "64x7x7x4x32": [ + 1, + 128, + 1 + ], + "64x7x7x5x32": [ + 1, + 160, + 1 + ], + "64x7x7x6x32": [ + 2, + 96, + 1 + ], + "64x7x7x7x32": [ + 2, + 112, + 1 + ], + "64x7x7x8x32": [ + 2, + 128, + 1 + ], + "64x7x7x4x64": [ + 2, + 896, + 7 + ], + "64x7x7x5x64": [ + 2, + 160, + 1 + ], + "64x7x7x6x64": [ + 2, + 192, + 1 + ], + "64x7x7x7x64": [ + 2, + 224, + 1 + ], + "64x7x7x8x64": [ + 2, + 256, + 1 + ], + "1x200x320x4x16": [ + 1, + 320, + 5 + ], + "1x200x320x5x16": [ + 1, + 320, + 4 + ], + "1x200x320x6x16": [ + 1, + 96, + 1 + ], + "1x200x320x7x16": [ + 1, + 224, + 2 + ], + "1x200x320x8x16": [ + 1, + 640, + 5 + ], + "1x200x320x4x32": [ + 1, + 128, + 1 + ], + "1x200x320x5x32": [ + 1, + 320, + 2 + ], + "1x200x320x6x32": [ + 1, + 384, + 2 + ], + "1x200x320x7x32": [ + 1, + 224, + 1 + ], + "1x200x320x8x32": [ + 1, + 256, + 1 + ], + "1x200x320x4x64": [ + 2, + 640, + 5 + ], + "1x200x320x5x64": [ + 2, + 800, + 5 + ], + "1x200x320x6x64": [ + 2, + 768, + 4 + ], + "1x200x320x7x64": [ + 2, + 448, + 2 + ], + "1x200x320x8x64": [ + 2, + 1024, + 4 + ], + "1x100x160x4x16": [ + 1, + 320, + 5 + ], + "1x100x160x5x16": [ + 1, + 640, + 8 + ], + "1x100x160x6x16": [ + 1, + 96, + 1 + ], + "1x100x160x7x16": [ + 1, + 224, + 2 + ], + "1x100x160x8x16": [ + 1, + 640, + 5 + ], + "1x100x160x4x32": [ + 1, + 256, + 2 + ], + "1x100x160x5x32": [ + 1, + 160, + 1 + ], + "1x100x160x6x32": [ + 1, + 384, + 2 + ], + "1x100x160x7x32": [ + 1, + 224, + 1 + ], + "1x100x160x8x32": [ + 1, + 512, + 2 + ], + "1x100x160x4x64": [ + 2, + 128, + 1 + ], + "1x100x160x5x64": [ + 2, + 160, + 1 + ], + "1x100x160x6x64": [ + 2, + 384, + 2 + ], + "1x100x160x7x64": [ + 2, + 448, + 2 + ], + "1x100x160x8x64": [ + 2, + 512, + 2 + ], + "1x50x80x4x16": [ + 1, + 320, + 5 + ], + "1x50x80x5x16": [ + 1, + 320, + 4 + ], + "1x50x80x6x16": [ + 1, + 96, + 1 + ], + "1x50x80x7x16": [ + 1, + 112, + 1 + ], + "1x50x80x8x16": [ + 1, + 512, + 4 + ], + "1x50x80x4x32": [ + 1, + 128, + 1 + ], + "1x50x80x5x32": [ + 1, + 320, + 2 + ], + "1x50x80x6x32": [ + 1, + 384, + 2 + ], + "1x50x80x7x32": [ + 1, + 224, + 1 + ], + "1x50x80x8x32": [ + 1, + 256, + 1 + ], + "1x50x80x4x64": [ + 2, + 256, + 2 + ], + "1x50x80x5x64": [ + 2, + 640, + 4 + ], + "1x50x80x6x64": [ + 2, + 768, + 4 + ], + "1x50x80x7x64": [ + 2, + 448, + 2 + ], + "1x50x80x8x64": [ + 2, + 1024, + 4 + ], + "1x25x40x4x16": [ + 1, + 320, + 5 + ], + "1x25x40x5x16": [ + 2, + 400, + 10 + ], + "1x25x40x6x16": [ + 1, + 192, + 2 + ], + "1x25x40x7x16": [ + 4, + 224, + 8 + ], + "1x25x40x8x16": [ + 4, + 160, + 5 + ], + "1x25x40x4x32": [ + 2, + 128, + 2 + ], + "1x25x40x5x32": [ + 1, + 320, + 2 + ], + "1x25x40x6x32": [ + 2, + 96, + 1 + ], + "1x25x40x7x32": [ + 2, + 112, + 1 + ], + "1x25x40x8x32": [ + 2, + 640, + 5 + ], + "1x25x40x4x64": [ + 2, + 128, + 1 + ], + "1x25x40x5x64": [ + 2, + 160, + 1 + ], + "1x25x40x6x64": [ + 2, + 192, + 1 + ], + "1x25x40x7x64": [ + 2, + 896, + 4 + ], + "1x25x40x8x64": [ + 2, + 512, + 2 + ], + "1x64x64x4x16": [ + 1, + 256, + 4 + ], + "1x64x64x5x16": [ + 2, + 40, + 1 + ], + "1x64x64x6x16": [ + 1, + 192, + 2 + ], + "1x64x64x7x16": [ + 1, + 224, + 2 + ], + "1x64x64x8x16": [ + 1, + 512, + 4 + ], + "1x64x64x4x32": [ + 2, + 64, + 1 + ], + "1x64x64x5x32": [ + 1, + 320, + 2 + ], + "1x64x64x6x32": [ + 1, + 192, + 1 + ], + "1x64x64x7x32": [ + 1, + 224, + 1 + ], + "1x64x64x8x32": [ + 1, + 256, + 1 + ], + "1x64x64x4x64": [ + 2, + 512, + 4 + ], + "1x64x64x5x64": [ + 2, + 640, + 4 + ], + "1x64x64x6x64": [ + 2, + 192, + 1 + ], + "1x64x64x7x64": [ + 2, + 224, + 1 + ], + "1x64x64x8x64": [ + 2, + 256, + 1 + ] +} \ No newline at end of file diff --git a/DCNv4_op/DCNv4/modules/__init__.py b/DCNv4_op/DCNv4/modules/__init__.py new file mode 100644 index 0000000..08e6bbd --- /dev/null +++ b/DCNv4_op/DCNv4/modules/__init__.py @@ -0,0 +1,10 @@ +# ------------------------------------------------------------------------------------------------ +# Deformable DETR +# Copyright (c) 2020 SenseTime. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------------------------------ +# Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +# ------------------------------------------------------------------------------------------------ + +from .flash_deform_attn import FlashDeformAttn +from .dcnv4 import DCNv4 diff --git a/DCNv4_op/DCNv4/modules/dcnv4.py b/DCNv4_op/DCNv4/modules/dcnv4.py new file mode 100644 index 0000000..560e9d7 --- /dev/null +++ b/DCNv4_op/DCNv4/modules/dcnv4.py @@ -0,0 +1,152 @@ +# -------------------------------------------------------- +# Deformable Convolution v4 +# Copyright (c) 2023 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + +from __future__ import absolute_import +from __future__ import print_function +from __future__ import division + +import math +import torch +from torch import nn +import torch.nn.functional as F +from torch.nn.init import xavier_uniform_, constant_ +from ..functions import DCNv4Function + +class CenterFeatureScaleModule(nn.Module): + def forward(self, + query, + center_feature_scale_proj_weight, + center_feature_scale_proj_bias): + center_feature_scale = F.linear(query, + weight=center_feature_scale_proj_weight, + bias=center_feature_scale_proj_bias).sigmoid() + return center_feature_scale + +class DCNv4(nn.Module): + def __init__( + self, + channels=64, + kernel_size=3, + stride=1, + pad=1, + dilation=1, + group=4, + offset_scale=1.0, + dw_kernel_size=None, + center_feature_scale=False, + remove_center=False, + output_bias=True, + without_pointwise=False, + **kwargs): + """ + DCNv4 Module + :param channels + :param kernel_size + :param stride + :param pad + :param dilation + :param group + :param offset_scale + :param act_layer + :param norm_layer + """ + super().__init__() + if channels % group != 0: + raise ValueError( + f'channels must be divisible by group, but got {channels} and {group}') + _d_per_group = channels // group + + # you'd better set _d_per_group to a power of 2 which is more efficient in our CUDA implementation + assert _d_per_group % 16 == 0 + + self.offset_scale = offset_scale + self.channels = channels + self.kernel_size = kernel_size + self.stride = stride + self.dilation = dilation + self.pad = pad + self.group = group + self.group_channels = channels // group + self.offset_scale = offset_scale + self.dw_kernel_size = dw_kernel_size + self.center_feature_scale = center_feature_scale + self.remove_center = int(remove_center) + self.without_pointwise = without_pointwise + + self.K = group * (kernel_size * kernel_size - self.remove_center) + if dw_kernel_size is not None: + self.offset_mask_dw = nn.Conv2d(channels, channels, dw_kernel_size, stride=1, padding=(dw_kernel_size - 1) // 2, groups=channels) + self.offset_mask = nn.Linear(channels, int(math.ceil((self.K * 3)/8)*8)) + if not without_pointwise: + self.value_proj = nn.Linear(channels, channels) + self.output_proj = nn.Linear(channels, channels, bias=output_bias) + self._reset_parameters() + + if center_feature_scale: + self.center_feature_scale_proj_weight = nn.Parameter( + torch.zeros((group, channels), dtype=torch.float)) + self.center_feature_scale_proj_bias = nn.Parameter( + torch.tensor(0.0, dtype=torch.float).view((1,)).repeat(group, )) + self.center_feature_scale_module = CenterFeatureScaleModule() + + def _reset_parameters(self): + constant_(self.offset_mask.weight.data, 0.) + constant_(self.offset_mask.bias.data, 0.) + if not self.without_pointwise: + xavier_uniform_(self.value_proj.weight.data) + constant_(self.value_proj.bias.data, 0.) + xavier_uniform_(self.output_proj.weight.data) + if self.output_proj.bias is not None: + constant_(self.output_proj.bias.data, 0.) + + def forward(self, input, shape=None): + """ + :param query (N, H, W, C) + :return output (N, H, W, C) + """ + N, L, C = input.shape + if shape is not None: + H, W = shape + else: + H, W = int(L**0.5), int(L**0.5) + + + x = input + if not self.without_pointwise: + x = self.value_proj(x) + x = x.reshape(N, H, W, -1) + if self.dw_kernel_size is not None: + offset_mask_input = self.offset_mask_dw(input.view(N, H, W, C).permute(0, 3, 1, 2)) + offset_mask_input = offset_mask_input.permute(0, 2, 3, 1).view(N, L, C) + else: + offset_mask_input = input + offset_mask = self.offset_mask(offset_mask_input).reshape(N, H, W, -1) + + x_proj = x + + x = DCNv4Function.apply( + x, offset_mask, + self.kernel_size, self.kernel_size, + self.stride, self.stride, + self.pad, self.pad, + self.dilation, self.dilation, + self.group, self.group_channels, + self.offset_scale, + 256, + self.remove_center + ) + + x = x.view(N, L, -1) + if self.center_feature_scale: + center_feature_scale = self.center_feature_scale_module( + x, self.center_feature_scale_proj_weight, self.center_feature_scale_proj_bias) + center_feature_scale = center_feature_scale[..., None].repeat( + 1, 1, 1, 1, self.channels // self.group).flatten(-2) + x = x * (1 - center_feature_scale) + x_proj * center_feature_scale + if not self.without_pointwise: + x = self.output_proj(x) + return x + diff --git a/DCNv4_op/DCNv4/modules/flash_deform_attn.py b/DCNv4_op/DCNv4/modules/flash_deform_attn.py new file mode 100644 index 0000000..4472fa6 --- /dev/null +++ b/DCNv4_op/DCNv4/modules/flash_deform_attn.py @@ -0,0 +1,141 @@ +# ------------------------------------------------------------------------------------------------ +# Deformable DETR +# Copyright (c) 2020 SenseTime. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------------------------------ +# Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +# ------------------------------------------------------------------------------------------------ + +from __future__ import absolute_import +from __future__ import print_function +from __future__ import division + +import warnings +import math + +import torch +from torch import nn +import torch.nn.functional as F +from torch.nn.init import xavier_uniform_, constant_ + +from ..functions import FlashDeformAttnFunction + + +def _is_power_of_2(n): + if (not isinstance(n, int)) or (n < 0): + raise ValueError("invalid input for _is_power_of_2: {} (type: {})".format(n, type(n))) + return (n & (n - 1) == 0) and n != 0 + + +class FlashDeformAttn(nn.Module): + def __init__(self, d_model=256, n_levels=4, n_heads=8, n_points=4): + """ + Multi-Scale Deformable Attention Module + :param d_model hidden dimension + :param n_levels number of feature levels + :param n_heads number of attention heads + :param n_points number of sampling points per attention head per feature level + """ + super().__init__() + if d_model % n_heads != 0: + raise ValueError("d_model must be divisible by n_heads, but got {} and {}".format(d_model, n_heads)) + _d_per_head = d_model // n_heads + # you'd better set _d_per_head to a power of 2 which is more efficient in our CUDA implementation + if not _is_power_of_2(_d_per_head): + warnings.warn( + "You'd better set d_model in MSDeformAttn to make the dimension of each attention head a power of 2 " + "which is more efficient in our CUDA implementation." + ) + + self.im2col_step = 64 + + self.d_model = d_model + self.n_levels = n_levels + self.n_heads = n_heads + self.n_points = n_points + + self.sampling_offsets = nn.Linear(d_model, n_heads * n_levels * n_points * 2) + self.attention_weights = nn.Linear(d_model, n_heads * n_levels * n_points) + self.value_proj = nn.Linear(d_model, d_model) + self.output_proj = nn.Linear(d_model, d_model) + + self._reset_parameters() + + def _reset_parameters(self): + constant_(self.sampling_offsets.weight.data, 0.0) + thetas = torch.arange(self.n_heads, dtype=torch.float32) * (2.0 * math.pi / self.n_heads) + grid_init = torch.stack([thetas.cos(), thetas.sin()], -1) + grid_init = ( + (grid_init / grid_init.abs().max(-1, keepdim=True)[0]) + .view(self.n_heads, 1, 1, 2) + .repeat(1, self.n_levels, self.n_points, 1) + ) + for i in range(self.n_points): + grid_init[:, :, i, :] *= i + 1 + with torch.no_grad(): + self.sampling_offsets.bias = nn.Parameter(grid_init.view(-1)) + constant_(self.attention_weights.weight.data, 0.0) + constant_(self.attention_weights.bias.data, 0.0) + xavier_uniform_(self.value_proj.weight.data) + constant_(self.value_proj.bias.data, 0.0) + xavier_uniform_(self.output_proj.weight.data) + constant_(self.output_proj.bias.data, 0.0) + + def forward( + self, + query, + reference_points, + input_flatten, + input_spatial_shapes, + input_level_start_index, + input_padding_mask=None, + ): + """ + :param query (N, Length_{query}, C) + :param reference_points (N, Length_{query}, n_levels, 2), range in [0, 1], top-left (0,0), bottom-right (1, 1), including padding area + or (N, Length_{query}, n_levels, 4), add additional (w, h) to form reference boxes + :param input_flatten (N, \sum_{l=0}^{L-1} H_l \cdot W_l, C) + :param input_spatial_shapes (n_levels, 2), [(H_0, W_0), (H_1, W_1), ..., (H_{L-1}, W_{L-1})] + :param input_level_start_index (n_levels, ), [0, H_0*W_0, H_0*W_0+H_1*W_1, H_0*W_0+H_1*W_1+H_2*W_2, ..., H_0*W_0+H_1*W_1+...+H_{L-1}*W_{L-1}] + :param input_padding_mask (N, \sum_{l=0}^{L-1} H_l \cdot W_l), True for padding elements, False for non-padding elements + + :return output (N, Length_{query}, C) + """ + N, Len_q, _ = query.shape + N, Len_in, _ = input_flatten.shape + assert (input_spatial_shapes[:, 0] * input_spatial_shapes[:, 1]).sum() == Len_in + + value = self.value_proj(input_flatten) + if input_padding_mask is not None: + value = value.masked_fill(input_padding_mask[..., None], float(0)) + value = value.view(N, Len_in, self.n_heads, self.d_model // self.n_heads) + sampling_offsets = self.sampling_offsets(query).view(N, Len_q, self.n_heads, self.n_levels, self.n_points, 2) + attention_weights = self.attention_weights(query).view(N, Len_q, self.n_heads, self.n_levels * self.n_points) + attention_weights = F.softmax(attention_weights, -1).view(N, Len_q, self.n_heads, self.n_levels, self.n_points) + # N, Len_q, n_heads, n_levels, n_points, 2 + if reference_points.shape[-1] == 2: + offset_normalizer = torch.stack([input_spatial_shapes[..., 1], input_spatial_shapes[..., 0]], -1) + sampling_locations = ( + reference_points[:, :, None, :, None, :] + + sampling_offsets / offset_normalizer[None, None, None, :, None, :] + ) + elif reference_points.shape[-1] == 4: + sampling_locations = ( + reference_points[:, :, None, :, None, :2] + + sampling_offsets / self.n_points * reference_points[:, :, None, :, None, 2:] * 0.5 + ) + else: + raise ValueError( + "Last dim of reference_points must be 2 or 4, but get {} instead.".format(reference_points.shape[-1]) + ) + + output = FlashDeformAttnFunction.apply( + value, + input_spatial_shapes, + input_level_start_index, + sampling_locations, + attention_weights, + self.im2col_step, + ) + output = self.output_proj(output) + return output diff --git a/DCNv4_op/MANIFEST.in b/DCNv4_op/MANIFEST.in new file mode 100644 index 0000000..54338ec --- /dev/null +++ b/DCNv4_op/MANIFEST.in @@ -0,0 +1,2 @@ +include src/* +include src/cuda/* diff --git a/DCNv4_op/__init__.py b/DCNv4_op/__init__.py new file mode 100644 index 0000000..ab94d0f --- /dev/null +++ b/DCNv4_op/__init__.py @@ -0,0 +1,2 @@ +from .functions import DCNv4Function, FlashDeformAttnFunction +from .modules import DCNv4, FlashDeformAttn \ No newline at end of file diff --git a/DCNv4_op/make.sh b/DCNv4_op/make.sh new file mode 100755 index 0000000..106b685 --- /dev/null +++ b/DCNv4_op/make.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# ------------------------------------------------------------------------------------------------ +# Deformable DETR +# Copyright (c) 2020 SenseTime. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------------------------------ +# Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +# ------------------------------------------------------------------------------------------------ + +python setup.py build install diff --git a/DCNv4_op/scripts/find_best.py b/DCNv4_op/scripts/find_best.py new file mode 100644 index 0000000..f3002c3 --- /dev/null +++ b/DCNv4_op/scripts/find_best.py @@ -0,0 +1,61 @@ +import json +import argparse +class LineParser: + def __init__(self) -> None: + self.data = {} + + def parse(self, line): + def startswith(line, lst): + for ele in lst: + if line.startswith(ele): + return True + return False + + if not startswith(line, ['1', '2', '3', '4', '5', '6', '7', '8', '9']): + return + + eles = line.strip().split() + key = eles[0] + if key not in self.data: + self.data[key] = [] + + self.data[key].append([eles[1], float(eles[2])]) + + def sort(self): + for k, v in self.data.items(): + nv = sorted(v, key=lambda x: x[1]) + self.data[k] = nv + + def display_best(self): + for k, v in self.data.items(): + print(f'{k} \t {v[0][0]} \t {v[0][1]:.4f} \t {v[1][0]} \t {v[1][1]:.4f}') + + def display_best_python(self, output): + res = {} + def parse(spec): + d_stride = int(spec.split('/')[0]) + thread = int(spec.split('/')[1].split('(')[0]) + m = int(spec.split('(')[1].split(')')[0]) + return d_stride, thread, m + + for k, v in self.data.items(): + res[k] = parse(v[0][0]) + + with open(output, "w") as f: + json.dump(res, f, indent=4) + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--input', type=str) + parser.add_argument('--output', type=str) + args = parser.parse_args() + + with open(args.input) as f: + lines = f.readlines() + + lineparser = LineParser() + for line in lines: + lineparser.parse(line) + lineparser.sort() + lineparser.display_best() + lineparser.display_best_python(args.output) \ No newline at end of file diff --git a/DCNv4_op/scripts/search_bwd.sh b/DCNv4_op/scripts/search_bwd.sh new file mode 100644 index 0000000..fb2afad --- /dev/null +++ b/DCNv4_op/scripts/search_bwd.sh @@ -0,0 +1,2 @@ +python search_dcnv4_bwd_engine.py > res_bwd.txt +python find_best.py --input res_bwd.txt --output table_bwd.py \ No newline at end of file diff --git a/DCNv4_op/scripts/search_dcnv4.py b/DCNv4_op/scripts/search_dcnv4.py new file mode 100644 index 0000000..07e3791 --- /dev/null +++ b/DCNv4_op/scripts/search_dcnv4.py @@ -0,0 +1,131 @@ + +from __future__ import absolute_import +from __future__ import print_function +from __future__ import division + +import time +import math +import torch +import torch.nn as nn +import math +from torch.autograd import gradcheck +import pandas as pd +from easydict import EasyDict as edict +import argparse + +from torch.cuda import Event + +from functions.dcnv3_func import DCNv3Function, dcnv3_core_pytorch +from functions.dcnv4_func import DCNv4Function +torch.set_printoptions(threshold=10000) + +torch.manual_seed(3) + + +#@torch.no_grad() +def speed_test(func, args, inputs, name='Unknown'): + + tic = Event(enable_timing=True) + toc = Event(enable_timing=True) + # warmup + for i in range(args.warmup_num): + func(*inputs) + + total_time = 0 + tic.record() + for i in range(args.test_num): + o = func(*inputs) + torch.cuda.synchronize() + toc.record() + + avg_time = tic.elapsed_time(toc) / args.test_num + # print( + # f'>>> {name: <10} finished {args.test_num} running, avg_time: {avg_time:.6f} ms') + return avg_time + +@torch.no_grad() +def test(N, H_in, W_in, M, D, spec=None): + Kh, Kw = 3, 3 + remove_center = False + P = Kh * Kw - remove_center + offset_scale = 2.0 + pad = 1 + dilation = 1 + stride = 1 + H_out = (H_in + 2 * pad - (dilation * (Kh - 1) + 1)) // stride + 1 + W_out = (W_in + 2 * pad - (dilation * (Kw - 1) + 1)) // stride + 1 + + input = torch.rand(N, H_in, W_in, M*D).cuda() + # print(input.shape) + offset = (torch.rand(N, H_out, W_out, M*P*2).cuda() * 2 - 1)*2 + # offset = (torch.rand(N, H_out, W_out, M*P*2).cuda() * 2 - 1)*0 + mask_origin = torch.rand(N, H_out, W_out, M, P).cuda() + 1e-5 + mask_origin = mask_origin.half() + mask = mask_origin + # mask = torch.nn.functional.softmax(mask_origin, dim=-1) + offset_mask = torch.cat([offset.unflatten(-1, (M, P * 2)), mask_origin.detach()], dim=-1).flatten(-2) + + im2col_step = 128 + + input = input.half() + offset = offset.half() + mask = mask.half() + offset_mask = offset_mask.half() + + dcnv3_args = [ + input, + offset, + mask, + Kh, Kw, stride, stride, Kh // 2, Kw // 2, dilation, dilation, M, D, offset_scale, + im2col_step, remove_center, + ] + output_pytorch = DCNv3Function.apply(*dcnv3_args) + + input1 = input.detach() + + def pad(om): + padded_zero = int(math.ceil(om.shape[3]/8)*8) - om.shape[3] + padded = torch.zeros(om.shape[0], om.shape[1], om.shape[2], padded_zero).to(om) + return torch.cat([om, padded], dim=-1) + + dcnv4_args = [ + input1, pad(offset_mask), + Kh, Kw, stride, stride, Kh // 2, Kw // 2, dilation, dilation, M, D, offset_scale, + im2col_step, remove_center, + spec[0], spec[1], 2, None + # 8, 512, 2, 256 + ] + output_flash_cuda = DCNv4Function.apply(*dcnv4_args) + + fwdok = torch.allclose(output_flash_cuda, output_pytorch, rtol=1e-2, atol=1e-3) + max_abs_err = (output_flash_cuda - output_pytorch).abs().max() + max_rel_err = ((output_flash_cuda - output_pytorch).abs() / + (output_pytorch.abs()+ 1e-3)).max() + # print('>>> forward half') + # print(f'* {fwdok} check_forward_equal_with_pytorch_float: max_abs_err {max_abs_err:.2e} max_rel_err {max_rel_err:.2e}') + if not fwdok: + print(f"Wrong: {N}x{H_in}x{W_in}x{M}x{D} \t {spec[0]}/{spec[1]}({spec[2]})") + return + # assert(fwdok) + + test_args = edict({'warmup_num': 10000, 'test_num': 10000}) + + exp_time_dcnv4 = speed_test(DCNv4Function.apply, test_args, dcnv4_args, name='exp') + torch.cuda.synchronize() + print(f"{N}x{H_in}x{W_in}x{M}x{D} \t {spec[0]}/{spec[1]}({spec[2]}): {exp_time_dcnv4}") + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument("--n", type=int) + parser.add_argument("--h", type=int) + parser.add_argument("--w", type=int) + parser.add_argument("--g", type=int) + parser.add_argument("--c", type=int) + parser.add_argument("--dstride", type=int) + parser.add_argument("--blockthread", type=int) + parser.add_argument("--multiplier", type=int) + args = parser.parse_args() + test(args.n, args.h, args.w, args.g, args.c, (args.dstride, args.blockthread, args.multiplier)) + + diff --git a/DCNv4_op/scripts/search_dcnv4_bwd.py b/DCNv4_op/scripts/search_dcnv4_bwd.py new file mode 100644 index 0000000..a28d7c5 --- /dev/null +++ b/DCNv4_op/scripts/search_dcnv4_bwd.py @@ -0,0 +1,200 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + +from __future__ import absolute_import +from __future__ import print_function +from __future__ import division + +import time +import torch +import torch.nn as nn +import math +from torch.autograd import gradcheck +import pandas as pd +from easydict import EasyDict as edict +import argparse + +from torch.cuda import Event + +from functions import DCNv4Function, DCNv3Function +torch.set_printoptions(threshold=10000) + + + +torch.manual_seed(3) + +def speed_test_backward(func, args, inputs, name='Unknown'): + # warmup + # for i in range(args.warmup_num): + # o = func(*inputs) + # o.sum().backward() + + total_time = 0 + len_input = len(inputs) + for i in range(args.warmup_num + args.test_num): + tic = Event(enable_timing=True) + toc = Event(enable_timing=True) + inputs[0] = inputs[0].detach() + inputs[0].requires_grad = True + if len_input > 1 and isinstance(inputs[1], torch.Tensor): + inputs[1] = inputs[1].detach() + inputs[1].requires_grad = True + if len_input > 2 and isinstance(inputs[2], torch.Tensor): + inputs[2] = inputs[2].detach() + inputs[2].requires_grad = True + + o = func(*inputs) + torch.cuda.synchronize() + tic.record() + o.sum().backward() + toc.record() + torch.cuda.synchronize() + _time = tic.elapsed_time(toc) + if i >= args.warmup_num: + total_time += _time + o = o.detach() + + # toc.record() + # torch.cuda.synchronize() + + avg_time = total_time / args.test_num + #print( + # f'>>> {name: <10} finished {args.test_num} running, avg_time: {avg_time:.6f} ms') + return avg_time + +# @torch.no_grad() +def test(N=64, H_in=32, W_in=32, M=4, D=16, spec=None): + """ + 64x56x56x128(G=4) + 2 64: 3.66 + - offset_mask collection write 3.4022 + - offset_mask collection 3.1968 + + """ + Kh, Kw = 3, 3 + remove_center = False + P = Kh * Kw - remove_center + offset_scale = 2.0 + pad = 1 + dilation = 1 + stride = 1 + H_out = (H_in + 2 * pad - (dilation * (Kh - 1) + 1)) // stride + 1 + W_out = (W_in + 2 * pad - (dilation * (Kw - 1) + 1)) // stride + 1 + + additions = [None, None, spec[0], spec[1], False] + input = torch.rand(N, H_in, W_in, M*D).cuda() * 10 + #offset = torch.rand(N, H_out, W_out, M*P*2).cuda() * 0 + offset = (torch.rand(N, H_out, W_out, M*P*2).cuda() * 2 - 1)*2 + mask_origin = torch.rand(N, H_out, W_out, M, P).cuda() + 1e-5 + mask_origin = mask_origin.half() + mask_origin.requires_grad = True + # offset_mask = torch.cat([offset.unflatten(-1, (M, P, 2)), mask_origin.detach().unsqueeze(-1)], dim=-1).flatten(-3) + # mask /= mask.sum(-1, keepdim=True) + # mask = torch.nn.functional.softmax(mask_origin, dim=-1, dtype=torch.float32) + mask = mask_origin + # mask = mask.reshape(N, H_out, W_out, M*P) + # offset_mask = torch.cat([offset.unflatten(-1, (M, P, 2)), mask.detach().unsqueeze(-1)], dim=-1).flatten(-3) + offset_mask = torch.cat([offset.detach().unflatten(-1, (M, P * 2)), mask_origin.detach()], dim=-1).flatten(-2) + + im2col_step = 128 + + input = input.half() + offset = offset.half() + mask = mask.half() + input.requires_grad = True + offset.requires_grad = True + # mask.requires_grad = True + output_pytorch = DCNv3Function.apply( + input, + offset, + mask, + Kh, Kw, stride, stride, Kh // 2, Kw // 2, dilation, dilation, M, D, offset_scale, + im2col_step, remove_center)#.detach().cpu() + (output_pytorch.sum()/10).backward() + + def pad(om): + padded_zero = int(math.ceil(om.shape[3]/8)*8) - om.shape[3] + padded = torch.zeros(om.shape[0], om.shape[1], om.shape[2], padded_zero).to(om) + return torch.cat([om, padded], dim=-1) + + # value_offset_mask = input.detach() + input1 = input.detach() + input1.requires_grad = True + offset_mask = offset_mask.half() + offset_mask.requires_grad = True + # offset_mask1.requires_grad = True + torch.cuda.profiler.cudart().cudaProfilerStart() + output_flash_cuda = DCNv4Function.apply( + input1, offset_mask, + Kh, Kw, stride, stride, Kh // 2, Kw // 2, dilation, dilation, M, D, offset_scale, + im2col_step, remove_center, *additions)#.detach().cpu() + (output_flash_cuda.sum()/10).backward() + torch.cuda.profiler.cudart().cudaProfilerStop() + + input_grad = input.grad + input2_grad = input1.grad + bwdok = torch.allclose(input_grad.float(), input2_grad.float(), rtol=1e-2, atol=1e-3) + rel_err = (input_grad.abs() - input2_grad.abs())/(input_grad.abs()+1e-3) + + offset_grad1 = offset.grad + offset_grad2 = offset_mask.grad.reshape(N, H_out, W_out, M, P*3)[..., :P*2].reshape(N, H_out, W_out, M*P*2) + + bwdok2 = torch.allclose(offset_grad1.float(), offset_grad2.float(), rtol=1e-2, atol=1e-3) + rel_err = (offset_grad1 - offset_grad2).abs() / (offset_grad1.abs()+1e-3) + + mask_grad1 = mask_origin.grad + mask_grad2 = offset_mask.grad.reshape(N, H_out, W_out, M, P*3)[..., P*2:].reshape(N, H_out, W_out, M, P) + + bwdok3 = torch.allclose(mask_grad1, mask_grad2, rtol=1e-2, atol=1e-3) + rel_err = (mask_grad1 - mask_grad2).abs() / (mask_grad1.abs()+1e-3) + + fwdok = torch.allclose(output_flash_cuda, output_pytorch, rtol=1e-2, atol=1e-3) + max_abs_err = (output_flash_cuda - output_pytorch).abs().max() + max_rel_err = ((output_flash_cuda - output_pytorch).abs() / + (output_pytorch.abs()+ 1e-3)).max() + if not (bwdok and bwdok2 and bwdok3): + print(f"Wrong: {N}x{H_in}x{W_in}x{M}x{D} \t {spec[0]}/{spec[1]}({spec[2]})") + return + # fn_args = [ + # input, + # offset, + # mask, + # Kh, Kw, stride, stride, Kh // 2, Kw // 2, dilation, dilation, M, D, offset_scale, + # im2col_step, remove_center + # ] + + flash_dcn_fn_args = [ + input1, + offset_mask, + Kh, Kw, stride, stride, Kh // 2, Kw // 2, dilation, dilation, M, D, offset_scale, + im2col_step, remove_center, *additions + ] + + test_args = edict({'warmup_num': 1000, 'test_num': 1000}) + try: + exp_time = speed_test_backward(DCNv4Function.apply, test_args, flash_dcn_fn_args, name='exp') + except: + print(f"Wrong: {N}x{H_in}x{W_in}x{M}x{D} \t {spec[0]}/{spec[1]}({spec[2]})") + return + + torch.cuda.synchronize() + print(f"{N}x{H_in}x{W_in}x{M}x{D} \t {spec[0]}/{spec[1]}({spec[2]}): {exp_time}") + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument("--n", type=int) + parser.add_argument("--h", type=int) + parser.add_argument("--w", type=int) + parser.add_argument("--g", type=int) + parser.add_argument("--c", type=int) + parser.add_argument("--dstride", type=int) + parser.add_argument("--blockthread", type=int) + parser.add_argument("--multiplier", type=int) + args = parser.parse_args() + test(args.n, args.h, args.w, args.g, args.c, (args.dstride, args.blockthread, args.multiplier)) + + diff --git a/DCNv4_op/scripts/search_dcnv4_bwd_engine.py b/DCNv4_op/scripts/search_dcnv4_bwd_engine.py new file mode 100644 index 0000000..25807b8 --- /dev/null +++ b/DCNv4_op/scripts/search_dcnv4_bwd_engine.py @@ -0,0 +1,24 @@ +import os + +def factors(N): + res = [] + for i in range(1, N+1): + if N % i == 0: + res.append(i) + return res + +if __name__ == '__main__': + BATCH=64 + for N, Hin, Win in [(BATCH, 56, 56), (BATCH, 28, 28), (BATCH, 14, 14), (BATCH, 7, 7), + (1, 200, 320), (1, 100, 160), (1, 50, 80), (1, 25, 40), (1, 64, 64)]: + for group_channel in [16, 32, 64]: + for group in [4, 5, 6, 7, 8]: + for d_stride in [1, 2, 4]: + for m in factors(N*Hin*Win): + if m > 64: + break + block_thread = group * (group_channel//d_stride) * m + if block_thread > 1024: + break + cmd = f"python search_dcnv4_bwd.py --n {N} --h {Hin} --w {Win} --g {group} --c {group_channel} --dstride {d_stride} --blockthread {block_thread} --multiplier {m}" + os.system(cmd) \ No newline at end of file diff --git a/DCNv4_op/scripts/search_dcnv4_engine.py b/DCNv4_op/scripts/search_dcnv4_engine.py new file mode 100644 index 0000000..a7a9def --- /dev/null +++ b/DCNv4_op/scripts/search_dcnv4_engine.py @@ -0,0 +1,24 @@ +import os + +def factors(N): + res = [] + for i in range(1, N+1): + if N % i == 0: + res.append(i) + return res + +if __name__ == '__main__': + BATCH=64 + for group_channel in [16, 32, 64]: + for group in [4, 5, 6, 7, 8]: + for N, Hin, Win in [(BATCH, 56, 56), (BATCH, 28, 28), (BATCH, 14, 14), (BATCH, 7, 7), + (1, 200, 320), (1, 100, 160), (1, 50, 80), (1, 25, 40), (1, 64, 64)]: + for d_stride in [2, 4, 8, 16]: + for m in factors(N*Hin*Win): + if m > 64: + break + block_thread = group * (group_channel//d_stride) * m + if block_thread > 1024: + break + cmd = f"python search_dcnv4.py --n {N} --h {Hin} --w {Win} --g {group} --c {group_channel} --dstride {d_stride} --blockthread {block_thread} --multiplier {m}" + os.system(cmd) \ No newline at end of file diff --git a/DCNv4_op/scripts/search_fwd.sh b/DCNv4_op/scripts/search_fwd.sh new file mode 100644 index 0000000..ce541b3 --- /dev/null +++ b/DCNv4_op/scripts/search_fwd.sh @@ -0,0 +1,2 @@ +python search_dcnv4_engine.py > res.txt +python find_best.py --input res.txt --output table.py \ No newline at end of file diff --git a/DCNv4_op/scripts/test_dcnv4.py b/DCNv4_op/scripts/test_dcnv4.py new file mode 100644 index 0000000..1fb8ae9 --- /dev/null +++ b/DCNv4_op/scripts/test_dcnv4.py @@ -0,0 +1,144 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + +from __future__ import absolute_import +from __future__ import print_function +from __future__ import division + +import time +import torch +import torch.nn as nn +import math +from torch.autograd import gradcheck +import pandas as pd +from easydict import EasyDict as edict + +from torch.cuda import Event + +from functions.dcnv3_func import DCNv3Function, dcnv3_core_pytorch +from functions.dcnv4_func import DCNv4Function +torch.set_printoptions(threshold=10000) + +H_in, W_in = 56, 56 +N, M, D = 64, 4, 32 + +# H_in, W_in = 28, 28 +# N, M, D = 64, 8, 32 + +# H_in, W_in = 14, 14 +# N, M, D = 64, 16, 32 + +# H_in, W_in = 7, 7 +# N, M, D = 64, 32, 32 + +# H_in, W_in = 8, 8 +# N, M, D = 128, 4, 16 + + +Kh, Kw = 3, 3 +remove_center = False +P = Kh * Kw - remove_center +offset_scale = 2.0 +pad = 1 +dilation = 1 +stride = 1 +H_out = (H_in + 2 * pad - (dilation * (Kh - 1) + 1)) // stride + 1 +W_out = (W_in + 2 * pad - (dilation * (Kw - 1) + 1)) // stride + 1 + +torch.manual_seed(3) + +#@torch.no_grad() +def speed_test(func, args, inputs, name='Unknown'): + + tic = Event(enable_timing=True) + toc = Event(enable_timing=True) + # warmup + for i in range(args.warmup_num): + func(*inputs) + + total_time = 0 + tic.record() + for i in range(args.test_num): + o = func(*inputs) + torch.cuda.synchronize() + toc.record() + + avg_time = tic.elapsed_time(toc) / args.test_num + print( + f'>>> {name: <10} finished {args.test_num} running, avg_time: {avg_time:.6f} ms') + return avg_time + +@torch.no_grad() +def check_forward_equal_with_pytorch_half(): + input = torch.rand(N, H_in, W_in, M*D).cuda() + print(input.shape) + offset = (torch.rand(N, H_out, W_out, M*P*2).cuda() * 2 - 1)*10 + # offset = (torch.rand(N, H_out, W_out, M*P*2).cuda() * 2 - 1)*0 + mask_origin = torch.rand(N, H_out, W_out, M, P).cuda() + 1e-5 + mask_origin = mask_origin.half() + mask = mask_origin + # mask = torch.nn.functional.softmax(mask_origin, dim=-1) + offset_mask = torch.cat([offset.unflatten(-1, (M, P * 2)), mask_origin.detach()], dim=-1).flatten(-2) + + im2col_step = 128 + + input = input.half() + offset = offset.half() + mask = mask.half() + offset_mask = offset_mask.half() + + dcnv3_args = [ + input, + offset, + mask, + Kh, Kw, stride, stride, Kh // 2, Kw // 2, dilation, dilation, M, D, offset_scale, + im2col_step, remove_center, + ] + output_pytorch = DCNv3Function.apply(*dcnv3_args) + + input1 = input.detach() + + def pad(om): + padded_zero = int(math.ceil(om.shape[3]/8)*8) - om.shape[3] + padded = torch.zeros(om.shape[0], om.shape[1], om.shape[2], padded_zero).to(om) + return torch.cat([om, padded], dim=-1) + + dcnv4_args = [ + input1, pad(offset_mask), + Kh, Kw, stride, stride, Kh // 2, Kw // 2, dilation, dilation, M, D, offset_scale, + im2col_step, remove_center, 8, 512, 2, 256, True, True, + ] + output_flash_cuda = DCNv4Function.apply(*dcnv4_args) + + fwdok = torch.allclose(output_flash_cuda, output_pytorch, rtol=1e-2, atol=1e-3) + max_abs_err = (output_flash_cuda - output_pytorch).abs().max() + max_rel_err = ((output_flash_cuda - output_pytorch).abs() / + (output_pytorch.abs()+ 1e-3)).max() + print('>>> forward half') + print(f'* {fwdok} check_forward_equal_with_pytorch_float: max_abs_err {max_abs_err:.2e} max_rel_err {max_rel_err:.2e}') + assert(fwdok) + + test_args = edict({'warmup_num': 1000, 'test_num': 1000}) + + exp_time_dcnv4 = speed_test(DCNv4Function.apply, test_args, dcnv4_args, name='exp') + exp_time_dcnv3 = speed_test(DCNv3Function.apply, test_args, dcnv3_args, name='exp') + torch.cuda.synchronize() + + results = [{}] + results[0]['dcnv3_time'] = exp_time_dcnv3 + results[0]['dcnv4_time'] = exp_time_dcnv4 + columns = list(results[0].keys()) + + outputs = pd.DataFrame(results, columns=columns) + with pd.option_context( + 'display.max_rows', None, 'display.max_columns', None, + 'display.max_colwidth', None, 'display.width', None, + 'display.precision', 4, ): + print(outputs) + + +if __name__ == '__main__': + check_forward_equal_with_pytorch_half() diff --git a/DCNv4_op/scripts/test_dcnv4_bwd.py b/DCNv4_op/scripts/test_dcnv4_bwd.py new file mode 100644 index 0000000..8b28d29 --- /dev/null +++ b/DCNv4_op/scripts/test_dcnv4_bwd.py @@ -0,0 +1,221 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + +from __future__ import absolute_import +from __future__ import print_function +from __future__ import division + +import time +import torch +import torch.nn as nn +import math +from torch.autograd import gradcheck +import pandas as pd +from easydict import EasyDict as edict + +from torch.cuda import Event + +from functions import DCNv4Function, DCNv3Function +torch.set_printoptions(threshold=10000) + +H_in, W_in = 56, 56 +N, M, D = 64, 4, 32 + +# H_in, W_in = 28, 28 +# N, M, D = 64, 16, 16 + +# H_in, W_in = 14, 14 +# N, M, D = 64, 32, 16 + + +# H_in, W_in = 7, 7 +# N, M, D = 64, 64, 16 + +# H_in, W_in = 8, 8 +# N, M, D = 128, 4, 16 + + +Kh, Kw = 3, 3 +remove_center = False +P = Kh * Kw - remove_center +offset_scale = 2.0 +pad = 1 +dilation = 1 +stride = 1 +H_out = (H_in + 2 * pad - (dilation * (Kh - 1) + 1)) // stride + 1 +W_out = (W_in + 2 * pad - (dilation * (Kw - 1) + 1)) // stride + 1 + +torch.manual_seed(3) + +def speed_test_backward(func, args, inputs, name='Unknown'): + # warmup + # for i in range(args.warmup_num): + # o = func(*inputs) + # o.sum().backward() + + total_time = 0 + len_input = len(inputs) + for i in range(args.warmup_num + args.test_num): + tic = Event(enable_timing=True) + toc = Event(enable_timing=True) + inputs[0] = inputs[0].detach() + inputs[0].requires_grad = True + if len_input > 1 and isinstance(inputs[1], torch.Tensor): + inputs[1] = inputs[1].detach() + inputs[1].requires_grad = True + if len_input > 2 and isinstance(inputs[2], torch.Tensor): + inputs[2] = inputs[2].detach() + inputs[2].requires_grad = True + + o = func(*inputs) + torch.cuda.synchronize() + tic.record() + o.sum().backward() + toc.record() + torch.cuda.synchronize() + _time = tic.elapsed_time(toc) + if i >= args.warmup_num: + total_time += _time + o = o.detach() + + # toc.record() + # torch.cuda.synchronize() + + avg_time = total_time / args.test_num + #print( + # f'>>> {name: <10} finished {args.test_num} running, avg_time: {avg_time:.6f} ms') + return avg_time + +# @torch.no_grad() +def check_forward_equal_with_pytorch_half(): + """ + 64x56x56x128(G=4) + 2 64: 3.66 + - offset_mask collection write 3.4022 + - offset_mask collection 3.1968 + + """ + additions = [8, 128, 2, 256, False] + input = torch.rand(N, H_in, W_in, M*D).cuda() * 10 + print(input.shape) + #offset = torch.rand(N, H_out, W_out, M*P*2).cuda() * 0 + offset = (torch.rand(N, H_out, W_out, M*P*2).cuda() * 2 - 1)*2 + mask_origin = torch.rand(N, H_out, W_out, M, P).cuda() + 1e-5 + mask_origin = mask_origin.half() + mask_origin.requires_grad = True + # offset_mask = torch.cat([offset.unflatten(-1, (M, P, 2)), mask_origin.detach().unsqueeze(-1)], dim=-1).flatten(-3) + # mask /= mask.sum(-1, keepdim=True) + # mask = torch.nn.functional.softmax(mask_origin, dim=-1, dtype=torch.float32) + mask = mask_origin + # mask = mask.reshape(N, H_out, W_out, M*P) + # offset_mask = torch.cat([offset.unflatten(-1, (M, P, 2)), mask.detach().unsqueeze(-1)], dim=-1).flatten(-3) + offset_mask = torch.cat([offset.detach().unflatten(-1, (M, P * 2)), mask_origin.detach()], dim=-1).flatten(-2) + + im2col_step = 128 + + input = input.half() + offset = offset.half() + mask = mask.half() + input.requires_grad = True + offset.requires_grad = True + # mask.requires_grad = True + output_pytorch = DCNv3Function.apply( + input, + offset, + mask, + Kh, Kw, stride, stride, Kh // 2, Kw // 2, dilation, dilation, M, D, offset_scale, + im2col_step, remove_center)#.detach().cpu() + (output_pytorch.sum()/10).backward() + + def pad(om): + padded_zero = int(math.ceil(om.shape[3]/8)*8) - om.shape[3] + padded = torch.zeros(om.shape[0], om.shape[1], om.shape[2], padded_zero).to(om) + return torch.cat([om, padded], dim=-1) + + # value_offset_mask = input.detach() + input1 = input.detach() + input1.requires_grad = True + offset_mask = offset_mask.half() + offset_mask.requires_grad = True + # offset_mask1.requires_grad = True + torch.cuda.profiler.cudart().cudaProfilerStart() + output_flash_cuda = DCNv4Function.apply( + input1, offset_mask, + Kh, Kw, stride, stride, Kh // 2, Kw // 2, dilation, dilation, M, D, offset_scale, + im2col_step, remove_center, *additions)#.detach().cpu() + (output_flash_cuda.sum()/10).backward() + torch.cuda.profiler.cudart().cudaProfilerStop() + + input_grad = input.grad + input2_grad = input1.grad + bwdok = torch.allclose(input_grad.float(), input2_grad.float(), rtol=1e-2, atol=1e-3) + print("bwdok") + print(bwdok) + rel_err = (input_grad.abs() - input2_grad.abs())/(input_grad.abs()+1e-3) + print(rel_err.max()) + + offset_grad1 = offset.grad + offset_grad2 = offset_mask.grad.reshape(N, H_out, W_out, M, P*3)[..., :P*2].reshape(N, H_out, W_out, M*P*2) + # print(offset_grad1) + # print("====================") + # print(offset_grad2) + bwdok2 = torch.allclose(offset_grad1.float(), offset_grad2.float(), rtol=1e-2, atol=1e-3) + print("bwdok2") + print(bwdok2) + rel_err = (offset_grad1 - offset_grad2).abs() / (offset_grad1.abs()+1e-3) + print(rel_err.max()) + + mask_grad1 = mask_origin.grad + mask_grad2 = offset_mask.grad.reshape(N, H_out, W_out, M, P*3)[..., P*2:].reshape(N, H_out, W_out, M, P) + + bwdok3 = torch.allclose(mask_grad1, mask_grad2, rtol=1e-2, atol=1e-3) + print("bwdok3") + print(bwdok3) + rel_err = (mask_grad1 - mask_grad2).abs() / (mask_grad1.abs()+1e-3) + print(rel_err.max()) + + fwdok = torch.allclose(output_flash_cuda, output_pytorch, rtol=1e-2, atol=1e-3) + max_abs_err = (output_flash_cuda - output_pytorch).abs().max() + max_rel_err = ((output_flash_cuda - output_pytorch).abs() / + (output_pytorch.abs()+ 1e-3)).max() + print('>>> forward half') + print(f'* {fwdok} check_forward_equal_with_pytorch_float: max_abs_err {max_abs_err:.2e} max_rel_err {max_rel_err:.2e}') + fn_args = [ + input, + offset, + mask, + Kh, Kw, stride, stride, Kh // 2, Kw // 2, dilation, dilation, M, D, offset_scale, + im2col_step, remove_center + ] + + flash_dcn_fn_args = [ + input1, + offset_mask, + Kh, Kw, stride, stride, Kh // 2, Kw // 2, dilation, dilation, M, D, offset_scale, + im2col_step, remove_center, *additions + ] + + test_args = edict({'warmup_num': 1000, 'test_num': 1000}) + exp_time = speed_test_backward( + DCNv4Function.apply, test_args, flash_dcn_fn_args, name='exp') + exp_time_base = speed_test_backward( + DCNv3Function.apply, test_args, fn_args, name='exp') + + results = [{}] + results[0]['time'] = exp_time + results[0]['time_base'] = exp_time_base + columns = list(results[0].keys()) + + outputs = pd.DataFrame(results, columns=columns) + with pd.option_context( + 'display.max_rows', None, 'display.max_columns', None, + 'display.max_colwidth', None, 'display.width', None, + 'display.precision', 4, ): + print(outputs) + + +if __name__ == '__main__': + check_forward_equal_with_pytorch_half() \ No newline at end of file diff --git a/DCNv4_op/scripts/test_flash_deform_attn.py b/DCNv4_op/scripts/test_flash_deform_attn.py new file mode 100644 index 0000000..91cfb78 --- /dev/null +++ b/DCNv4_op/scripts/test_flash_deform_attn.py @@ -0,0 +1,174 @@ +# ------------------------------------------------------------------------------------------------ +# Deformable DETR +# Copyright (c) 2020 SenseTime. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------------------------------ +# Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +# ------------------------------------------------------------------------------------------------ + +from __future__ import absolute_import +from __future__ import print_function +from __future__ import division +from easydict import EasyDict as edict +from torch.cuda import Event +import pandas as pd + +import time +import torch +import torch.nn as nn +from torch.autograd import gradcheck + +from functions import MSDeformAttnFunction, FlashDeformAttnFunction, ms_deform_attn_core_pytorch + + +# N, M, D = 1, 4, 8 +# # Lq, L, P = 2, 2, 2 +# # shapes = torch.as_tensor([(6, 4), (3, 2)], dtype=torch.long).cuda() +# Lq, L, P = 1, 2, 8 +# shapes = torch.as_tensor([(8, 16), (4, 8)], dtype=torch.long).cuda() + +# N, M, D = 1, 8, 32 +# # Lq, L, P = 2, 2, 2 +# # shapes = torch.as_tensor([(6, 4), (3, 2)], dtype=torch.long).cuda() +# Lq, L, P = 300, 4, 4 +# # shapes = torch.as_tensor([(134, 151), (67, 76), (34, 38), (17, 19)], dtype=torch.long).cuda() +# # shapes = torch.as_tensor([(134, 151), (67, 76), (34, 38), (16, 16)], dtype=torch.long).cuda() +# # shapes = torch.as_tensor([(134, 151), (67, 76), (34, 38), (17, 19)], dtype=torch.long).cuda() +# # shapes = torch.as_tensor([(17, 19), (4, 4)], dtype=torch.long).cuda() +# shapes = torch.as_tensor([(100, 151), (50, 76), (25, 38), (13, 19)], dtype=torch.long).cuda() +# # shapes = torch.as_tensor([(110, 151)], dtype=torch.long).cuda() + +# B:6 +# H:232 +# W:400 +# G:5 +# D: 16 +# channels: 80 +# kernel: 3 points = 3 * 3 +# num_split = 45 = kernel *kernel * G + +H = 256 +W = 256 +N, M, D = 1, 8, 32 +Lq, L, P = 100*152, 4, 8 + +shapes = torch.Tensor([[100, 152], [ 50, 76], [ 25, 38], [ 13, 19]]).long().cuda() + +# x = x.reshape([B, H*W, G, D + self.num_split * 3]) +# shapes = torch.as_tensor([(H, W)], dtype=torch.long).cuda() +# shapes = torch.as_tensor([(H, W), (H // 2, W // 2)], dtype=torch.long).cuda() +# shapes = torch.as_tensor([(H, W), (H // 2, W // 2), (H // 4, W // 4), (H // 8, W // 8)], dtype=torch.long).cuda() + +level_start_index = torch.cat((shapes.new_zeros((1,)), shapes.prod(1).cumsum(0)[:-1])) +S = sum([(H * W).item() for H, W in shapes]) +print(S) + +def get_reference_points(spatial_shapes, device): + reference_points_list = [] + for lvl, (H_, W_) in enumerate(spatial_shapes): + + ref_y, ref_x = torch.meshgrid(torch.linspace(0.5, H_ - 0.5, H_, dtype=torch.float32, device=device), + torch.linspace(0.5, W_ - 0.5, W_, dtype=torch.float32, device=device)) + ref_y = ref_y.reshape(-1)[None] / (H_) + ref_x = ref_x.reshape(-1)[None] / (W_) + ref = torch.stack((ref_x, ref_y), -1) + reference_points_list.append(ref) + reference_points = torch.cat(reference_points_list, 1) + # reference_points = reference_points[:, :, None] * valid_ratios[:, None] + return reference_points + + +torch.manual_seed(3) + +@torch.no_grad() +def speed_test(func, args, inputs, name='Unknown'): + + tic = Event(enable_timing=True) + toc = Event(enable_timing=True) + # warmup + for i in range(args.warmup_num): + func(*inputs) + + tic.record() + for i in range(args.test_num): + func(*inputs) + toc.record() + torch.cuda.synchronize() + + avg_time = tic.elapsed_time(toc) / args.test_num + print( + f'>>> {name: <10} finished {args.test_num} running, avg_time: {avg_time:.6f} ms') + return avg_time + + +@torch.no_grad() +def check_forward_equal_with_pytorch_half(): + value = torch.rand(N, S, M, D).cuda() * 0.01 + # offset = (torch.rand(N, Lq, M, L, P, 2).cuda() * 2 - 1) / 10 + sampling_locations = torch.rand(N, Lq, M, L, P, 2).cuda() + attention_weights = torch.rand(N, Lq, M, L, P).cuda() + 1e-5 + sampling_loc_attn = torch.cat([sampling_locations.reshape(N, Lq, M, L*P*2), attention_weights.reshape(N, Lq, M, L*P)], dim=-1) + attention_weights = torch.nn.functional.softmax(attention_weights.flatten(-2, -1), dim=-1).unflatten(-1, (L, P)) + + + im2col_step = 128 + + flash_fn_args = ( + value.half(), + shapes, + level_start_index, + sampling_loc_attn.half(), + im2col_step, + P, 16 + ) + output_cuda = ( + FlashDeformAttnFunction.apply(*flash_fn_args) + .detach() + .cpu() + ).double() + + fn_args = ( + value, + shapes, + level_start_index, + sampling_locations, + attention_weights, + im2col_step, + ) + + output_pytorch = ( + MSDeformAttnFunction.apply(*fn_args) + .detach().double() + .cpu() + ) + + max_abs_err = (output_cuda - output_pytorch).abs().max() + max_rel_err = ((output_cuda - output_pytorch).abs() / output_pytorch.abs()).max() + fwdok = torch.allclose(output_cuda, output_pytorch, rtol=1e-2, atol=1e-3) + + print( + f"* {fwdok} check_forward_equal_with_pytorch_float: max_abs_err {max_abs_err:.2e} max_rel_err {max_rel_err:.2e}" + ) + + test_args = edict({'warmup_num': 1000, 'test_num': 1000}) + exp_time_base = speed_test( + MSDeformAttnFunction.apply, test_args, fn_args, name='exp') + exp_time = speed_test( + FlashDeformAttnFunction.apply, test_args, flash_fn_args, name='exp') + + results = [{}] + results[0]['time'] = exp_time + results[0]['time_base'] = exp_time_base + columns = list(results[0].keys()) + + outputs = pd.DataFrame(results, columns=columns) + with pd.option_context( + 'display.max_rows', None, 'display.max_columns', None, + 'display.max_colwidth', None, 'display.width', None, + 'display.precision', 4, ): + print(outputs) + + +if __name__ == "__main__": + check_forward_equal_with_pytorch_half() + diff --git a/DCNv4_op/scripts/test_flash_deform_attn_backward.py b/DCNv4_op/scripts/test_flash_deform_attn_backward.py new file mode 100644 index 0000000..711c977 --- /dev/null +++ b/DCNv4_op/scripts/test_flash_deform_attn_backward.py @@ -0,0 +1,194 @@ +# ------------------------------------------------------------------------------------------------ +# Deformable DETR +# Copyright (c) 2020 SenseTime. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------------------------------ +# Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +# ------------------------------------------------------------------------------------------------ + +from __future__ import absolute_import +from __future__ import print_function +from __future__ import division +from easydict import EasyDict as edict +from torch.cuda import Event +import pandas as pd + +import time +import torch +import torch.nn as nn +from torch.autograd import gradcheck + +from functions import MSDeformAttnFunction, ms_deform_attn_core_pytorch, FlashDeformAttnFunction + + +H = 256 +W = 256 +N, M, D = 1, 8, 16 +Lq, L, P = H * W, 1, 8 + +# x = x.reshape([B, H*W, G, D + self.num_split * 3]) +shapes = torch.as_tensor([(H, W)], dtype=torch.long).cuda() +# shapes = torch.as_tensor([(H, W), (H // 2, W // 2)], dtype=torch.long).cuda() +# shapes = torch.as_tensor([(H, W), (H // 2, W // 2), (H // 4, W // 4), (H // 8, W // 8)], dtype=torch.long).cuda() + +H = 256 +W = 256 +N, M, D = 1, 8, 32 +Lq, L, P = 100*152, 4, 8 + +shapes = torch.Tensor([[100, 152], [ 50, 76], [ 25, 38], [ 13, 19]]).long().cuda() + +level_start_index = torch.cat((shapes.new_zeros((1,)), shapes.prod(1).cumsum(0)[:-1])) +S = sum([(H * W).item() for H, W in shapes]) + +def get_reference_points(spatial_shapes, device): + reference_points_list = [] + for lvl, (H_, W_) in enumerate(spatial_shapes): + + ref_y, ref_x = torch.meshgrid(torch.linspace(0.5, H_ - 0.5, H_, dtype=torch.float32, device=device), + torch.linspace(0.5, W_ - 0.5, W_, dtype=torch.float32, device=device)) + ref_y = ref_y.reshape(-1)[None] / (H_) + ref_x = ref_x.reshape(-1)[None] / (W_) + ref = torch.stack((ref_x, ref_y), -1) + reference_points_list.append(ref) + reference_points = torch.cat(reference_points_list, 1) + # reference_points = reference_points[:, :, None] * valid_ratios[:, None] + return reference_points + + +torch.manual_seed(3) + +@torch.no_grad() +def speed_test(func, args, inputs, name='Unknown'): + + tic = Event(enable_timing=True) + toc = Event(enable_timing=True) + # warmup + for i in range(args.warmup_num): + func(*inputs) + + tic.record() + for i in range(args.test_num): + func(*inputs) + toc.record() + torch.cuda.synchronize() + + avg_time = tic.elapsed_time(toc) / args.test_num + print( + f'>>> {name: <10} finished {args.test_num} running, avg_time: {avg_time:.6f} ms') + return avg_time + + +def check_forward_equal_with_pytorch_half(): + value = torch.rand(N, S, M, D).cuda() * 0.01 + offset = (torch.rand(N, Lq, M, L, P, 2).cuda() * 2 - 1) / 10 + sampling_locations = torch.rand(N, Lq, M, L, P, 2).cuda() + attention_weights_origin = torch.rand(N, Lq, M, L, P).cuda() + 1e-5 + attention_weights_origin.requires_grad = True + sampling_loc_attn = torch.cat([sampling_locations.detach().reshape(N, Lq, M, L*P*2), attention_weights_origin.detach().reshape(N, Lq, M, L*P)], dim=-1) + + attention_weights = torch.nn.functional.softmax(attention_weights_origin.flatten(-2, -1), dim=-1).unflatten(-1, (L, P)) + + + im2col_step = 128 + + value.requires_grad = True + sampling_loc_attn.requires_grad = True + output_cuda = ( + FlashDeformAttnFunction.apply( + value.float(), + shapes, + level_start_index, + sampling_loc_attn.float(), + im2col_step, + ) + ) + (output_cuda.float().sum()/10).backward() + + + value1 = value.detach() + value1.requires_grad = True + sampling_locations.requires_grad = True + #attention_weights.requires_grad = True + output_pytorch = ( + ms_deform_attn_core_pytorch(value1, shapes, sampling_locations, attention_weights) + ) + (output_pytorch.sum()/10).backward() + + max_abs_err = (output_cuda.float() - output_pytorch).abs().max() + max_rel_err = ((output_cuda.float() - output_pytorch).abs() / output_pytorch.abs()).max() + fwdok = torch.allclose(output_cuda.float(), output_pytorch, rtol=1e-2, atol=1e-3) + print(fwdok) + print(max_abs_err, max_rel_err) + #exit() + + bwdok1 = torch.allclose(value.grad, value1.grad, rtol=1e-2, atol=1e-3) + print(bwdok1) + # rel_err = (sampling_locations.grad - sampling_loc_attn.grad[..., :L*P*2].reshape(*sampling_locations.shape)).abs()/(sampling_locations.grad.abs()+1e-3) + # print(rel_err.max()) + + locgrad1 = sampling_locations.grad + locgrad2 = sampling_loc_attn.grad[..., :L*P*2].reshape(*sampling_locations.shape) + bwdok2 = torch.allclose(locgrad1, locgrad2, rtol=1e-2, atol=1e-3) + print(bwdok2) + rel_err = (locgrad1 - locgrad2).abs()/(locgrad1.abs()+1e-3) + print(rel_err.max()) + + attngrad1 = attention_weights_origin.grad + attngrad2 = sampling_loc_attn.grad[..., L*P*2:].reshape(*attention_weights_origin.shape) + bwdok3 = torch.allclose(locgrad1, locgrad2, rtol=1e-2, atol=1e-3) + print(bwdok3) + rel_err = (attngrad1 - attngrad2).abs()/(attngrad1.abs()+1e-3) + print(rel_err.max()) + exit() + #exit() + + # pdb.set_trace() + max_abs_err = (output_cuda - output_pytorch).abs().max() + max_rel_err = ((output_cuda - output_pytorch).abs() / output_pytorch.abs()).max() + fwdok = torch.allclose(output_cuda, output_pytorch, rtol=1e-2, atol=1e-3) + + print( + f"* {fwdok} check_forward_equal_with_pytorch_float: max_abs_err {max_abs_err:.2e} max_rel_err {max_rel_err:.2e}" + ) + + + fn_args = ( + value, + shapes, + level_start_index, + sampling_locations, + attention_weights, + im2col_step, + ) + + flash_dcn_fn_args = ( + value.half(), + shapes, + level_start_index, + sampling_loc_attn.half(), + im2col_step, + ) + + + test_args = edict({'warmup_num': 50, 'test_num': 100}) + exp_time = speed_test( + FlashMSDeformAttnFunction.apply, test_args, flash_dcn_fn_args, name='exp') + exp_time_base = speed_test( + MSDeformAttnFunction.apply, test_args, fn_args, name='exp') + + results = [{}] + results[0]['time'] = exp_time + results[0]['time_base'] = exp_time_base + columns = list(results[0].keys()) + + outputs = pd.DataFrame(results, columns=columns) + with pd.option_context( + 'display.max_rows', None, 'display.max_columns', None, + 'display.max_colwidth', None, 'display.width', None, + 'display.precision', 4, ): + print(outputs) + + +if __name__ == "__main__": + check_forward_equal_with_pytorch_half() \ No newline at end of file diff --git a/DCNv4_op/setup.py b/DCNv4_op/setup.py new file mode 100644 index 0000000..1fa8531 --- /dev/null +++ b/DCNv4_op/setup.py @@ -0,0 +1,72 @@ +# ------------------------------------------------------------------------------------------------ +# Deformable Convolution v4 +# Copyright (c) 2024 OpenGVLab +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------------------------------ +# Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +# ------------------------------------------------------------------------------------------------ + +import os +import glob + +import torch + +from torch.utils.cpp_extension import CUDA_HOME +from torch.utils.cpp_extension import CppExtension +from torch.utils.cpp_extension import CUDAExtension + +from setuptools import find_packages +from setuptools import setup + +requirements = ["torch", "torchvision"] + +def get_extensions(): + this_dir = os.path.dirname(os.path.abspath(__file__)) + extensions_dir = os.path.join(this_dir, "src") + + main_file = glob.glob(os.path.join(extensions_dir, "*.cpp")) + source_cpu = glob.glob(os.path.join(extensions_dir, "cpu", "*.cpp")) + source_cuda = glob.glob(os.path.join(extensions_dir, "cuda", "*.cu")) + + sources = main_file + source_cpu + extension = CppExtension + extra_compile_args = {"cxx": []} + define_macros = [] + + if torch.cuda.is_available() and CUDA_HOME is not None: + extension = CUDAExtension + sources += source_cuda + define_macros += [("WITH_CUDA", None)] + extra_compile_args["nvcc"] = [ + "-DCUDA_HAS_FP16=1", + "-D__CUDA_NO_HALF_OPERATORS__", + "-D__CUDA_NO_HALF_CONVERSIONS__", + "-D__CUDA_NO_HALF2_OPERATORS__", + "-O3", + ] + else: + raise NotImplementedError('Cuda is not available') + + sources = [os.path.join(extensions_dir, s) for s in sources] + include_dirs = [extensions_dir] + ext_modules = [ + extension( + "DCNv4.ext", + sources, + include_dirs=include_dirs, + define_macros=define_macros, + extra_compile_args=extra_compile_args, + ) + ] + return ext_modules + +setup( + name="DCNv4", + version="1.0.0", + author="Yuwen Xiong, Feng Wang", + url="", + description="PyTorch Wrapper for CUDA Functions of DCNv4", + packages=['DCNv4', 'DCNv4/functions', 'DCNv4/modules'], + ext_modules=get_extensions(), + cmdclass={"build_ext": torch.utils.cpp_extension.BuildExtension}, +) diff --git a/DCNv4_op/src/cuda/common.h b/DCNv4_op/src/cuda/common.h new file mode 100644 index 0000000..bf8f803 --- /dev/null +++ b/DCNv4_op/src/cuda/common.h @@ -0,0 +1,212 @@ +#ifndef FMSDACOMMON +#define FMSDACOMMON +#include +#include +#include + +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +constexpr int kWarpSize = 32; +#define opmath_t at::opmath_type + +inline int GET_BLOCKS(const int N, const int num_threads) { + return (N + num_threads - 1) / num_threads; +} + +#define CUDA_KERNEL_LOOP(i, n) \ + for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < (n); \ + i += blockDim.x * gridDim.x) + +inline bool check_backward_warpp(int d_stride, int D){ + int n_group_threads = D / d_stride; + return (n_group_threads <= kWarpSize) && (kWarpSize % n_group_threads == 0); +} + +template +__device__ void ms_deform_attn_im2col_bilinear( + opmath_t out_reg_array[], const scalar_t *&p_value, const int &height, + const int &width, const opmath_t &h_px, const opmath_t &w_px, + const opmath_t &attn, const int &w_stride, const int &base_ptr) { + + const int h_low = floor(h_px); + const int w_low = floor(w_px); + const int h_high = h_low + 1; + const int w_high = w_low + 1; + const opmath_t lh = h_px - h_low; + const opmath_t lw = w_px - w_low; + const opmath_t hh = 1 - lh; + const opmath_t hw = 1 - lw; + + const opmath_t w1 = hh * hw, w2 = hh * lw, w3 = lh * hw, w4 = lh * lw; + + const int h_stride = width * w_stride; + const int h_low_ptr_offset = h_low * h_stride; + const int h_high_ptr_offset = h_low_ptr_offset + h_stride; + const int w_low_ptr_offset = w_low * w_stride; + const int w_high_ptr_offset = w_low_ptr_offset + w_stride; + + int idx1 = h_low_ptr_offset + w_low_ptr_offset + base_ptr; + int idx2 = h_low_ptr_offset + w_high_ptr_offset + base_ptr; + int idx3 = h_high_ptr_offset + w_low_ptr_offset + base_ptr; + int idx4 = h_high_ptr_offset + w_high_ptr_offset + base_ptr; + + scalar_t v1_array[c_per_thread] = {0.}; + scalar_t v2_array[c_per_thread] = {0.}; + scalar_t v3_array[c_per_thread] = {0.}; + scalar_t v4_array[c_per_thread] = {0.}; + + if (h_low >= 0 && w_low >= 0) { + auto p1 = p_value + idx1; + *(transfer_t *)(v1_array) = *(transfer_t *)(p1); + } + + if (h_low >= 0 && w_high < width) { + auto p2 = p_value + idx2; + *(transfer_t *)(v2_array) = *(transfer_t *)(p2); + } + if (h_high < height && w_low >= 0) { + auto p3 = p_value + idx3; + *(transfer_t *)(v3_array) = *(transfer_t *)(p3); + } + if (h_high < height && w_high < width) { + auto p4 = p_value + idx4; + *(transfer_t *)(v4_array) = *(transfer_t *)(p4); + } +#pragma unroll + for (int i = 0; i < c_per_thread; i++) { + out_reg_array[i] += + (opmath_t)attn * + (w1 * (opmath_t)v1_array[i] + w2 * (opmath_t)v2_array[i] + + w3 * (opmath_t)v3_array[i] + w4 * (opmath_t)v4_array[i]); + } +} + +template +__device__ void ms_deform_attn_col2im_bilinear( + const scalar_t *&p_value, const int &height, const int &width, + const opmath_t &h_px, const opmath_t &w_px, const opmath_t &attn, + const int &w_stride, const int &base_ptr, const opmath_t offset_scale_h, + const opmath_t offset_scale_w, const scalar_t *&top_grad, + opmath_t *&grad_im, opmath_t *grad_offset) { + + const int h_low = floor(h_px); + const int w_low = floor(w_px); + const int h_high = h_low + 1; + const int w_high = w_low + 1; + const opmath_t lh = h_px - h_low; + const opmath_t lw = w_px - w_low; + const opmath_t hh = 1 - lh; + const opmath_t hw = 1 - lw; + + const opmath_t w1 = hh * hw, w2 = hh * lw, w3 = lh * hw, w4 = lh * lw; + + scalar_t _top_grad_array[c_per_thread] = {0.}; + *(transfer_t *)(_top_grad_array) = *(transfer_t *)(top_grad); + + opmath_t top_grad_array[c_per_thread] = {0.}; + for (int i = 0; i < c_per_thread; ++i) { + top_grad_array[i] = (opmath_t)(_top_grad_array[i]); + } + + const int h_stride = width * w_stride; + const int h_low_ptr_offset = h_low * h_stride; + const int h_high_ptr_offset = h_low_ptr_offset + h_stride; + const int w_low_ptr_offset = w_low * w_stride; + const int w_high_ptr_offset = w_low_ptr_offset + w_stride; + + int idx1 = h_low_ptr_offset + w_low_ptr_offset + base_ptr; + int idx2 = h_low_ptr_offset + w_high_ptr_offset + base_ptr; + int idx3 = h_high_ptr_offset + w_low_ptr_offset + base_ptr; + int idx4 = h_high_ptr_offset + w_high_ptr_offset + base_ptr; + + scalar_t v1_array[c_per_thread] = {0.}; + scalar_t v2_array[c_per_thread] = {0.}; + scalar_t v3_array[c_per_thread] = {0.}; + scalar_t v4_array[c_per_thread] = {0.}; + + opmath_t grad_h_weight[c_per_thread] = {0.}; + opmath_t grad_w_weight[c_per_thread] = {0.}; + + if (h_low >= 0 && w_low >= 0) { + auto p1 = p_value + idx1; + *(transfer_t *)(v1_array) = *(transfer_t *)(p1); +#pragma unroll + for (int i = 0; i < c_per_thread; ++i) { + grad_h_weight[i] -= hw * v1_array[i]; + grad_w_weight[i] -= hh * v1_array[i]; + atomicAdd(grad_im + idx1 + i, top_grad_array[i] * attn * w1); + } + } + + if (h_low >= 0 && w_high < width) { + auto p2 = p_value + idx2; + *(transfer_t *)(v2_array) = *(transfer_t *)(p2); +#pragma unroll + for (int i = 0; i < c_per_thread; ++i) { + grad_h_weight[i] -= lw * v2_array[i]; + grad_w_weight[i] += hh * v2_array[i]; + atomicAdd(grad_im + idx2 + i, top_grad_array[i] * attn * w2); + } + } + if (h_high < height && w_low >= 0) { + auto p3 = p_value + idx3; + *(transfer_t *)(v3_array) = *(transfer_t *)(p3); +#pragma unroll + for (int i = 0; i < c_per_thread; ++i) { + grad_h_weight[i] += hw * v3_array[i]; + grad_w_weight[i] -= lh * v3_array[i]; + atomicAdd(grad_im + idx3 + i, top_grad_array[i] * attn * w3); + } + } + if (h_high < height && w_high < width) { + auto p4 = p_value + idx4; + *(transfer_t *)(v4_array) = *(transfer_t *)(p4); +#pragma unroll + for (int i = 0; i < c_per_thread; ++i) { + grad_h_weight[i] += lw * v4_array[i]; + grad_w_weight[i] += lh * v4_array[i]; + atomicAdd(grad_im + idx4 + i, top_grad_array[i] * attn * w4); + } + } + + opmath_t _grad_offset_x = 0; + opmath_t _grad_offset_y = 0; +#pragma unroll + for (int i = 0; i < c_per_thread; ++i) { + _grad_offset_x += + grad_w_weight[i] * top_grad_array[i]; // channel aware term + _grad_offset_y += + grad_h_weight[i] * top_grad_array[i]; // channel aware term + } + _grad_offset_x *= (offset_scale_w * attn); // channel shared term + _grad_offset_y *= (offset_scale_h * attn); // channel shared term + + *grad_offset = _grad_offset_x; + *(grad_offset + 1) = _grad_offset_y; + + opmath_t current_val; + opmath_t _grad_offset_z = 0; +#pragma unroll + for (int i = 0; i < c_per_thread; i++) { + current_val = (opmath_t)(w1 * v1_array[i] + w2 * v2_array[i] + + w3 * v3_array[i] + w4 * v4_array[i]); + _grad_offset_z += current_val * top_grad_array[i]; + } + *(grad_offset + 2) = _grad_offset_z; +} + + + +#endif \ No newline at end of file diff --git a/DCNv4_op/src/cuda/dcnv4_col2im_cuda.cuh b/DCNv4_op/src/cuda/dcnv4_col2im_cuda.cuh new file mode 100644 index 0000000..78bb8dc --- /dev/null +++ b/DCNv4_op/src/cuda/dcnv4_col2im_cuda.cuh @@ -0,0 +1,561 @@ +#include +#include +#include + +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include "common.h" + +template +__global__ void backward_kernel_dcn( + const scalar_t *p_value, const scalar_t *p_offset, + const scalar_t *grad_output, const int G, const int D, const int Q, + const int kernel_h, const int kernel_w, const int stride_h, + const int stride_w, const int pad_h, const int pad_w, const int dilation_h, + const int dilation_w, const int height_in, const int width_in, + const int height_out, const int width_out, const opmath_t offset_scale, + const int remove_center, const int block_multiplier, opmath_t *grad_im, + opmath_t *grad_offset, const int padded_offset_dim) { + + extern __shared__ char _s[]; + + const int &qi = (blockIdx.x * block_multiplier % Q) + threadIdx.z; + const int &bi = blockIdx.x * block_multiplier / Q; + + const int &di_s = threadIdx.x * d_stride; + const int &gi = threadIdx.y; + constexpr int li = 0; + + opmath_t *const cache_g_mask_before_softmax = (opmath_t *)(_s); // mG x K + opmath_t *const cache_grad_offset = + (opmath_t *)(cache_g_mask_before_softmax + + block_multiplier * G * K); // mG x blockDim.x x 3 + opmath_t *const p_mask_shm = + (opmath_t *)(cache_grad_offset + block_multiplier * G * blockDim.x * 3) + + (threadIdx.z * G + gi) * K; + + const scalar_t *p_offset_ptr = p_offset + (bi*Q + qi)*padded_offset_dim + gi*K*3; + + const int mask_length = K; + const int num_thread = (D / d_stride); + const int num_iter = mask_length / num_thread; + const int remainder = mask_length - num_iter * num_thread; + + const scalar_t *top_grad = grad_output + ((bi * Q + qi) * G + gi) * D + di_s; + + __syncthreads(); + for (int i = 0; i < num_iter; i++) { + *(p_mask_shm + num_thread * i + threadIdx.x) = + *(scalar_t *)(p_offset_ptr + K * 2 + num_thread * i + threadIdx.x); + } + if (remainder > 0 && threadIdx.x < remainder) { + *(p_mask_shm + num_thread * num_iter + threadIdx.x) = *( + scalar_t *)(p_offset_ptr + K * 2 + num_thread * num_iter + threadIdx.x); + } + + if (softmax) { + __syncthreads(); + // transfer offset from global memory to shared memory > + + // Calculate softmax over L and K + if (threadIdx.x == 0) { // gi != 0, di = 0, li = 0 + opmath_t softmax_max = -1e100; + opmath_t softmax_sum = 0.0; + + // get max + for (int j = 0; j < K; j++) { + softmax_max = max(softmax_max, p_mask_shm[j]); + } + + // get sumexp + for (int j = 0; j < K; j++) { + opmath_t exp_results = exp(p_mask_shm[j] - softmax_max); + p_mask_shm[j] = exp_results; + softmax_sum += exp_results; + } + + // normalize + for (int j = 0; j < K; j++) { + p_mask_shm[j] /= softmax_sum; + } + } + + __syncthreads(); + } + + int offset_idx = 0; + int mask_idx = 0; + + const int w_stride = G * D; + const int base_ptr = gi * D + di_s; + const scalar_t *p_value_ptr = + p_value + (bi * (height_in * width_in)) * (G * D); + opmath_t *grad_im_ptr = grad_im + (bi * (height_in * width_in)) * (G * D); + + const int p0_w = ((dilation_w * (kernel_w - 1)) >> 1) - pad_w + + (qi % width_out) * stride_w; + const int p0_h = ((dilation_h * (kernel_h - 1)) >> 1) - pad_h + + (qi / width_out) * stride_h; + const opmath_t p0_w_ = + p0_w - ((dilation_w * (kernel_w - 1)) >> 1) * offset_scale; + const opmath_t p0_h_ = + p0_h - ((dilation_h * (kernel_h - 1)) >> 1) * offset_scale; + const int center_h = kernel_h / 2; + const int center_w = kernel_w / 2; + + grad_offset += (bi*Q + qi)*padded_offset_dim + gi*K*3; + opmath_t *grad_offset_softmax = grad_offset + K * 2; + + int cache_grad_off_idx = + ((threadIdx.z * G + threadIdx.y) * blockDim.x + threadIdx.x) * 3; + for (int i = 0; i < kernel_w; ++i) { + for (int j = 0; j < kernel_h; ++j) { + if (i != center_w || j != center_h || !remove_center) { + const opmath_t w_im = + p0_w_ + (i * dilation_w + (opmath_t)p_offset_ptr[offset_idx]) * + offset_scale; + const opmath_t h_im = + p0_h_ + (j * dilation_h + (opmath_t)p_offset_ptr[offset_idx + 1]) * + offset_scale; + const opmath_t attn = p_mask_shm[mask_idx]; + cache_grad_offset[cache_grad_off_idx] = 0; + cache_grad_offset[cache_grad_off_idx + 1] = 0; + cache_grad_offset[cache_grad_off_idx + 2] = 0; + + if (h_im > -1 && w_im > -1 && h_im < height_in && w_im < width_in) { + ms_deform_attn_col2im_bilinear( + p_value_ptr, height_in, width_in, h_im, w_im, attn, w_stride, + base_ptr, offset_scale, offset_scale, top_grad, grad_im_ptr, + cache_grad_offset + cache_grad_off_idx); + } + + // aggregated across different channel for offset + + __syncthreads(); + if (threadIdx.x == 0) { // + int _didx = (threadIdx.z * G + threadIdx.y) * blockDim.x * 3; + opmath_t _grad_w = cache_grad_offset[_didx], + _grad_h = cache_grad_offset[_didx + 1], + _grad_a = cache_grad_offset[_didx + 2]; + + for (int c_id = 1; c_id < blockDim.x; ++c_id) { + _grad_w += cache_grad_offset[_didx + 3 * c_id]; + _grad_h += cache_grad_offset[_didx + 3 * c_id + 1]; + _grad_a += cache_grad_offset[_didx + 3 * c_id + 2]; + } + + *(grad_offset) = _grad_w; // B x H x W x G x L x K x 3 + *(grad_offset + 1) = _grad_h; // B x H x W x G x L x K x 3 + if (softmax) { + cache_g_mask_before_softmax[(threadIdx.z * G + threadIdx.y) * K + + mask_idx] = _grad_a * attn; + } + else{ + grad_offset_softmax[mask_idx] = _grad_a; + } + } + __syncthreads(); + + offset_idx += 2; + mask_idx += 1; + grad_offset += 2; + } + } + } + // backward for softmax + if(softmax){ + if (threadIdx.x == 0) { + const opmath_t* group_g_mask = cache_g_mask_before_softmax + (threadIdx.z*G + threadIdx.y)*K; + #pragma unroll + for (int i = 0; i < K; ++i) { + opmath_t sum = 0.; + for (int j = 0; j < K; ++j) { + sum += group_g_mask[j]; // dL/di * di/dj + } + *(grad_offset_softmax) = group_g_mask[i] - p_mask_shm[i] * sum; + + grad_offset_softmax += 1; + } + } + __syncthreads(); + } +} + +template +__global__ void backward_kernel_dcn_warp_primitive( + const scalar_t *p_value, const scalar_t *p_offset, + const scalar_t *grad_output, const int G, const int D, const int Q, + const int kernel_h, const int kernel_w, const int stride_h, + const int stride_w, const int pad_h, const int pad_w, const int dilation_h, + const int dilation_w, const int height_in, const int width_in, + const int height_out, const int width_out, const opmath_t offset_scale, + const int remove_center, const int block_multiplier, opmath_t *grad_im, + opmath_t *grad_offset, const int padded_offset_dim) { + + extern __shared__ char _s[]; + + const int &qi = (blockIdx.x * block_multiplier % Q) + threadIdx.z; + const int &bi = blockIdx.x * block_multiplier / Q; + + + const int &di_s = threadIdx.x * d_stride; + const int &gi = threadIdx.y; + + constexpr int li = 0; + const int tid = (threadIdx.z * blockDim.y + threadIdx.y)*blockDim.x + threadIdx.x; + + const int lane_id = tid % kWarpSize; + + // find the position of current group in the current warp + const int group_per_warp = kWarpSize / blockDim.x; + const int group_in_warp_id = (threadIdx.z * G + threadIdx.y) % group_per_warp; + const unsigned lane_mask = ((1 << blockDim.x) - 1) << (group_in_warp_id * blockDim.x); + + opmath_t *const p_mask_shm = (opmath_t *)(_s) + (threadIdx.z * G + gi) * K; + opmath_t *cache_g_mask_before_softmax = (opmath_t *)((opmath_t *)(_s) + block_multiplier * G * K) + + (threadIdx.z*G+gi)*K; // only used by threadIdx.x = 0 + + const scalar_t *p_offset_ptr = p_offset + (bi*Q + qi)*padded_offset_dim + gi*K*3; + + const int mask_length = K; + const int num_thread = (D / d_stride); + const int num_iter = mask_length / num_thread; + const int remainder = mask_length - num_iter * num_thread; + + const scalar_t *top_grad = grad_output + ((bi * Q + qi) * G + gi) * D + di_s; + + __syncthreads(); + for (int i = 0; i < num_iter; i++) { + *(p_mask_shm + num_thread * i + threadIdx.x) = + *(scalar_t *)(p_offset_ptr + K * 2 + num_thread * i + threadIdx.x); + } + if (remainder > 0 && threadIdx.x < remainder) { + *(p_mask_shm + num_thread * num_iter + threadIdx.x) = *( + scalar_t *)(p_offset_ptr + K * 2 + num_thread * num_iter + threadIdx.x); + } + + if (softmax) { + __syncthreads(); + // transfer offset from global memory to shared memory > + + // Calculate softmax over L and K + if (threadIdx.x == 0) { // gi != 0, di = 0, li = 0 + opmath_t softmax_max = -1e100; + opmath_t softmax_sum = 0.0; + + // get max + for (int j = 0; j < K; j++) { + softmax_max = max(softmax_max, p_mask_shm[j]); + } + + // get sumexp + for (int j = 0; j < K; j++) { + opmath_t exp_results = exp(p_mask_shm[j] - softmax_max); + p_mask_shm[j] = exp_results; + softmax_sum += exp_results; + } + + // normalize + for (int j = 0; j < K; j++) { + p_mask_shm[j] /= softmax_sum; + } + } + + __syncthreads(); + } + + int offset_idx = 0; + int mask_idx = 0; + + const int w_stride = G * D; + const int base_ptr = gi * D + di_s; + const scalar_t *p_value_ptr = + p_value + (bi * (height_in * width_in)) * (G * D); + opmath_t *grad_im_ptr = grad_im + (bi * (height_in * width_in)) * (G * D); + + const int p0_w = ((dilation_w * (kernel_w - 1)) >> 1) - pad_w + + (qi % width_out) * stride_w; + const int p0_h = ((dilation_h * (kernel_h - 1)) >> 1) - pad_h + + (qi / width_out) * stride_h; + const opmath_t p0_w_ = + p0_w - ((dilation_w * (kernel_w - 1)) >> 1) * offset_scale; + const opmath_t p0_h_ = + p0_h - ((dilation_h * (kernel_h - 1)) >> 1) * offset_scale; + const int center_h = kernel_h / 2; + const int center_w = kernel_w / 2; + + grad_offset += (bi * Q + qi)*padded_offset_dim + gi*K*3; + opmath_t *grad_offset_softmax = grad_offset + K * 2; + + int cache_grad_off_idx = + ((threadIdx.z * G + threadIdx.y) * blockDim.x + threadIdx.x) * 3; + + opmath_t reg_grad_offset[3] = {0.}; + for (int i = 0; i < kernel_w; ++i) { + for (int j = 0; j < kernel_h; ++j) { + if (i != center_w || j != center_h || !remove_center) { + const opmath_t w_im = + p0_w_ + (i * dilation_w + (opmath_t)p_offset_ptr[offset_idx]) * + offset_scale; + const opmath_t h_im = + p0_h_ + (j * dilation_h + (opmath_t)p_offset_ptr[offset_idx + 1]) * + offset_scale; + const opmath_t attn = p_mask_shm[mask_idx]; + reg_grad_offset[0] = 0; + reg_grad_offset[1] = 0; + reg_grad_offset[2] = 0; + + if (h_im > -1 && w_im > -1 && h_im < height_in && w_im < width_in) { + ms_deform_attn_col2im_bilinear( + p_value_ptr, height_in, width_in, h_im, w_im, attn, w_stride, + base_ptr, offset_scale, offset_scale, top_grad, grad_im_ptr, + reg_grad_offset); + } + + // aggregated across different channel for offset + for (uint32_t offset = blockDim.x>>1; offset > 0; offset >>= 1){ + reg_grad_offset[0] += __shfl_down_sync(lane_mask, reg_grad_offset[0], offset); + reg_grad_offset[1] += __shfl_down_sync(lane_mask, reg_grad_offset[1], offset); + reg_grad_offset[2] += __shfl_down_sync(lane_mask, reg_grad_offset[2], offset); + } + + if (threadIdx.x == 0) { // + *(grad_offset) = reg_grad_offset[0]; // B x H x W x G x L x K x 3 + *(grad_offset + 1) = reg_grad_offset[1]; // B x H x W x G x L x K x 3 + if (softmax) { + cache_g_mask_before_softmax[mask_idx] = reg_grad_offset[2] * attn; + } + else{ + grad_offset_softmax[mask_idx] = reg_grad_offset[2]; + } + } + offset_idx += 2; + mask_idx += 1; + grad_offset += 2; + } + } + } + // backward for softmax + if(softmax){ + if (threadIdx.x == 0) { + opmath_t sum = 0.; + #pragma unroll + for (int i=0; i < K; ++i){ + sum += cache_g_mask_before_softmax[i]; + } + #pragma unroll + for (int i = 0; i < K; ++i) { + *(grad_offset_softmax) = cache_g_mask_before_softmax[i] - p_mask_shm[i] * sum; + grad_offset_softmax += 1; + } + } + } +} + +template +void _dcnv4_col2im_cuda( + cudaStream_t stream, + const scalar_t *value, // B, H * W, (G * D) + const scalar_t *p_offset, // B, H * W, (G*K*3) + const scalar_t *grad_output, // B, H_out*W_out, G * D + const int kernel_h, const int kernel_w, const int stride_h, + const int stride_w, const int pad_h, const int pad_w, const int dilation_h, + const int dilation_w, const int G, const int D, const int B, + const int height_in, const int width_in, const int height_out, + const int width_out, const opmath_t offset_scale, const int remove_center, + opmath_t *grad_im, opmath_t *grad_offset, const int block_thread, + const bool softmax, const int padded_offset_dim) { + + constexpr int L = 1; + + auto kernel = + backward_kernel_dcn_warp_primitive; + + int N = height_in * width_in; + int Q = height_out * width_out; + int K = kernel_h * kernel_w; + + if (remove_center) { + K -= 1; + } + + if (softmax) { + switch (K) { + case 9: + if(check_backward_warpp(d_stride, D)){ + kernel = backward_kernel_dcn_warp_primitive; + } + else{ + kernel = backward_kernel_dcn; + } + break; + case 8: + if(check_backward_warpp(d_stride, D)){ + kernel = backward_kernel_dcn_warp_primitive; + } + else { + kernel = backward_kernel_dcn; + } + break; + default: + printf("K=%ld\n", K); + throw std::invalid_argument("invalid kernel shape"); + } + } else { + switch (K) { + case 9: + if(check_backward_warpp(d_stride, D)){ + kernel = backward_kernel_dcn_warp_primitive; + } + else{ + kernel = backward_kernel_dcn; + } + break; + case 8: + if(check_backward_warpp(d_stride, D)){ + kernel = backward_kernel_dcn_warp_primitive; + } + else { + kernel = backward_kernel_dcn; + } + break; + default: + printf("K=%ld\n", K); + throw std::invalid_argument("invalid kernel shape"); + } + } + + const int block_multiplier = block_thread / (D / d_stride) / G; + assert((B*Q) % block_multiplier == 0); + + dim3 num_blocks(B*Q / block_multiplier); + dim3 num_threads(D / d_stride, G, block_multiplier); + + const int blockdimX = D / d_stride; + + int shm_size = sizeof(opmath_t) * (G * block_multiplier * K) * 2; + if(!check_backward_warpp(d_stride, D)){ + shm_size = sizeof(opmath_t) * ((G * block_multiplier * K) * 2 + G * block_multiplier * blockdimX * 3); + } + + cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, + shm_size); + + kernel<<>>( + value, p_offset, grad_output, G, D, Q, kernel_h, kernel_w, stride_h, + stride_w, pad_h, pad_w, dilation_h, dilation_w, height_in, width_in, + height_out, width_out, offset_scale, remove_center, block_multiplier, + grad_im, grad_offset, padded_offset_dim); + + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) { + printf("error in dcnv4_im2col_cuda: %s\n", cudaGetErrorString(err)); + printf("launch arguments: gridDim=(%d, %d, %d), blockDim=(%d, %d, %d), " + "shm_size=%d\n\n", + num_blocks.x, num_blocks.y, num_blocks.z, num_threads.x, + num_threads.y, num_threads.z, shm_size); + AT_ASSERTM(false, "kernel launch error"); + } +} + +template +void dcnv4_col2im_cuda( + cudaStream_t stream, + const scalar_t *value, // B, H * W, (G * D) + const scalar_t *p_offset, // B, H * W, (G*K*3) + const scalar_t *grad_output, // B, H_out*W_out, G * D + const int kernel_h, const int kernel_w, const int stride_h, + const int stride_w, const int pad_h, const int pad_w, const int dilation_h, + const int dilation_w, const int G, const int D, const int B, + const int height_in, const int width_in, const int height_out, + const int width_out, const opmath_t offset_scale, const int remove_center, + opmath_t *grad_im, opmath_t *grad_offset, const int d_stride, + const int block_thread, const bool softmax, const int padded_offset_dim) { + + assert(D % d_stride == 0); + const int size_scalar = sizeof(scalar_t); + if (size_scalar == 2) { + switch (d_stride) { + case 1: + _dcnv4_col2im_cuda( + stream, value, p_offset, grad_output, kernel_h, kernel_w, stride_h, + stride_w, pad_h, pad_w, dilation_h, dilation_w, G, D, B, height_in, + width_in, height_out, width_out, offset_scale, remove_center, grad_im, + grad_offset, block_thread, softmax, padded_offset_dim); + break; + case 2: + _dcnv4_col2im_cuda( + stream, value, p_offset, grad_output, kernel_h, kernel_w, stride_h, + stride_w, pad_h, pad_w, dilation_h, dilation_w, G, D, B, height_in, + width_in, height_out, width_out, offset_scale, remove_center, grad_im, + grad_offset, block_thread, softmax, padded_offset_dim); + break; + case 4: + _dcnv4_col2im_cuda( + stream, value, p_offset, grad_output, kernel_h, kernel_w, stride_h, + stride_w, pad_h, pad_w, dilation_h, dilation_w, G, D, B, height_in, + width_in, height_out, width_out, offset_scale, remove_center, grad_im, + grad_offset, block_thread, softmax, padded_offset_dim); + break; + case 8: + _dcnv4_col2im_cuda( + stream, value, p_offset, grad_output, kernel_h, kernel_w, stride_h, + stride_w, pad_h, pad_w, dilation_h, dilation_w, G, D, B, height_in, + width_in, height_out, width_out, offset_scale, remove_center, grad_im, + grad_offset, block_thread, softmax, padded_offset_dim); + break; + case 16: + _dcnv4_col2im_cuda( + stream, value, p_offset, grad_output, kernel_h, kernel_w, stride_h, + stride_w, pad_h, pad_w, dilation_h, dilation_w, G, D, B, height_in, + width_in, height_out, width_out, offset_scale, remove_center, grad_im, + grad_offset, block_thread, softmax, padded_offset_dim); + break; + } + } else { + assert(size_scalar == 4); + switch (d_stride) { + case 1: + _dcnv4_col2im_cuda( + stream, value, p_offset, grad_output, kernel_h, kernel_w, stride_h, + stride_w, pad_h, pad_w, dilation_h, dilation_w, G, D, B, height_in, + width_in, height_out, width_out, offset_scale, remove_center, grad_im, + grad_offset, block_thread, softmax, padded_offset_dim); + break; + case 2: + _dcnv4_col2im_cuda( + stream, value, p_offset, grad_output, kernel_h, kernel_w, stride_h, + stride_w, pad_h, pad_w, dilation_h, dilation_w, G, D, B, height_in, + width_in, height_out, width_out, offset_scale, remove_center, grad_im, + grad_offset, block_thread, softmax, padded_offset_dim); + break; + case 4: + _dcnv4_col2im_cuda( + stream, value, p_offset, grad_output, kernel_h, kernel_w, stride_h, + stride_w, pad_h, pad_w, dilation_h, dilation_w, G, D, B, height_in, + width_in, height_out, width_out, offset_scale, remove_center, grad_im, + grad_offset, block_thread, softmax, padded_offset_dim); + break; + case 8: + _dcnv4_col2im_cuda( + stream, value, p_offset, grad_output, kernel_h, kernel_w, stride_h, + stride_w, pad_h, pad_w, dilation_h, dilation_w, G, D, B, height_in, + width_in, height_out, width_out, offset_scale, remove_center, grad_im, + grad_offset, block_thread, softmax, padded_offset_dim); + break; + } + } +} \ No newline at end of file diff --git a/DCNv4_op/src/cuda/dcnv4_cuda.cu b/DCNv4_op/src/cuda/dcnv4_cuda.cu new file mode 100644 index 0000000..fd158d1 --- /dev/null +++ b/DCNv4_op/src/cuda/dcnv4_cuda.cu @@ -0,0 +1,175 @@ +/*! +************************************************************************************************** +* Deformable DETR +* Copyright (c) 2020 SenseTime. All Rights Reserved. +* Licensed under the Apache License, Version 2.0 [see LICENSE for details] +************************************************************************************************** +* Modified from +*https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +************************************************************************************************** +*/ + +#include "cuda/dcnv4_im2col_cuda.cuh" +#include "cuda/dcnv4_col2im_cuda.cuh" +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +at::Tensor dcnv4_cuda_forward( + const at::Tensor &value, + const at::Tensor &p_offset, + const int kernel_h, const int kernel_w, const int stride_h, + const int stride_w, const int pad_h, const int pad_w, const int dilation_h, + const int dilation_w, const int group, const int group_channels, + const float offset_scale, const int im2col_step, const int remove_center, + const int d_stride, const int block_thread, const bool softmax) { + AT_ASSERTM(value.is_contiguous(), "input tensor has to be contiguous"); + AT_ASSERTM(value.type().is_cuda(), "input must be a CUDA tensor"); + AT_ASSERTM(p_offset.is_contiguous(), "input tensor has to be contiguous"); + AT_ASSERTM(p_offset.type().is_cuda(), "input must be a CUDA tensor"); + + const int batch = value.size(0); + const int height_in = value.size(1); + const int width_in = value.size(2); + const int channels = value.size(3); + const int padded_offset_dim = p_offset.size(3); + + // tensor core requirement + assert(padded_offset_dim % 8 == 0); + + const int height_out = + (height_in + 2 * pad_h - (dilation_h * (kernel_h - 1) + 1)) / stride_h + + 1; + const int width_out = + (width_in + 2 * pad_w - (dilation_w * (kernel_w - 1) + 1)) / stride_w + 1; + + const int im2col_step_ = std::min(batch, im2col_step); + AT_ASSERTM(batch % im2col_step_ == 0, "batch(", batch, + ") must divide im2col_step(", im2col_step_, ")"); + AT_ASSERTM( + channels == (group * group_channels), + "Input channels and group times group channels wont match: (%d vs %d).", + channels, group * group_channels); + + auto output = at::zeros( + {batch, height_out, width_out, group * group_channels}, value.options()); + + const int batch_n = im2col_step_; + auto output_n = output.view({batch / batch_n, batch_n, height_out, width_out, + group * group_channels}); + auto per_value_size = height_in * width_in * channels; + auto per_offset_size = height_out * width_out * padded_offset_dim; + + for (int n = 0; n < batch / im2col_step_; ++n) { + auto columns = output_n.select(0, n); + AT_DISPATCH_FLOATING_TYPES_AND2( + at::ScalarType::Half, at::ScalarType::BFloat16, value.scalar_type(), + "dcnv4_forward_cuda", ([&] { + dcnv4_im2col_cuda( + at::cuda::getCurrentCUDAStream(), + value.data_ptr() + n * im2col_step_ * per_value_size, + p_offset.data_ptr() + + n * im2col_step_ * per_offset_size, + columns.data_ptr(), kernel_h, kernel_w, stride_h, + stride_w, pad_h, pad_w, dilation_h, dilation_w, group, + group_channels, batch_n, height_in, width_in, height_out, + width_out, offset_scale, remove_center, d_stride, block_thread, + softmax, padded_offset_dim); + })); + } + + return output; +} + +std::vector +dcnv4_cuda_backward( + const at::Tensor &value, + const at::Tensor &p_offset, + const int kernel_h, const int kernel_w, const int stride_h, + const int stride_w, const int pad_h, const int pad_w, const int dilation_h, + const int dilation_w, const int group, const int group_channels, + const float offset_scale, const int im2col_step, const at::Tensor &grad_output, + const int remove_center, const int d_stride, const int block_thread, + const bool softmax) { + AT_ASSERTM(value.is_contiguous(), "input tensor has to be contiguous"); + AT_ASSERTM(p_offset.is_contiguous(), "offset tensor has to be contiguous"); + AT_ASSERTM(grad_output.is_contiguous(), + "grad_output tensor has to be contiguous"); + + AT_ASSERTM(value.type().is_cuda(), "input must be a CUDA tensor"); + AT_ASSERTM(p_offset.type().is_cuda(), "offset must be a CUDA tensor"); + AT_ASSERTM(grad_output.type().is_cuda(), + "grad_output must be a CUDA tensor"); + + const int batch = value.size(0); + const int height_in = value.size(1); + const int width_in = value.size(2); + const int channels = value.size(3); + const int padded_offset_dim = p_offset.size(3); + assert(padded_offset_dim % 8 == 0); + + const int height_out = + (height_in + 2 * pad_h - (dilation_h * (kernel_h - 1) + 1)) / stride_h + + 1; + const int width_out = + (width_in + 2 * pad_w - (dilation_w * (kernel_w - 1) + 1)) / stride_w + 1; + + const int im2col_step_ = std::min(batch, im2col_step); + AT_ASSERTM(batch % im2col_step_ == 0, "batch(", batch, + ") must divide im2col_step(", im2col_step_, ")"); + AT_ASSERTM( + channels == (group * group_channels), + "Input channels and group times group channels wont match: (%d vs %d).", + channels, group * group_channels); + + auto dtype = value.dtype(); + if (dtype == at::kHalf){ + dtype = at::kFloat; + } + + auto grad_input = at::zeros_like(value, dtype); + auto grad_offset = at::zeros_like(p_offset, dtype); + + const int batch_n = im2col_step_; + auto grad_output_n = grad_output.view({batch / batch_n, batch_n, height_out, width_out, + group, group_channels}); + auto per_value_size = height_in * width_in * channels; + auto per_offset_size = height_out * width_out * padded_offset_dim; + + for (int n = 0; n < batch / im2col_step_; ++n) { + auto columns = grad_output_n.select(0, n); + AT_DISPATCH_FLOATING_TYPES_AND_HALF(value.type(), + "dcnv4_backward_cuda", ([&] { + dcnv4_col2im_cuda( + at::cuda::getCurrentCUDAStream(), + value.data_ptr() + n * im2col_step_ * per_value_size, + p_offset.data_ptr() + + n * im2col_step_ * per_offset_size, + columns.data_ptr(), kernel_h, kernel_w, stride_h, + stride_w, pad_h, pad_w, dilation_h, dilation_w, group, + group_channels, batch_n, height_in, width_in, height_out, + width_out, offset_scale, remove_center, + grad_input.data() + n * im2col_step_ * per_value_size, + grad_offset.data() + + n * im2col_step_ * per_offset_size, + d_stride, block_thread, softmax, padded_offset_dim + ); + })); + } + + if(value.dtype() == torch::kHalf){ + return {grad_input.to(torch::kHalf), grad_offset.to(torch::kHalf)}; + } + else{ + return {grad_input, grad_offset}; + } +} + diff --git a/DCNv4_op/src/cuda/dcnv4_cuda.h b/DCNv4_op/src/cuda/dcnv4_cuda.h new file mode 100644 index 0000000..5d42bdd --- /dev/null +++ b/DCNv4_op/src/cuda/dcnv4_cuda.h @@ -0,0 +1,33 @@ +/*! +************************************************************************************************** +* Deformable DETR +* Copyright (c) 2020 SenseTime. All Rights Reserved. +* Licensed under the Apache License, Version 2.0 [see LICENSE for details] +************************************************************************************************** +* Modified from +*https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +************************************************************************************************** +*/ + +#pragma once +#include + +at::Tensor dcnv4_cuda_forward( + const at::Tensor &value, + const at::Tensor &p_offset, + const int kernel_h, const int kernel_w, const int stride_h, + const int stride_w, const int pad_h, const int pad_w, const int dilation_h, + const int dilation_w, const int group, const int group_channels, + const float offset_scale, const int im2col_step, const int remove_center, + const int d_stride, const int block_thread, const bool softmax); + +std::vector +dcnv4_cuda_backward( + const at::Tensor &value, + const at::Tensor &p_offset, + const int kernel_h, const int kernel_w, const int stride_h, + const int stride_w, const int pad_h, const int pad_w, const int dilation_h, + const int dilation_w, const int group, const int group_channels, + const float offset_scale, const int im2col_step, const at::Tensor &grad_output, + const int remove_center, const int d_stride, const int block_thread, + const bool softmax); \ No newline at end of file diff --git a/DCNv4_op/src/cuda/dcnv4_im2col_cuda.cuh b/DCNv4_op/src/cuda/dcnv4_im2col_cuda.cuh new file mode 100644 index 0000000..3f460f2 --- /dev/null +++ b/DCNv4_op/src/cuda/dcnv4_im2col_cuda.cuh @@ -0,0 +1,416 @@ +#include +#include +#include + +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include "common.h" + +template +__global__ void forward_kernel_dcn( + const scalar_t *p_value, const scalar_t *p_offset, scalar_t *p_output, + const int G, const int D, const int Q, const int kernel_h, + const int kernel_w, const int stride_h, const int stride_w, const int pad_h, + const int pad_w, const int dilation_h, const int dilation_w, + const int height_in, const int width_in, const int height_out, + const int width_out, const opmath_t offset_scale, const int remove_center, + const int block_multiplier, const int padded_offset_dim) { + + const int &qi = (blockIdx.x * block_multiplier % Q) + threadIdx.z; + const int &bi = blockIdx.x * block_multiplier / Q; + + const int &di_s = threadIdx.x * d_stride; + const int &gi = threadIdx.y; + constexpr int li = 0; + + extern __shared__ char _s[]; + opmath_t *const p_mask_shm = + (opmath_t *)(_s) + ((threadIdx.z * G + gi) * L + li) * K; + + opmath_t p_out_shm[d_stride] = {0.}; + + const scalar_t *p_offset_ptr = p_offset + (bi*Q + qi)*padded_offset_dim + gi*K*3; + + const int mask_length = K; + const int num_thread = (D / d_stride); + const int num_iter = mask_length / num_thread; + const int remainder = mask_length - num_iter * num_thread; + + for (int i = 0; i < num_iter; i++) { + *(p_mask_shm + num_thread * i + threadIdx.x) = + *(scalar_t *)(p_offset_ptr + K * 2 + num_thread * i + threadIdx.x); + } + if (remainder > 0 && threadIdx.x < remainder) { + *(p_mask_shm + num_thread * num_iter + threadIdx.x) = *( + scalar_t *)(p_offset_ptr + K * 2 + num_thread * num_iter + threadIdx.x); + } + + int mask_idx; + if (softmax) { + __syncthreads(); + + // Calculate softmax over L and K + if (threadIdx.x == 0) { // gi != 0, di = 0, li = 0 + opmath_t softmax_max = -1e100; + opmath_t softmax_sum = 0.0; + + // get max + // #pragma unroll + for (int j = 0; j < K; j++) { + softmax_max = max(softmax_max, p_mask_shm[j]); + } + + // get sumexp + // #pragma unroll + for (int j = 0; j < K; j++) { + opmath_t exp_results = exp(p_mask_shm[j] - softmax_max); + p_mask_shm[j] = exp_results; + softmax_sum += exp_results; + } + + // normalize + // #pragma unroll + for (int j = 0; j < K; j++) { + p_mask_shm[j] /= softmax_sum; + } + } + + __syncthreads(); + } + int offset_idx = 0; + mask_idx = 0; + + const int w_stride = G * D; + const int base_ptr = gi * D + di_s; + const scalar_t *p_value_ptr = + p_value + (bi * (height_in * width_in)) * (G * D); + + const int p0_w = ((dilation_w * (kernel_w - 1)) >> 1) - pad_w + + (qi % width_out) * stride_w; + const int p0_h = ((dilation_h * (kernel_h - 1)) >> 1) - pad_h + + (qi / width_out) * stride_h; + const opmath_t p0_w_ = + p0_w - ((dilation_w * (kernel_w - 1)) >> 1) * offset_scale; + const opmath_t p0_h_ = + p0_h - ((dilation_h * (kernel_h - 1)) >> 1) * offset_scale; + const int center_h = kernel_h / 2; + const int center_w = kernel_w / 2; + + int out_idx = ((bi * Q + qi) * G + gi) * D + di_s; + + for (int i = 0; i < kernel_w; ++i) { + for (int j = 0; j < kernel_h; ++j) { + if (i != center_w || j != center_h || !remove_center) { + const opmath_t w_im = + p0_w_ + (i * dilation_w + (opmath_t)p_offset_ptr[offset_idx]) * + offset_scale; + const opmath_t h_im = + p0_h_ + (j * dilation_h + (opmath_t)p_offset_ptr[offset_idx + 1]) * + offset_scale; + const opmath_t attn = p_mask_shm[mask_idx]; + + if (h_im > -1 && w_im > -1 && h_im < height_in && w_im < width_in) { + ms_deform_attn_im2col_bilinear( + p_out_shm, p_value_ptr, height_in, width_in, h_im, w_im, attn, + w_stride, base_ptr); + } + offset_idx += 2; + mask_idx += 1; + } + } + } + scalar_t *fp16_regs = (scalar_t *)(p_out_shm); +#pragma unroll + for (int ds = 0; ds < d_stride; ds++) { + fp16_regs[ds] = p_out_shm[ds]; + } + *(transfer_t *)(p_output + out_idx) = *(transfer_t *)(p_out_shm); +} + +template +__global__ void forward_kernel_dcn_reg( + const scalar_t *p_value, const scalar_t *p_offset, scalar_t *p_output, + const int G, const int D, const int Q, const int kernel_h, + const int kernel_w, const int stride_h, const int stride_w, const int pad_h, + const int pad_w, const int dilation_h, const int dilation_w, + const int height_in, const int width_in, const int height_out, + const int width_out, const opmath_t offset_scale, const int remove_center, + const int block_multiplier, const int padded_offset_dim) { + + const int &qi = (blockIdx.x * block_multiplier % Q) + threadIdx.z; + const int &bi = blockIdx.x * block_multiplier / Q; + + const int &di_s = threadIdx.x * d_stride; + const int &gi = threadIdx.y; + constexpr int li = 0; + + opmath_t p_mask_shm[K] = {0.}; + opmath_t p_out_shm[d_stride] = {0.}; + + const scalar_t *p_offset_ptr = p_offset + (bi*Q + qi)*padded_offset_dim + gi*K*3; + const int mask_length = K; + const int num_thread = (D / d_stride); + const int num_iter = mask_length / num_thread; + const int remainder = mask_length - num_iter * num_thread; + + for (int i=0; i < K; i++){ + p_mask_shm[i] = *(p_offset_ptr + K*2 + i); + } + + if (softmax) { + // Calculate softmax over L and K + opmath_t softmax_max = -1e100; + opmath_t softmax_sum = 0.0; + // get max + for (int j = 0; j < K; j++) { + softmax_max = max(softmax_max, p_mask_shm[j]); + } + + // get sumexp + for (int j = 0; j < K; j++) { + opmath_t exp_results = exp(p_mask_shm[j] - softmax_max); + p_mask_shm[j] = exp_results; + softmax_sum += exp_results; + } + + // normalize + for (int j = 0; j < K; j++) { + p_mask_shm[j] /= softmax_sum; + } + } + + int offset_idx = 0; + int mask_idx = 0; + + const int w_stride = G * D; + const int base_ptr = gi * D + di_s; + const scalar_t *p_value_ptr = + p_value + (bi * (height_in * width_in)) * (G * D); + + const int p0_w = ((dilation_w * (kernel_w - 1)) >> 1) - pad_w + + (qi % width_out) * stride_w; + const int p0_h = ((dilation_h * (kernel_h - 1)) >> 1) - pad_h + + (qi / width_out) * stride_h; + const opmath_t p0_w_ = + p0_w - ((dilation_w * (kernel_w - 1)) >> 1) * offset_scale; + const opmath_t p0_h_ = + p0_h - ((dilation_h * (kernel_h - 1)) >> 1) * offset_scale; + const int center_h = kernel_h / 2; + const int center_w = kernel_w / 2; + + int out_idx = ((bi * Q + qi) * G + gi) * D + di_s; + + for (int i = 0; i < kernel_w; ++i) { + for (int j = 0; j < kernel_h; ++j) { + if (i != center_w || j != center_h || !remove_center) { + const opmath_t w_im = + p0_w_ + (i * dilation_w + (opmath_t)p_offset_ptr[offset_idx]) * + offset_scale; + const opmath_t h_im = + p0_h_ + (j * dilation_h + (opmath_t)p_offset_ptr[offset_idx + 1]) * + offset_scale; + const opmath_t attn = p_mask_shm[mask_idx]; + + if (h_im > -1 && w_im > -1 && h_im < height_in && w_im < width_in) { + ms_deform_attn_im2col_bilinear( + p_out_shm, p_value_ptr, height_in, width_in, h_im, w_im, attn, + w_stride, base_ptr); + } + offset_idx += 2; + mask_idx += 1; + } + } + } + scalar_t *fp16_regs = (scalar_t *)(p_out_shm); +#pragma unroll + for (int ds = 0; ds < d_stride; ds++) { + fp16_regs[ds] = p_out_shm[ds]; + } + + *(transfer_t *)(p_output + out_idx) = *(transfer_t *)(p_out_shm); +} + +template +void _dcnv4_im2col_cuda(cudaStream_t stream, + const scalar_t *value, // B, H * W, (G * D) + const scalar_t *p_offset, // B, H * W, G * K * 3) + scalar_t *output, // B, H_out*W_out, G * D + const int kernel_h, const int kernel_w, + const int stride_h, const int stride_w, + const int pad_h, const int pad_w, + const int dilation_h, const int dilation_w, + const int G, const int D, const int B, + const int height_in, const int width_in, + const int height_out, const int width_out, + const opmath_t offset_scale, + const int remove_center, const int block_thread, + const int softmax, + const int padded_offset_dim) { + + constexpr int L = 1; + + auto kernel = forward_kernel_dcn_reg; + + int N = height_in * width_in; + int Q = height_out * width_out; + int K = kernel_h * kernel_w; + + if (remove_center) { + K -= 1; + } + if (softmax) { + switch (K) { + case 9: + kernel = forward_kernel_dcn_reg; + break; + case 8: + kernel = forward_kernel_dcn_reg; + break; + default: + printf("K=%ld\n", K); + throw std::invalid_argument("invalid kernel shape"); + } + } else { + switch (K) { + case 9: + kernel = forward_kernel_dcn_reg; + break; + case 8: + kernel = forward_kernel_dcn_reg; + break; + default: + printf("K=%ld\n", K); + throw std::invalid_argument("invalid kernel shape"); + } + } + + const int block_multiplier = block_thread / (D / d_stride) / G; + assert((B*Q) % block_multiplier == 0); + + dim3 num_blocks(B*Q / block_multiplier); + dim3 num_threads(D / d_stride, G, block_multiplier); + + int shm_size = 0; + + cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, + shm_size); + + kernel<<>>( + value, p_offset, output, G, D, Q, kernel_h, kernel_w, stride_h, stride_w, + pad_h, pad_w, dilation_h, dilation_w, height_in, width_in, height_out, + width_out, offset_scale, remove_center, block_multiplier, padded_offset_dim); + + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) { + printf("error in dcnv4_im2col_cuda: %s\n", cudaGetErrorString(err)); + printf("launch arguments: gridDim=(%d, %d, %d), blockDim=(%d, %d, %d), " + "shm_size=%d\n\n", + num_blocks.x, num_blocks.y, num_blocks.z, num_threads.x, + num_threads.y, num_threads.z, shm_size); + AT_ASSERTM(false, "kernel launch error"); + } +} + +template +void dcnv4_im2col_cuda( + cudaStream_t stream, + const scalar_t *value, // B, H * W, (G * D) + const scalar_t *p_offset, // B, H * W, G * K * 3) + scalar_t *output, // B, H_out*W_out, G * D + const int kernel_h, const int kernel_w, const int stride_h, + const int stride_w, const int pad_h, const int pad_w, const int dilation_h, + const int dilation_w, const int G, const int D, const int B, + const int height_in, const int width_in, const int height_out, + const int width_out, const opmath_t offset_scale, const int remove_center, + const int d_stride, const int block_thread, const bool softmax, + const int padded_offset_dim) { + + assert(D % d_stride == 0); + if (sizeof(scalar_t) == 2) { + switch (d_stride) { + case 1: + _dcnv4_im2col_cuda( + stream, value, p_offset, output, kernel_h, kernel_w, stride_h, + stride_w, pad_h, pad_w, dilation_h, dilation_w, G, D, B, height_in, + width_in, height_out, width_out, offset_scale, remove_center, + block_thread, softmax, padded_offset_dim); + break; + case 2: + _dcnv4_im2col_cuda( + stream, value, p_offset, output, kernel_h, kernel_w, stride_h, + stride_w, pad_h, pad_w, dilation_h, dilation_w, G, D, B, height_in, + width_in, height_out, width_out, offset_scale, remove_center, + block_thread, softmax, padded_offset_dim); + break; + case 4: + _dcnv4_im2col_cuda( + stream, value, p_offset, output, kernel_h, kernel_w, stride_h, + stride_w, pad_h, pad_w, dilation_h, dilation_w, G, D, B, height_in, + width_in, height_out, width_out, offset_scale, remove_center, + block_thread, softmax, padded_offset_dim); + break; + case 8: + _dcnv4_im2col_cuda( + stream, value, p_offset, output, kernel_h, kernel_w, stride_h, + stride_w, pad_h, pad_w, dilation_h, dilation_w, G, D, B, height_in, + width_in, height_out, width_out, offset_scale, remove_center, + block_thread, softmax, padded_offset_dim); + break; + case 16: + _dcnv4_im2col_cuda( + stream, value, p_offset, output, kernel_h, kernel_w, stride_h, + stride_w, pad_h, pad_w, dilation_h, dilation_w, G, D, B, height_in, + width_in, height_out, width_out, offset_scale, remove_center, + block_thread, softmax, padded_offset_dim); + break; + } + } else { + assert(sizeof(scalar_t) == 4); + switch (d_stride) { + case 1: + _dcnv4_im2col_cuda( + stream, value, p_offset, output, kernel_h, kernel_w, stride_h, + stride_w, pad_h, pad_w, dilation_h, dilation_w, G, D, B, height_in, + width_in, height_out, width_out, offset_scale, remove_center, + block_thread, softmax, padded_offset_dim); + break; + case 2: + _dcnv4_im2col_cuda( + stream, value, p_offset, output, kernel_h, kernel_w, stride_h, + stride_w, pad_h, pad_w, dilation_h, dilation_w, G, D, B, height_in, + width_in, height_out, width_out, offset_scale, remove_center, + block_thread, softmax, padded_offset_dim); + break; + case 4: + _dcnv4_im2col_cuda( + stream, value, p_offset, output, kernel_h, kernel_w, stride_h, + stride_w, pad_h, pad_w, dilation_h, dilation_w, G, D, B, height_in, + width_in, height_out, width_out, offset_scale, remove_center, + block_thread, softmax, padded_offset_dim); + break; + case 8: + _dcnv4_im2col_cuda( + stream, value, p_offset, output, kernel_h, kernel_w, stride_h, + stride_w, pad_h, pad_w, dilation_h, dilation_w, G, D, B, height_in, + width_in, height_out, width_out, offset_scale, remove_center, + block_thread, softmax, padded_offset_dim); + break; + default: + printf("not supported for d_stride > 8 for fp32"); + throw std::invalid_argument("invalid d_stride"); + } + } +} \ No newline at end of file diff --git a/DCNv4_op/src/cuda/flash_deform_attn_cuda.cu b/DCNv4_op/src/cuda/flash_deform_attn_cuda.cu new file mode 100644 index 0000000..3038ea5 --- /dev/null +++ b/DCNv4_op/src/cuda/flash_deform_attn_cuda.cu @@ -0,0 +1,162 @@ +/*! +************************************************************************************************** +* Deformable DETR +* Copyright (c) 2020 SenseTime. All Rights Reserved. +* Licensed under the Apache License, Version 2.0 [see LICENSE for details] +************************************************************************************************** +* Modified from +*https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +************************************************************************************************** +*/ + +#include "cuda/flash_deform_im2col_cuda.cuh" +#include "cuda/flash_deform_col2im_cuda.cuh" +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +at::Tensor flash_deform_attn_cuda_forward( + const at::Tensor &value, const at::Tensor &spatial_shapes, + const at::Tensor &level_start_index, const at::Tensor &sampling_loc_attn, + const int im2col_step = 64, const int K=8, const int d_stride=8, + const int block_thread=0) { + AT_ASSERTM(value.is_contiguous(), "value tensor has to be contiguous"); + AT_ASSERTM(spatial_shapes.is_contiguous(), + "spatial_shapes tensor has to be contiguous"); + AT_ASSERTM(level_start_index.is_contiguous(), + "level_start_index tensor has to be contiguous"); + AT_ASSERTM(sampling_loc_attn.is_contiguous(), + "sampling_loc_attn tensor has to be contiguous"); + + AT_ASSERTM(value.type().is_cuda(), "value must be a CUDA tensor"); + AT_ASSERTM(spatial_shapes.type().is_cuda(), + "spatial_shapes must be a CUDA tensor"); + AT_ASSERTM(level_start_index.type().is_cuda(), + "level_start_index must be a CUDA tensor"); + AT_ASSERTM(sampling_loc_attn.type().is_cuda(), + "sampling_loc_attn must be a CUDA tensor"); + + const int batch = value.size(0); + const int spatial_size = value.size(1); + const int num_heads = value.size(2); + const int num_channels = value.size(3); + + const int num_levels = spatial_shapes.size(0); + const int num_query = sampling_loc_attn.size(1); + const int num_point = K; + + const int im2col_step_ = std::min(batch, im2col_step); + AT_ASSERTM(batch % im2col_step_ == 0, "batch(", batch, + ") must divide im2col_step(", im2col_step_, ")"); + + auto output = + at::zeros({batch, num_query, num_heads, num_channels}, value.options()); + + auto per_value_size = spatial_size * num_heads * num_channels; + auto per_offset_size = num_query * num_heads * num_levels * num_point * 3; + auto per_out_size = num_query * num_heads * num_channels; + + for (int n = 0; n < batch / im2col_step_; ++n) { + AT_DISPATCH_FLOATING_TYPES_AND2( + at::ScalarType::Half, at::ScalarType::BFloat16, value.scalar_type(), + "flash_deform_attn_forward_cuda", ([&] { + flash_deformable_im2col_cuda( + at::cuda::getCurrentCUDAStream(), + value.data_ptr() + n * im2col_step_ * per_value_size, + spatial_shapes.data(), level_start_index.data(), + sampling_loc_attn.data_ptr() + + n * im2col_step_ * per_offset_size, + output.data_ptr() + n * im2col_step_ * per_out_size, + im2col_step_, spatial_size, num_heads, num_channels, num_levels, + num_query, num_point, d_stride, block_thread, true); + })); + } + output = output.view({batch, num_query, num_heads * num_channels}); + return output; +} + +std::vector +flash_deform_attn_cuda_backward( + const at::Tensor &value, const at::Tensor &spatial_shapes, + const at::Tensor &level_start_index, const at::Tensor &sampling_loc_attn, + const at::Tensor &grad_output, const int im2col_step = 64, const int K=8, + const int d_stride=2, const int block_thread=0) { + AT_ASSERTM(value.is_contiguous(), "value tensor has to be contiguous"); + AT_ASSERTM(spatial_shapes.is_contiguous(), + "spatial_shapes tensor has to be contiguous"); + AT_ASSERTM(level_start_index.is_contiguous(), + "level_start_index tensor has to be contiguous"); + AT_ASSERTM(sampling_loc_attn.is_contiguous(), + "sampling_loc_attn tensor has to be contiguous"); + AT_ASSERTM(grad_output.is_contiguous(), + "grad_output tensor has to be contiguous"); + + AT_ASSERTM(value.type().is_cuda(), "value must be a CUDA tensor"); + AT_ASSERTM(spatial_shapes.type().is_cuda(), + "spatial_shapes must be a CUDA tensor"); + AT_ASSERTM(level_start_index.type().is_cuda(), + "level_start_index must be a CUDA tensor"); + AT_ASSERTM(sampling_loc_attn.type().is_cuda(), + "sampling_loc_attn must be a CUDA tensor"); + AT_ASSERTM(grad_output.type().is_cuda(), + "grad_output must be a CUDA tensor"); + + const int batch = value.size(0); + const int spatial_size = value.size(1); + const int num_heads = value.size(2); + const int num_channels = value.size(3); + + const int num_levels = spatial_shapes.size(0); + const int num_query = sampling_loc_attn.size(1); + const int num_point = K; + + const int im2col_step_ = std::min(batch, im2col_step); + AT_ASSERTM(batch % im2col_step_ == 0, "batch(", batch, + ") must divide im2col_step(", im2col_step_, ")"); + + auto dtype = value.dtype(); + if (dtype == at::kHalf){ + dtype = at::kFloat; + } + + auto grad_input = at::zeros_like(value, dtype); + auto grad_offset = at::zeros_like(sampling_loc_attn, dtype); + + auto per_value_size = spatial_size * num_heads * num_channels; + auto per_offset_size = num_query * num_heads * num_levels * num_point * 3; + auto per_out_size = num_query * num_heads * num_channels; + + for (int n = 0; n < batch / im2col_step_; ++n) { + AT_DISPATCH_FLOATING_TYPES_AND_HALF(value.type(), + "flash_deform_attn_backward_cuda", ([&] { + flash_deformable_col2im_cuda( + at::cuda::getCurrentCUDAStream(), + value.data_ptr() + n * im2col_step_ * per_value_size, + spatial_shapes.data(), level_start_index.data(), + sampling_loc_attn.data_ptr() + + n * im2col_step_ * per_offset_size, + grad_output.data_ptr() + n * im2col_step_ * per_out_size, + im2col_step_, spatial_size, num_heads, num_channels, num_levels, + num_query, num_point, + grad_input.data() + n * im2col_step_ * per_value_size, + grad_offset.data() + n * im2col_step_ * per_offset_size, + d_stride, block_thread + ); + })); + } + + if(value.dtype() == torch::kHalf){ + return {grad_input.to(torch::kHalf), grad_offset.to(torch::kHalf)}; + } + else{ + return {grad_input, grad_offset}; + } +} diff --git a/DCNv4_op/src/cuda/flash_deform_attn_cuda.h b/DCNv4_op/src/cuda/flash_deform_attn_cuda.h new file mode 100644 index 0000000..ed00b2c --- /dev/null +++ b/DCNv4_op/src/cuda/flash_deform_attn_cuda.h @@ -0,0 +1,25 @@ +/*! +************************************************************************************************** +* Deformable DETR +* Copyright (c) 2020 SenseTime. All Rights Reserved. +* Licensed under the Apache License, Version 2.0 [see LICENSE for details] +************************************************************************************************** +* Modified from +*https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +************************************************************************************************** +*/ + +#pragma once +#include + +at::Tensor flash_deform_attn_cuda_forward( + const at::Tensor &value, const at::Tensor &spatial_shapes, + const at::Tensor &level_start_index, const at::Tensor &sampling_loc_attn, + const int im2col_step, const int K, const int d_stride, const int block_thread); + +std::vector +flash_deform_attn_cuda_backward( + const at::Tensor &value, const at::Tensor &spatial_shapes, + const at::Tensor &level_start_index, const at::Tensor &sampling_loc_attn, + const at::Tensor &grad_output, const int im2col_step, const int K, + const int d_stride, const int block_thread); \ No newline at end of file diff --git a/DCNv4_op/src/cuda/flash_deform_col2im_cuda.cuh b/DCNv4_op/src/cuda/flash_deform_col2im_cuda.cuh new file mode 100644 index 0000000..f1754a4 --- /dev/null +++ b/DCNv4_op/src/cuda/flash_deform_col2im_cuda.cuh @@ -0,0 +1,580 @@ +#include +#include +#include + +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include "common.h" + +template +__global__ void +backward_kernel(const scalar_t *p_value, const int64_t *data_spatial_shapes, + const int64_t *data_level_start_index, const scalar_t *p_offset, + const scalar_t *grad_output, const int N, const int G, + const int D, const int Q, + const int block_multiplier, opmath_t *grad_im, + opmath_t *grad_offset) { + + extern __shared__ char _s[]; + + const int &qi = (blockIdx.x * block_multiplier % Q) + threadIdx.z; + const int &bi = blockIdx.x * block_multiplier / Q; + + const int &di_s = threadIdx.x * d_stride; + const int &gi = threadIdx.y; + + opmath_t *cache_g_mask_before_softmax = + (opmath_t *)(_s); // (block_multiplier*G) * (L * K) + opmath_t *cache_grad_offset = + (opmath_t *)(cache_g_mask_before_softmax + + block_multiplier * G * L * + K); // (block_multiplier*G*D/d_stride*3) + opmath_t *const p_mask_shm = + ((opmath_t *)(cache_grad_offset + + block_multiplier * G * D / d_stride * 3)) + + (threadIdx.z * G + gi) * L * K; // G*block_multiplier * L * K + + const scalar_t *p_offset_ptr = + p_offset + (((bi * Q + qi) * G + gi) * L) * K * 3; + const int mask_length = L * K; + const int num_thread = (D / d_stride); + const int num_iter = mask_length / num_thread; + const int remainder = mask_length - num_iter * num_thread; + const scalar_t *top_grad = grad_output + ((bi * Q + qi) * G + gi) * D + di_s; + + for (int i = 0; i < num_iter; i++) { + *(p_mask_shm + num_thread * i + threadIdx.x) = + *(scalar_t *)(p_offset_ptr + L * K * 2 + num_thread * i + threadIdx.x); + } + if (remainder > 0 && threadIdx.x < remainder) { + *(p_mask_shm + num_thread * num_iter + threadIdx.x) = + *(scalar_t *)(p_offset_ptr + L * K * 2 + num_thread * num_iter + + threadIdx.x); + } + + __syncthreads(); + // Calculate softmax over L and K + if (threadIdx.x == 0) { // gi != 0, di = 0, li = 0 + opmath_t softmax_max = -1e100; + opmath_t softmax_sum = 0.0; + + // get max + for (int j = 0; j < L * K; j++) { + softmax_max = max(softmax_max, p_mask_shm[j]); + } + + // get sumexp + for (int j = 0; j < L * K; j++) { + opmath_t exp_results = exp(p_mask_shm[j] - softmax_max); + p_mask_shm[j] = exp_results; + softmax_sum += exp_results; + } + + // normalize + for (int j = 0; j < L * K; j++) { + p_mask_shm[j] /= softmax_sum; + } + } + + __syncthreads(); + + int offset_idx = 0; + int mask_idx = 0; + const int w_stride = G * D; + const int base_ptr = gi * D + di_s; + + for (int li = 0; li < L; li++) { + + const int spatial_h = data_spatial_shapes[li * 2]; + const int spatial_w = data_spatial_shapes[li * 2 + 1]; + const int level_start_id = data_level_start_index[li]; + const scalar_t *p_value_ptr = p_value + (bi * N + level_start_id) * G * D; + opmath_t *grad_im_ptr = grad_im + (bi * N + level_start_id) * G * D; + + int cache_grad_off_idx = + ((threadIdx.z * G + threadIdx.y) * blockDim.x + threadIdx.x) * 3; + for (int ki = 0; ki < K; ki++) { + const opmath_t loc_w = p_offset_ptr[offset_idx]; + const opmath_t loc_h = p_offset_ptr[offset_idx + 1]; + const opmath_t attn = p_mask_shm[mask_idx]; + const opmath_t h_im = loc_h * spatial_h - 0.5; + const opmath_t w_im = loc_w * spatial_w - 0.5; + // for cache_grad_offset (mG) x D/d x 3 + cache_grad_offset[cache_grad_off_idx] = 0; + cache_grad_offset[cache_grad_off_idx + 1] = 0; + cache_grad_offset[cache_grad_off_idx + 2] = 0; + + if (h_im > -1 && w_im > -1 && h_im < spatial_h && w_im < spatial_w) { + ms_deform_attn_col2im_bilinear( + p_value_ptr, spatial_h, spatial_w, h_im, w_im, attn, w_stride, + base_ptr, spatial_h, spatial_w, top_grad, grad_im_ptr, + cache_grad_offset + cache_grad_off_idx); + + // aggregate across different channel for offset + __syncthreads(); + if (threadIdx.x == 0) { + int _didx = (threadIdx.z * G + threadIdx.y) * blockDim.x * 3; + opmath_t _grad_w = cache_grad_offset[_didx]; + opmath_t _grad_h = cache_grad_offset[_didx + 1]; + opmath_t _grad_a = cache_grad_offset[_didx + 2]; + for (int c_id = 1; c_id < blockDim.x; ++c_id) { + _grad_w += cache_grad_offset[_didx + 3 * c_id]; + _grad_h += cache_grad_offset[_didx + 3 * c_id + 1]; + _grad_a += cache_grad_offset[_didx + 3 * c_id + 2]; + } + + grad_offset[((bi * Q + qi) * G + gi) * L * K * 3 + li * K * 2 + + ki * 2] = _grad_w; + grad_offset[((bi * Q + qi) * G + gi) * L * K * 3 + li * K * 2 + + ki * 2 + 1] = _grad_h; + cache_g_mask_before_softmax + [((threadIdx.y + threadIdx.z * G) * L + li) * K + ki] = _grad_a; + } + } + __syncthreads(); + + offset_idx += 2; + mask_idx += 1; + } + } + // backward for softmax + if (threadIdx.x == 0) { + for (int i = 0; i < L * K; ++i) { + opmath_t grad_i = 0.; + const opmath_t *group_g_mask = cache_g_mask_before_softmax + + (threadIdx.y + threadIdx.z * G) * L * K; + for (int j = 0; j < L * K; ++j) { + if (i != j) { + grad_i -= group_g_mask[j] * p_mask_shm[i] * p_mask_shm[j]; + } else { + grad_i += group_g_mask[i] * p_mask_shm[i] * (1 - p_mask_shm[i]); + } + } + grad_offset[((bi * Q + qi) * G + gi) * L * K * 3 + L * K * 2 + i] = + grad_i; + } + } + __syncthreads(); +} + +template +__global__ void +backward_kernel_warp_primitive(const scalar_t *p_value, const int64_t *data_spatial_shapes, + const int64_t *data_level_start_index, const scalar_t *p_offset, + const scalar_t *grad_output, const int N, const int G, + const int D, const int Q, + const int block_multiplier, opmath_t *grad_im, + opmath_t *grad_offset) { + + extern __shared__ char _s[]; + + const int &qi = (blockIdx.x * block_multiplier % Q) + threadIdx.z; + const int &bi = blockIdx.x * block_multiplier / Q; + + const int &di_s = threadIdx.x * d_stride; + const int &gi = threadIdx.y; + + const int tid = (threadIdx.z * blockDim.y + threadIdx.y)*blockDim.x + threadIdx.x; + const int lane_id = tid % kWarpSize; + const int group_per_warp = kWarpSize / blockDim.x; + const int group_in_warp_id = (threadIdx.z * G + threadIdx.y) % group_per_warp; + const unsigned lane_mask = ((1 << blockDim.x) - 1) << (group_in_warp_id * blockDim.x); + + opmath_t *cache_g_mask_before_softmax = + (opmath_t *)(_s); // (block_multiplier*G) * (L * K) + + opmath_t *const p_mask_shm = + ((opmath_t *)(cache_g_mask_before_softmax + block_multiplier * G * L * K)) + + (threadIdx.z * G + gi) * L * K; // G*block_multiplier * L * K + + const scalar_t *p_offset_ptr = + p_offset + (((bi * Q + qi) * G + gi) * L) * K * 3; + const int mask_length = L * K; + const int num_thread = (D / d_stride); + const int num_iter = mask_length / num_thread; + const int remainder = mask_length - num_iter * num_thread; + const scalar_t *top_grad = grad_output + ((bi * Q + qi) * G + gi) * D + di_s; + + for (int i = 0; i < num_iter; i++) { + *(p_mask_shm + num_thread * i + threadIdx.x) = + *(scalar_t *)(p_offset_ptr + L * K * 2 + num_thread * i + threadIdx.x); + } + if (remainder > 0 && threadIdx.x < remainder) { + *(p_mask_shm + num_thread * num_iter + threadIdx.x) = + *(scalar_t *)(p_offset_ptr + L * K * 2 + num_thread * num_iter + + threadIdx.x); + } + + __syncthreads(); + // Calculate softmax over L and K + if (threadIdx.x == 0) { // gi != 0, di = 0, li = 0 + opmath_t softmax_max = -1e100; + opmath_t softmax_sum = 0.0; + + // get max + for (int j = 0; j < L * K; j++) { + softmax_max = max(softmax_max, p_mask_shm[j]); + } + + // get sumexp + for (int j = 0; j < L * K; j++) { + opmath_t exp_results = exp(p_mask_shm[j] - softmax_max); + p_mask_shm[j] = exp_results; + softmax_sum += exp_results; + } + + // normalize + for (int j = 0; j < L * K; j++) { + p_mask_shm[j] /= softmax_sum; + } + } + + __syncthreads(); + + int offset_idx = 0; + int mask_idx = 0; + const int w_stride = G * D; + const int base_ptr = gi * D + di_s; + + for (int li = 0; li < L; li++) { + + const int spatial_h = data_spatial_shapes[li * 2]; + const int spatial_w = data_spatial_shapes[li * 2 + 1]; + const int level_start_id = data_level_start_index[li]; + const scalar_t *p_value_ptr = p_value + (bi * N + level_start_id) * G * D; + opmath_t *grad_im_ptr = grad_im + (bi * N + level_start_id) * G * D; + + int cache_grad_off_idx = + ((threadIdx.z * G + threadIdx.y) * blockDim.x + threadIdx.x) * 3; + + opmath_t reg_grad_offset[3] = {0.}; + for (int ki = 0; ki < K; ki++) { + const opmath_t loc_w = p_offset_ptr[offset_idx]; + const opmath_t loc_h = p_offset_ptr[offset_idx + 1]; + const opmath_t attn = p_mask_shm[mask_idx]; + const opmath_t h_im = loc_h * spatial_h - 0.5; + const opmath_t w_im = loc_w * spatial_w - 0.5; + reg_grad_offset[0] = 0; + reg_grad_offset[1] = 0; + reg_grad_offset[2] = 0; + + if (h_im > -1 && w_im > -1 && h_im < spatial_h && w_im < spatial_w) { + ms_deform_attn_col2im_bilinear( + p_value_ptr, spatial_h, spatial_w, h_im, w_im, attn, w_stride, + base_ptr, spatial_h, spatial_w, top_grad, grad_im_ptr, + reg_grad_offset); + + // aggregate across different channel for offset + for (uint32_t offset = blockDim.x>>1; offset > 0; offset >>= 1){ + reg_grad_offset[0] += __shfl_down_sync(lane_mask, reg_grad_offset[0], offset); + reg_grad_offset[1] += __shfl_down_sync(lane_mask, reg_grad_offset[1], offset); + reg_grad_offset[2] += __shfl_down_sync(lane_mask, reg_grad_offset[2], offset); + } + + if (threadIdx.x == 0) { + grad_offset[((bi * Q + qi) * G + gi) * L * K * 3 + li * K * 2 + + ki * 2] = reg_grad_offset[0]; + grad_offset[((bi * Q + qi) * G + gi) * L * K * 3 + li * K * 2 + + ki * 2 + 1] = reg_grad_offset[1]; + cache_g_mask_before_softmax + [((threadIdx.y + threadIdx.z * G) * L + li) * K + ki] = reg_grad_offset[2]; + } + } + __syncthreads(); + + offset_idx += 2; + mask_idx += 1; + } + } + // backward for softmax + if (threadIdx.x == 0) { + for (int i = 0; i < L * K; ++i) { + opmath_t grad_i = 0.; + const opmath_t *group_g_mask = cache_g_mask_before_softmax + + (threadIdx.y + threadIdx.z * G) * L * K; + for (int j = 0; j < L * K; ++j) { + if (i != j) { + grad_i -= group_g_mask[j] * p_mask_shm[i] * p_mask_shm[j]; + } else { + grad_i += group_g_mask[i] * p_mask_shm[i] * (1 - p_mask_shm[i]); + } + } + grad_offset[((bi * Q + qi) * G + gi) * L * K * 3 + L * K * 2 + i] = + grad_i; + } + } + __syncthreads(); +} + +template +void _flash_deformable_col2im_cuda( + cudaStream_t stream, + const scalar_t *value, // B, N, G, D + const int64_t *data_spatial_shapes, // L * 2 + const int64_t *data_level_start_index, // L + const scalar_t *offset, // B, N, G, L, K, 3 + const scalar_t *grad_output, // B, N, G, D + const int B, const int N, const int G, const int D, const int L, + const int Q, opmath_t *grad_im, opmath_t *grad_offset, + const int block_thread) { + + assert(D % d_stride == 0); + + const int block_multiplier = block_thread / (D / d_stride) / G; + assert((B*Q) % block_multiplier == 0); + dim3 num_blocks(B*Q / block_multiplier); + dim3 num_threads(D / d_stride, G, block_multiplier); + + int shm_size; + if(check_backward_warpp(d_stride, D)){ + shm_size = + sizeof(opmath_t) * (block_multiplier * G * L * K) + + sizeof(opmath_t) * (G * block_multiplier * L * K); + } + else{ + shm_size = + sizeof(opmath_t) * (block_multiplier * G * L * K) + + sizeof(opmath_t) * (G * block_multiplier * L * K) + + sizeof(opmath_t) * (G * block_multiplier * D / d_stride * 3); + } + + auto kernel = backward_kernel_warp_primitive; + + switch (L) { + case 1: + if(check_backward_warpp(d_stride, D)){ + kernel = backward_kernel_warp_primitive; + } else { + kernel = backward_kernel; + } + break; + case 2: + if(check_backward_warpp(d_stride, D)){ + kernel = backward_kernel_warp_primitive; + } else { + kernel = backward_kernel; + } + break; + case 3: + if(check_backward_warpp(d_stride, D)){ + kernel = backward_kernel_warp_primitive; + } else { + kernel = backward_kernel; + } + break; + case 4: + if(check_backward_warpp(d_stride, D)){ + kernel = backward_kernel_warp_primitive; + } else { + kernel = backward_kernel; + } + break; + case 5: + if(check_backward_warpp(d_stride, D)){ + kernel = backward_kernel_warp_primitive; + } else { + kernel = backward_kernel; + } + break; + default: + printf("L=%ld\n", L); + throw std::invalid_argument("invalid number of scales"); + } + cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, + shm_size); + + kernel<<>>( + value, data_spatial_shapes, data_level_start_index, offset, grad_output, + N, G, D, Q, block_multiplier, grad_im, grad_offset); + + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) { + printf("error in flash_deformable_im2col_cuda: %s\n", + cudaGetErrorString(err)); + printf("launch arguments: gridDim=(%d, %d, %d), blockDim=(%d, %d, %d), " + "shm_size=%d, Q=%d\n\n", + num_blocks.x, num_blocks.y, num_blocks.z, num_threads.x, + num_threads.y, num_threads.z, shm_size, Q); + AT_ASSERTM(false, "kernel launch error"); + } +} + +template +void flash_deformable_col2im_cuda_inner( + cudaStream_t stream, + const scalar_t *value, // B, N, G, D + const int64_t *data_spatial_shapes, // L * 2 + const int64_t *data_level_start_index, // L + const scalar_t *offset, // B, N, G, L, K, 3 + const scalar_t *grad_output, // B, N, G, D + const int B, const int N, const int G, const int D, const int L, + const int Q, opmath_t *grad_im, opmath_t *grad_offset, + const int d_stride, const int block_thread) { + + assert(D % d_stride == 0); + if(sizeof(scalar_t) == 2) { + switch(d_stride) { + case 1: + _flash_deformable_col2im_cuda( + stream, + value, // B, N, G, D + data_spatial_shapes, // L * 2 + data_level_start_index, // L + offset, // B, N, G, L, K, 3 + grad_output, // B, N, G, D + B, N, G, D, L, Q, grad_im, grad_offset, + block_thread); + break; + case 2: + _flash_deformable_col2im_cuda( + stream, + value, // B, N, G, D + data_spatial_shapes, // L * 2 + data_level_start_index, // L + offset, // B, N, G, L, K, 3 + grad_output, // B, N, G, D + B, N, G, D, L, Q, grad_im, grad_offset, + block_thread); + break; + case 4: + _flash_deformable_col2im_cuda( + stream, + value, // B, N, G, D + data_spatial_shapes, // L * 2 + data_level_start_index, // L + offset, // B, N, G, L, K, 3 + grad_output, // B, N, G, D + B, N, G, D, L, Q, grad_im, grad_offset, + block_thread); + break; + case 8: + _flash_deformable_col2im_cuda( + stream, + value, // B, N, G, D + data_spatial_shapes, // L * 2 + data_level_start_index, // L + offset, // B, N, G, L, K, 3 + grad_output, // B, N, G, D + B, N, G, D, L, Q, grad_im, grad_offset, + block_thread); + break; + case 16: + _flash_deformable_col2im_cuda( + stream, + value, // B, N, G, D + data_spatial_shapes, // L * 2 + data_level_start_index, // L + offset, // B, N, G, L, K, 3 + grad_output, // B, N, G, D + B, N, G, D, L, Q, grad_im, grad_offset, + block_thread); + break; + default: + printf("not supported for d_stride > 16 for fp16"); + throw std::invalid_argument("invalid d_stride"); + } + } else { + assert(sizeof(scalar_t) == 4); + switch(d_stride) { + case 1: + _flash_deformable_col2im_cuda( + stream, + value, // B, N, G, D + data_spatial_shapes, // L * 2 + data_level_start_index, // L + offset, // B, N, G, L, K, 3 + grad_output, // B, N, G, D + B, N, G, D, L, Q, grad_im, grad_offset, + block_thread); + break; + case 2: + _flash_deformable_col2im_cuda( + stream, + value, // B, N, G, D + data_spatial_shapes, // L * 2 + data_level_start_index, // L + offset, // B, N, G, L, K, 3 + grad_output, // B, N, G, D + B, N, G, D, L, Q, grad_im, grad_offset, + block_thread); + break; + case 4: + _flash_deformable_col2im_cuda( + stream, + value, // B, N, G, D + data_spatial_shapes, // L * 2 + data_level_start_index, // L + offset, // B, N, G, L, K, 3 + grad_output, // B, N, G, D + B, N, G, D, L, Q, grad_im, grad_offset, + block_thread); + break; + case 8: + _flash_deformable_col2im_cuda( + stream, + value, // B, N, G, D + data_spatial_shapes, // L * 2 + data_level_start_index, // L + offset, // B, N, G, L, K, 3 + grad_output, // B, N, G, D + B, N, G, D, L, Q, grad_im, grad_offset, + block_thread); + break; + default: + printf("not supported for d_stride > 8 for fp32"); + throw std::invalid_argument("invalid d_stride"); + } + } +} + +template +void flash_deformable_col2im_cuda( + cudaStream_t stream, + const scalar_t *value, // B, N, G, D + const int64_t *data_spatial_shapes, // L * 2 + const int64_t *data_level_start_index, // L + const scalar_t *offset, // B, N, G, L, K, 3 + const scalar_t *grad_output, // B, N, G, D + const int B, const int N, const int G, const int D, const int L, + const int Q, const int K, opmath_t *grad_im, opmath_t *grad_offset, + const int d_stride, const int block_thread) { + + switch (K) { + case 4: + flash_deformable_col2im_cuda_inner( + stream, + value, // B, N, G, D + data_spatial_shapes, // L * 2 + data_level_start_index, // L + offset, // B, N, G, L, K, 3 + grad_output, // B, N, G, D + B, N, G, D, L, Q, grad_im, grad_offset, + d_stride, block_thread); + break; + case 8: + flash_deformable_col2im_cuda_inner( + stream, + value, // B, N, G, D + data_spatial_shapes, // L * 2 + data_level_start_index, // L + offset, // B, N, G, L, K, 3 + grad_output, // B, N, G, D + B, N, G, D, L, Q, grad_im, grad_offset, + d_stride, block_thread); + break; + default: + printf("not supported for K not in [4, 8]"); + throw std::invalid_argument("invalid K"); + } +} \ No newline at end of file diff --git a/DCNv4_op/src/cuda/flash_deform_im2col_cuda.cuh b/DCNv4_op/src/cuda/flash_deform_im2col_cuda.cuh new file mode 100644 index 0000000..6c6d9d5 --- /dev/null +++ b/DCNv4_op/src/cuda/flash_deform_im2col_cuda.cuh @@ -0,0 +1,451 @@ +/*! +************************************************************************** +* Deformable DETR +* Copyright (c) 2020 SenseTime. All Rights Reserved. +* Licensed under the Apache License, Version 2.0 [see LICENSE for details] +************************************************************************** +* Modified from DCN (https://github.com/msracver/Deformable-ConvNets) +* Copyright (c) 2018 Microsoft +************************************************************************** +*/ +#include +#include +#include + +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include "common.h" + +template +__global__ void +forward_kernel(const scalar_t *p_value, const int64_t *data_spatial_shapes, + const int64_t *data_level_start_index, const scalar_t *p_offset, + scalar_t *p_output, const int N, const int G, const int D, + const int Q, const int block_multiplier) { + + extern __shared__ char _s[]; + + const int &qi = (blockIdx.x * block_multiplier % Q) + threadIdx.z; + const int &bi = blockIdx.x * block_multiplier / Q; + + const int &di_s = threadIdx.x * d_stride; + const int &gi = threadIdx.y; + + opmath_t p_out_shm[d_stride] = {0.}; + opmath_t *const p_mask_shm = + (opmath_t *)(_s) + (threadIdx.z * G + gi) * L * K; + + const scalar_t *p_offset_ptr = + p_offset + (((bi * Q + qi) * G + gi) * L) * K * 3; + const int mask_length = L * K; + const int num_thread = (D / d_stride); + const int num_iter = mask_length / num_thread; + const int remainder = mask_length - num_iter * num_thread; + for (int i = 0; i < num_iter; i++) { + *(p_mask_shm + num_thread * i + threadIdx.x) = + *(scalar_t *)(p_offset_ptr + L * K * 2 + num_thread * i + threadIdx.x); + } + if (remainder > 0 && threadIdx.x < remainder) { + *(p_mask_shm + num_thread * num_iter + threadIdx.x) = + *(scalar_t *)(p_offset_ptr + L * K * 2 + num_thread * num_iter + + threadIdx.x); + } + + __syncthreads(); + // Calculate softmax over L and K + if (threadIdx.x == 0) { // di = 0 + opmath_t softmax_max = -1e100; + opmath_t softmax_sum = 0.0; + + // get max + for (int j = 0; j < L * K; j++) { + softmax_max = max(softmax_max, p_mask_shm[j]); + } + + // get sumexp + for (int j = 0; j < L * K; j++) { + opmath_t exp_results = exp(p_mask_shm[j] - softmax_max); + p_mask_shm[j] = exp_results; + softmax_sum += exp_results; + } + + // normalize + for (int j = 0; j < L * K; j++) { + p_mask_shm[j] /= softmax_sum; + } + } + + __syncthreads(); + + int offset_idx = 0; + int mask_idx = 0; + const int w_stride = G * D; + const int base_ptr = gi * D + di_s; + + for (int li = 0; li < L; li++) { + const int spatial_h = data_spatial_shapes[li * 2]; + const int spatial_w = data_spatial_shapes[li * 2 + 1]; + const int level_start_id = data_level_start_index[li]; + const scalar_t *p_value_ptr = p_value + (bi * N + level_start_id) * G * D; + + for (int ki = 0; ki < K; ki++) { + const opmath_t loc_w = p_offset_ptr[offset_idx]; + const opmath_t loc_h = p_offset_ptr[offset_idx + 1]; + const opmath_t attn = p_mask_shm[mask_idx]; + const opmath_t h_im = loc_h * spatial_h - 0.5; + const opmath_t w_im = loc_w * spatial_w - 0.5; + if (h_im > -1 && w_im > -1 && h_im < spatial_h && w_im < spatial_w) { + ms_deform_attn_im2col_bilinear( + p_out_shm, p_value_ptr, spatial_h, spatial_w, h_im, w_im, attn, + w_stride, base_ptr); + } + offset_idx += 2; + mask_idx += 1; + } + } + + int out_idx = ((bi * Q + qi) * G + gi) * D + di_s; + + scalar_t *fp16_regs = (scalar_t *)(p_out_shm); +#pragma unroll + for (int ds = 0; ds < d_stride; ds++) { + fp16_regs[ds] = p_out_shm[ds]; + } + + *(transfer_t *)(p_output + out_idx) = *(transfer_t *)(p_out_shm); +} + +template +__global__ void +forward_kernel_reg(const scalar_t *p_value, const int64_t *data_spatial_shapes, + const int64_t *data_level_start_index, const scalar_t *p_offset, + scalar_t *p_output, const int N, const int G, const int D, + const int Q, const int block_multiplier) { + + const int &qi = (blockIdx.x * block_multiplier % Q) + threadIdx.z; + const int &bi = blockIdx.x * block_multiplier / Q; + + const int &di_s = threadIdx.x * d_stride; + const int &gi = threadIdx.y; + + opmath_t p_out_shm[d_stride] = {0.}; + opmath_t p_mask_shm[L*K] = {0.}; + + const scalar_t *p_offset_ptr = + p_offset + (((bi * Q + qi) * G + gi) * L) * K * 3; + + for (int i=0; i < L*K; i++){ + p_mask_shm[i] = *(p_offset_ptr + L * K * 2 + i); + } + + // Calculate softmax over L and K + opmath_t softmax_max = -1e100; + opmath_t softmax_sum = 0.0; + + // get max + for (int j = 0; j < L * K; j++) { + softmax_max = max(softmax_max, p_mask_shm[j]); + } + + // get sumexp + for (int j = 0; j < L * K; j++) { + opmath_t exp_results = exp(p_mask_shm[j] - softmax_max); + p_mask_shm[j] = exp_results; + softmax_sum += exp_results; + } + + // normalize + for (int j = 0; j < L * K; j++) { + p_mask_shm[j] /= softmax_sum; + } + + int offset_idx = 0; + int mask_idx = 0; + const int w_stride = G * D; + const int base_ptr = gi * D + di_s; + + for (int li = 0; li < L; li++) { + const int spatial_h = data_spatial_shapes[li * 2]; + const int spatial_w = data_spatial_shapes[li * 2 + 1]; + const int level_start_id = data_level_start_index[li]; + const scalar_t *p_value_ptr = p_value + (bi * N + level_start_id) * G * D; + + for (int ki = 0; ki < K; ki++) { + const opmath_t loc_w = p_offset_ptr[offset_idx]; + const opmath_t loc_h = p_offset_ptr[offset_idx + 1]; + const opmath_t attn = p_mask_shm[mask_idx]; + const opmath_t h_im = loc_h * spatial_h - 0.5; + const opmath_t w_im = loc_w * spatial_w - 0.5; + if (h_im > -1 && w_im > -1 && h_im < spatial_h && w_im < spatial_w) { + ms_deform_attn_im2col_bilinear( + p_out_shm, p_value_ptr, spatial_h, spatial_w, h_im, w_im, attn, + w_stride, base_ptr); + } + offset_idx += 2; + mask_idx += 1; + } + } + + int out_idx = ((bi * Q + qi) * G + gi) * D + di_s; + + scalar_t *fp16_regs = (scalar_t *)(p_out_shm); +#pragma unroll + for (int ds = 0; ds < d_stride; ds++) { + fp16_regs[ds] = p_out_shm[ds]; + } + + *(transfer_t *)(p_output + out_idx) = *(transfer_t *)(p_out_shm); +} + +template +void _flash_deformable_im2col_cuda( + cudaStream_t stream, + const scalar_t *value, // B, N, G, D + const int64_t *data_spatial_shapes, // L * 2 + const int64_t *data_level_start_index, // L + const scalar_t *offset, // B, N, G, L, K, 3 + scalar_t *output, // B, N, G, D + const int B, const int N, const int G, const int D, const int L, + const int Q, const int block_thread, + const bool _use_reg) { + + assert(D % d_stride == 0); + + const int block_multiplier = block_thread / (D / d_stride) / G;; + assert((B*Q) % block_multiplier == 0); + dim3 num_blocks(B*Q / block_multiplier); + dim3 num_threads(D / d_stride, G, block_multiplier); + + const int shm_size = 0; + + auto kernel = forward_kernel_reg; + + switch (L) { + case 1: + kernel = forward_kernel_reg; + break; + case 2: + kernel = forward_kernel_reg; + break; + case 3: + kernel = forward_kernel_reg; + break; + case 4: + kernel = forward_kernel_reg; + break; + case 5: + kernel = forward_kernel_reg; + break; + default: + printf("L=%ld\n", L); + throw std::invalid_argument("invalid number of scales"); + } + + cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, + shm_size); + + kernel<<>>( + value, data_spatial_shapes, data_level_start_index, offset, output, N, G, + D, Q, block_multiplier); + + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) { + printf("error in flash_deformable_im2col_cuda: %s\n", + cudaGetErrorString(err)); + printf("launch arguments: gridDim=(%d, %d, %d), blockDim=(%d, %d, %d), " + "shm_size=%d, Q=%d\n\n", + num_blocks.x, num_blocks.y, num_blocks.z, num_threads.x, + num_threads.y, num_threads.z, shm_size, Q); + AT_ASSERTM(false, "kernel launch error"); + } +} + +template +void flash_deformable_im2col_cuda_inner( + cudaStream_t stream, + const scalar_t *value, // B, N, G, D + const int64_t *data_spatial_shapes, // L * 2 + const int64_t *data_level_start_index, // L + const scalar_t *offset, // B, N, G, L, K, 3 + scalar_t *output, // B, N, G, D + const int B, const int N, const int G, const int D, const int L, + const int Q, const int d_stride, + const int block_thread, + const bool _use_reg) { + + assert(D % d_stride == 0); + if(sizeof(scalar_t) == 2) { + switch(d_stride) { + case 1: + _flash_deformable_im2col_cuda( + stream, + value, // B, N, G, D + data_spatial_shapes, // L * 2 + data_level_start_index, // L + offset, // B, N, G, L, K, 3 + output, // B, N, G, D + B, N, G, D, L, Q, + block_thread, + _use_reg); + break; + case 2: + _flash_deformable_im2col_cuda( + stream, + value, // B, N, G, D + data_spatial_shapes, // L * 2 + data_level_start_index, // L + offset, // B, N, G, L, K, 3 + output, // B, N, G, D + B, N, G, D, L, Q, + block_thread, + _use_reg); + break; + case 4: + _flash_deformable_im2col_cuda( + stream, + value, // B, N, G, D + data_spatial_shapes, // L * 2 + data_level_start_index, // L + offset, // B, N, G, L, K, 3 + output, // B, N, G, D + B, N, G, D, L, Q, + block_thread, + _use_reg); + break; + case 8: + _flash_deformable_im2col_cuda( + stream, + value, // B, N, G, D + data_spatial_shapes, // L * 2 + data_level_start_index, // L + offset, // B, N, G, L, K, 3 + output, // B, N, G, D + B, N, G, D, L, Q, + block_thread, + _use_reg); + break; + case 16: + _flash_deformable_im2col_cuda( + stream, + value, // B, N, G, D + data_spatial_shapes, // L * 2 + data_level_start_index, // L + offset, // B, N, G, L, K, 3 + output, // B, N, G, D + B, N, G, D, L, Q, + block_thread, + _use_reg); + break; + default: + printf("not supported for d_stride > 16 for fp16"); + throw std::invalid_argument("invalid d_stride"); + } + } else { + assert(sizeof(scalar_t) == 4); + switch(d_stride) { + case 1: + _flash_deformable_im2col_cuda( + stream, + value, // B, N, G, D + data_spatial_shapes, // L * 2 + data_level_start_index, // L + offset, // B, N, G, L, K, 3 + output, // B, N, G, D + B, N, G, D, L, Q, + block_thread, + _use_reg); + break; + case 2: + _flash_deformable_im2col_cuda( + stream, + value, // B, N, G, D + data_spatial_shapes, // L * 2 + data_level_start_index, // L + offset, // B, N, G, L, K, 3 + output, // B, N, G, D + B, N, G, D, L, Q, + block_thread, + _use_reg); + break; + case 4: + _flash_deformable_im2col_cuda( + stream, + value, // B, N, G, D + data_spatial_shapes, // L * 2 + data_level_start_index, // L + offset, // B, N, G, L, K, 3 + output, // B, N, G, D + B, N, G, D, L, Q, + block_thread, + _use_reg); + break; + case 8: + _flash_deformable_im2col_cuda( + stream, + value, // B, N, G, D + data_spatial_shapes, // L * 2 + data_level_start_index, // L + offset, // B, N, G, L, K, 3 + output, // B, N, G, D + B, N, G, D, L, Q, + block_thread, + _use_reg); + break; + default: + printf("not supported for d_stride > 8 for fp32"); + throw std::invalid_argument("invalid d_stride"); + } + } +} + +template +void flash_deformable_im2col_cuda( + cudaStream_t stream, + const scalar_t *value, // B, N, G, D + const int64_t *data_spatial_shapes, // L * 2 + const int64_t *data_level_start_index, // L + const scalar_t *offset, // B, N, G, L, K, 3 + scalar_t *output, // B, N, G, D + const int B, const int N, const int G, const int D, const int L, + const int Q, const int K, const int d_stride, + const int block_thread, + const bool _use_reg) { + switch (K) { + case 4: + flash_deformable_im2col_cuda_inner( + stream, + value, // B, N, G, D + data_spatial_shapes, // L * 2 + data_level_start_index, // L + offset, // B, N, G, L, K, 3 + output, // B, N, G, D + B, N, G, D, L, Q, d_stride, + block_thread, _use_reg); + break; + case 8: + flash_deformable_im2col_cuda_inner( + stream, + value, // B, N, G, D + data_spatial_shapes, // L * 2 + data_level_start_index, // L + offset, // B, N, G, L, K, 3 + output, // B, N, G, D + B, N, G, D, L, Q, d_stride, + block_thread, _use_reg); + break; + default: + printf("not supported for K not in [4, 8]"); + throw std::invalid_argument("invalid K"); + } +} \ No newline at end of file diff --git a/DCNv4_op/src/dcnv4.h b/DCNv4_op/src/dcnv4.h new file mode 100644 index 0000000..0065f00 --- /dev/null +++ b/DCNv4_op/src/dcnv4.h @@ -0,0 +1,107 @@ +/*! +************************************************************************************************** +* Deformable DETR +* Copyright (c) 2020 SenseTime. All Rights Reserved. +* Licensed under the Apache License, Version 2.0 [see LICENSE for details] +************************************************************************************************** +* Modified from +*https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +************************************************************************************************** +*/ + +#pragma once + +#ifdef WITH_CUDA +#include "cuda/dcnv4_cuda.h" +#include "cuda/flash_deform_attn_cuda.h" +#endif + +at::Tensor flash_deform_attn_forward(const at::Tensor &value, + const at::Tensor &spatial_shapes, + const at::Tensor &level_start_index, + const at::Tensor &sampling_loc_attn, + const int im2col_step, const int K, + const int d_stride, const int block_thread) { + if (value.device().is_cuda()) { +#ifdef WITH_CUDA + return flash_deform_attn_cuda_forward(value, spatial_shapes, + level_start_index, + sampling_loc_attn, im2col_step, + K, d_stride, block_thread); +#else + AT_ERROR("Not compiled with GPU support"); +#endif + } + AT_ERROR("Not implemented on the CPU"); +} + +std::vector +flash_deform_attn_backward(const at::Tensor &value, + const at::Tensor &spatial_shapes, + const at::Tensor &level_start_index, + const at::Tensor &sampling_loc_attn, + const at::Tensor &grad_output, + const int im2col_step, + const int K, + const int d_stride, const int block_thread){ + if (value.device().is_cuda()) { +#ifdef WITH_CUDA + return flash_deform_attn_cuda_backward(value, + spatial_shapes, + level_start_index, + sampling_loc_attn, + grad_output, + im2col_step, + K, d_stride, + block_thread); +#else + AT_ERROR("Not compiled with GPU support"); +#endif + } + AT_ERROR("Not implemented on the CPU"); +} + +at::Tensor dcnv4_forward( + const at::Tensor &value, + const at::Tensor &p_offset, + const int kernel_h, const int kernel_w, const int stride_h, + const int stride_w, const int pad_h, const int pad_w, const int dilation_h, + const int dilation_w, const int group, const int group_channels, + const float offset_scale, const int im2col_step, const int remove_center, + const int d_stride, const int block_thread, const bool softmax) { + if (value.device().is_cuda()) { +#ifdef WITH_CUDA + return dcnv4_cuda_forward( + value, p_offset, kernel_h, kernel_w, stride_h, stride_w, pad_h, + pad_w, dilation_h, dilation_w, group, group_channels, offset_scale, + im2col_step, remove_center, d_stride, block_thread, softmax); +#else + AT_ERROR("Not compiled with GPU support"); +#endif + } + AT_ERROR("Not implemented on the CPU"); +} + +std::vector +dcnv4_backward( + const at::Tensor &value, + const at::Tensor &p_offset, + const int kernel_h, const int kernel_w, const int stride_h, + const int stride_w, const int pad_h, const int pad_w, const int dilation_h, + const int dilation_w, const int group, const int group_channels, + const float offset_scale, const int im2col_step, const at::Tensor &grad_output, + const int remove_center, const int d_stride, const int block_thread, + const bool softmax){ + if (value.device().is_cuda()) { +#ifdef WITH_CUDA + return dcnv4_cuda_backward( + value, p_offset, kernel_h, kernel_w, stride_h, stride_w, pad_h, + pad_w, dilation_h, dilation_w, group, group_channels, offset_scale, + im2col_step, grad_output, remove_center, d_stride, block_thread, + softmax); +#else + AT_ERROR("Not compiled with GPU support"); +#endif + } + AT_ERROR("Not implemented on the CPU"); +} \ No newline at end of file diff --git a/DCNv4_op/src/vision.cpp b/DCNv4_op/src/vision.cpp new file mode 100644 index 0000000..c559398 --- /dev/null +++ b/DCNv4_op/src/vision.cpp @@ -0,0 +1,21 @@ +/*! +************************************************************************************************** +* Deformable DETR +* Copyright (c) 2020 SenseTime. All Rights Reserved. +* Licensed under the Apache License, Version 2.0 [see LICENSE for details] +************************************************************************************************** +* Modified from +*https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +************************************************************************************************** +*/ + +#include "dcnv4.h" + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("flash_deform_attn_forward", &flash_deform_attn_forward, + "flash_deform_attn_forward"); + m.def("flash_deform_attn_backward", &flash_deform_attn_backward, + "flash_deform_attn_backward"); + m.def("dcnv4_forward", &dcnv4_forward, "dcnv4_forward"); + m.def("dcnv4_backward", &dcnv4_backward, "dcnv4_backward"); +} diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..a3fac81 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2022 OpenGVLab + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..ae7d1d1 --- /dev/null +++ b/README.md @@ -0,0 +1,158 @@ + + +# [DCNv4](https://arxiv.org/pdf/2401.06197.pdf) + + +## News +- `Jan 15, 2024`: 🚀 Compared with InternImage, the new FlashInternImage powered with DCNv4 has faster inference speed, faster convergence, and better performance!!! +- `Jan 15, 2024`: 🚀 "DCNv4" is released！ + + +## Introduction +We introduce Deformable Convolution v4 (DCNv4), a highly efficient and effective operator designed for a broad spectrum of vision applications. DCNv4 addresses the limitations of its predecessor, DCNv3, with two key enhancements: 1. removing softmax normalization in spatial aggregation to enhance its dynamic property and expressive power and 2. optimizing memory access to minimize redundant operations for speedup. These improvements result in a significantly faster convergence compared to DCNv3 and a substantial increase in processing speed, with DCNv4 achieving more than three times the forward speed. +DCNv4 demonstrates exceptional performance across various tasks, including image classification, instance and semantic segmentation, and notably, image generation. +When integrated into generative models like U-Net in the latent diffusion model, DCNv4 outperforms its baseline, underscoring its possibility to enhance generative models. +In practical applications, replacing DCNv3 with DCNv4 in the InternImage model to create FlashInternImage results in up to 80\% speed increase and further performance improvement without further modifications. +The advancements in speed and efficiency of DCNv4, combined with its robust performance across diverse vision tasks, show its potential as a foundational building block for future vision models. + +## Released Models + + + +
+ ImageNet Image Classification +
+
+ +| name | pretrain | resolution | acc@1 | #param | download | +| :------------: | :----------: | :--------: | :---: | :----: | :---------------------------------------------------------------------------------------------------------------------------------------------------------------: | +| FlashInternImage-T | ImageNet-1K | 224x224 | 83.6 | 30M | [ckpt](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/flash_intern_image_t_1k_224.pth) \| [cfg](classification/configs/flash_intern_image_t_1k_224.yaml) | +| FlashInternImage-S | ImageNet-1K | 224x224 | 84.4 | 50M | [ckpt](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/flash_intern_image_s_1k_224.pth) \| [cfg](classification/configs/flash_intern_image_s_1k_224.yaml) | +| FlashInternImage-B | ImageNet-1K | 224x224 | 84.9 | 97M | [ckpt](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/flash_intern_image_b_1k_224.pth) \| [cfg](classification/configs/flash_intern_image_b_1k_224.yaml) | +| FlashInternImage-L | ImageNet-22K | 384x384 | 88.1 | 223M | [ckpt](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/flash_internimage_l_22kto1k_384.pth) \| [cfg](classification/configs/flash_intern_image_l_22kto1k_384.yaml) | +
+ +
+ +
+ COCO Object Detection and Instance Segmentation +
+
+ +| backbone |method | schd | box mAP | mask mAP |Config | Download | +| :-----------------:| :----------: | :---------: | :-----: |:------: | :-----: | :---: | +| FlashInternImage-T |Mask-RCNN| 1x | 48.0 | 43.1 | [config](./detection/configs/coco/mask_rcnn_flash_intern_image_t_fpn_1x_coco.py) | [ckpt](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/mask_rcnn_flash_internimage_t_fpn_1x_coco.pth) \| [log](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/mask_rcnn_flash_internimage_t_fpn_1x_coco.log) | +| FlashInternImage-T |Mask-RCNN | 3x | 49.5 | 44.0 | [config](./detection/configs/coco/mask_rcnn_flash_intern_image_t_fpn_3x_coco.py) | [ckpt](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/mask_rcnn_flash_internimage_t_fpn_3x_coco.pth) \| [log](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/mask_rcnn_flash_internimage_t_fpn_3x_coco.log) | +| FlashInternImage-S |Mask-RCNN| 1x | 49.2 | 44.0 | [config](./detection/configs/coco/mask_rcnn_flash_intern_image_s_fpn_1x_coco.py) | [ckpt](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/mask_rcnn_flash_internimage_s_fpn_1x_coco.pth) \| [log](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/mask_rcnn_flash_internimage_s_fpn_1x_coco.log) | +| FlashInternImage-S |Mask-RCNN | 3x | 50.5 | 44.9 | [config](./detection/configs/coco/mask_rcnn_flash_intern_image_s_fpn_3x_coco.py) | [ckpt](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/mask_rcnn_flash_internimage_s_fpn_3x_coco.pth) \| [log](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/mask_rcnn_flash_internimage_s_fpn_3x_coco.log) | +| FlashInternImage-B |Mask-RCNN | 1x | 50.1 | 44.5 | [config](./detection/configs/coco/mask_rcnn_flash_intern_image_b_fpn_1x_coco.py) | [ckpt](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/mask_rcnn_flash_internimage_b_fpn_1x_coco.pth) \| [log](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/mask_rcnn_flash_internimage_b_fpn_1x_coco.log) | +| FlashInternImage-B |Mask-RCNN| 3x | 50.6 | 45.4 | [config](./detection/configs/coco/mask_rcnn_flash_intern_image_b_fpn_3x_coco.py)| [ckpt](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/mask_rcnn_flash_internimage_b_fpn_3x_coco.pth) \| [log](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/mask_rcnn_flash_internimage_b_fpn_3x_coco.log) | + +| backbone | method| schd | box mAP | mask mAP | Config | Download | +| :------------:| :---------: | :---------: | :-----: | :------: | :---: | :---: | +| FlashInternImage-L |Cascade Mask R-CNN | 1x | 55.6 | 48.2 | [config](./detection/configs/coco/cascade_flash_intern_image_l_fpn_1x_coco.py) | [ckpt](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/cascade_flash_internimage_l_fpn_1x_coco.pth) \| [log](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/cascade_flash_internimage_l_fpn_1x_coco.log) +| FlashInternImage-L |Cascade Mask R-CNN | 3x | 56.7 | 48.9 | [config](./detection/configs/coco/cascade_flash_intern_image_l_fpn_3x_coco.py) | [ckpt](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/cascade_flash_internimage_l_fpn_3x_coco.pth) | + +| backbone |method | lr type | pretrain | schd | box mAP | Config | Download | +| :------------: | :---------: | :---------: |:---------: | :---------: | :-----: | :---: | :-----: | +| FlashInternImage-T |DINO| layer-wise lr | ImageNet-1K | 1x | 54.7 | [config](./detection/configs/coco/dino_4scale_flash_internimage_t_1x_coco.py) | [ckpt](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/dino_4scale_flash_internimage_t_1x_coco.pth) \| [log](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/dino_4scale_flash_internimage_t_1x_coco.json) | +| FlashInternImage-S |DINO | layer-wise lr | ImageNet-1K | 1x | 55.3 | [config](./detection/configs/coco/dino_4scale_flash_internimage_s_1x_coco.py) | [ckpt](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/dino_4scale_flash_internimage_s_1x_coco.pth) \| [log](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/dino_4scale_flash_internimage_s_1x_coco.log) | +| FlashInternImage-B |DINO| layer-wise lr | ImageNet-1K | 1x | 56.0 | [config](./detection/configs/coco/dino_4scale_flash_internimage_b_1x_coco.py) | [ckpt](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/dino_4scale_flash_internimage_b_1x_coco.pth) \| [log](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/dino_4scale_flash_internimage_b_1x_coco.log) | +| FlashInternImage-L |DINO | 0.1x backbone lr | ImageNet-22K | 1x | 58.8 | [config](./detection/configs/coco/dino_4scale_flash_internimage_l_1x_coco.py) | [ckpt](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/dino_4scale_flash_internimage_l_1x_coco.pth) \| [log](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/dino_4scale_flash_internimage_l_1x_coco.log) | + + +
+ +
+ + +
+ ADE20K Semantic Segmentation +
+
+ +| backbone |method | resolution | mIoU (ss/ms) | Config | Download | +|:--------------:|:----------:|:----------:|:-----------:|:-----------:|:----------: +| FlashInternImage-T|UperNet | 512x512 | 49.3 / 50.3 | [config](./segmentation/configs/ade20k/upernet_flash_internimage_t_512_160k_ade20k.py) | [ckpt](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/upernet_flash_internimage_t_512_160k_ade20k.pth) \| [log](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/upernet_flash_internimage_t_512_160k_ade20k.log) | +| FlashInternImage-S |UperNet | 512x512 | 50.6 / 51.6 | [config](./segmentation/configs/ade20k/upernet_flash_internimage_s_512_160k_ade20k.py) | [ckpt](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/upernet_flash_internimage_s_512_160k_ade20k.pth) \| [log](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/upernet_flash_internimage_s_512_160k_ade20k.log) | +| FlashInternImage-B |UperNet | 512x512 | 52.0 / 52.6 | [config](./segmentation/configs/ade20k/upernet_flash_internimage_b_512_160k_ade20k.py) | [ckpt](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/upernet_flash_internimage_b_512_160k_ade20k.pth) \| [log](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/upernet_flash_internimage_s_512_160k_ade20k.log) | +| FlashInternImage-L |UperNet | 640x640 | 55.6 / 56.0 | [config](./segmentation/configs/ade20k/upernet_flash_internimage_l_640_160k_ade20k.py)| [ckpt](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/upernet_flash_internimage_l_640_160k_ade20k.pth) \| [log](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/upernet_flash_internimage_l_640_160k_ade20k.log) | + + +| backbone |method | resolution | mIoU (ss) | Config | Download | +|:--------------:|:----------:|:----------:|:-----------:|:-----------:|:----------: +| FlashInternImage-T |Mask2Former| 512x512 | 51.2 | [config](./segmentation/configs/ade20k/mask2former_flash_internimage_t_512_160k_ade20k_ss.py) | [ckpt](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/mask2former_flash_internimage_t_512_160k_ade20k_ss.pth) \| [log](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/mask2former_flash_internimage_t_512_160k_ade20k_ss.log) | +| FlashInternImage-S |Mask2Former| 640x640 | 52.6 | [config](./segmentation/configs/ade20k/mask2former_flash_internimage_s_640_160k_ade20k_ss.py) | [ckpt](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/mask2former_flash_internimage_s_640_160k_ade20k_ss.pth) \| [log](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/mask2former_flash_internimage_s_640_160k_ade20k_ss.log) | +| FlashInternImage-B |Mask2Former| 640x640 | 53.4 | [config](./segmentation/configs/ade20k/mask2former_flash_internimage_b_640_160k_ade20k_ss.py) | [ckpt](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/mask2former_flash_internimage_b_640_160k_ade20k_ss.pth) \| [log](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/mask2former_flash_internimage_b_640_160k_ade20k_ss.log) | +| FlashInternImage-L |Mask2Former| 640x640 | 56.7 | [config](./segmentation/configs/ade20k/mask2former_flash_internimage_l_640_160k_ade20k_ss.py)| [ckpt](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/mask2former_flash_internimage_l_640_160k_ade20k_ss.pth) \| [log](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/mask2former_flash_internimage_l_640_160k_ade20k_ss.log) | + + + +
+ +
+ +## Citations + +If this work is helpful for your research, please consider citing the following BibTeX entry. + +```bibtex + +@article{xiong2024efficient, + title={Efficient Deformable ConvNets: Rethinking Dynamic and Sparse Operator for Vision Applications}, + author={Yuwen Xiong and Zhiqi Li and Yuntao Chen and Feng Wang and Xizhou Zhu and Jiapeng Luo and Wenhai Wang and Tong Lu and Hongsheng Li and Yu Qiao and Lewei Lu and Jie Zhou and Jifeng Dai}, + journal={arXiv preprint arXiv:2401.06197}, + year={2024} +} + +@article{wang2022internimage, + title={InternImage: Exploring Large-Scale Vision Foundation Models with Deformable Convolutions}, + author={Wang, Wenhai and Dai, Jifeng and Chen, Zhe and Huang, Zhenhang and Li, Zhiqi and Zhu, Xizhou and Hu, Xiaowei and Lu, Tong and Lu, Lewei and Li, Hongsheng and others}, + journal={arXiv preprint arXiv:2211.05778}, + year={2022} +} + +@inproceedings{zhu2022uni, + title={Uni-perceiver: Pre-training unified architecture for generic perception for zero-shot and few-shot tasks}, + author={Zhu, Xizhou and Zhu, Jinguo and Li, Hao and Wu, Xiaoshi and Li, Hongsheng and Wang, Xiaohua and Dai, Jifeng}, + booktitle={CVPR}, + pages={16804--16815}, + year={2022} +} + +@article{zhu2022uni, + title={Uni-perceiver-moe: Learning sparse generalist models with conditional moes}, + author={Zhu, Jinguo and Zhu, Xizhou and Wang, Wenhai and Wang, Xiaohua and Li, Hongsheng and Wang, Xiaogang and Dai, Jifeng}, + journal={arXiv preprint arXiv:2206.04674}, + year={2022} +} + +@article{li2022uni, + title={Uni-Perceiver v2: A Generalist Model for Large-Scale Vision and Vision-Language Tasks}, + author={Li, Hao and Zhu, Jinguo and Jiang, Xiaohu and Zhu, Xizhou and Li, Hongsheng and Yuan, Chun and Wang, Xiaohua and Qiao, Yu and Wang, Xiaogang and Wang, Wenhai and others}, + journal={arXiv preprint arXiv:2211.09808}, + year={2022} +} + +@article{yang2022bevformer, + title={BEVFormer v2: Adapting Modern Image Backbones to Bird's-Eye-View Recognition via Perspective Supervision}, + author={Yang, Chenyu and Chen, Yuntao and Tian, Hao and Tao, Chenxin and Zhu, Xizhou and Zhang, Zhaoxiang and Huang, Gao and Li, Hongyang and Qiao, Yu and Lu, Lewei and others}, + journal={arXiv preprint arXiv:2211.10439}, + year={2022} +} + +@article{su2022towards, + title={Towards All-in-one Pre-training via Maximizing Multi-modal Mutual Information}, + author={Su, Weijie and Zhu, Xizhou and Tao, Chenxin and Lu, Lewei and Li, Bin and Huang, Gao and Qiao, Yu and Wang, Xiaogang and Zhou, Jie and Dai, Jifeng}, + journal={arXiv preprint arXiv:2211.09807}, + year={2022} +} + +@inproceedings{li2022bevformer, + title={Bevformer: Learning bird’s-eye-view representation from multi-camera images via spatiotemporal transformers}, + author={Li, Zhiqi and Wang, Wenhai and Li, Hongyang and Xie, Enze and Sima, Chonghao and Lu, Tong and Qiao, Yu and Dai, Jifeng}, + booktitle={ECCV}, + pages={1--18}, + year={2022}, +} +``` diff --git a/classification/README.md b/classification/README.md new file mode 100644 index 0000000..e671007 --- /dev/null +++ b/classification/README.md @@ -0,0 +1,190 @@ +# FlashInternImage for Image Classification + +This folder contains the implementation of the FlashInternImage for image classification. + +## Usage + +### Install + +- Clone this repo: + +```bash +git clone https://github.com/OpenGVLab/DCNv4.git +cd DCNv4 +``` + +- Create a conda virtual environment and activate it: + +```bash +conda create -n internimage python=3.7 -y +conda activate internimage +``` + +- Install `CUDA>=10.2` with `cudnn>=7` following + the [official installation instructions](https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html) +- Install `PyTorch>=1.10.0` and `torchvision>=0.9.0` with `CUDA>=10.2`: + +For examples, to install torch==1.11 with CUDA==11.3: +```bash +pip install torch==1.11.0+cu113 torchvision==0.12.0+cu113 -f https://download.pytorch.org/whl/torch_stable.html +``` + +- Install `timm==0.6.11` and `mmcv-full==1.5.0`: + +```bash +pip install -U openmim +mim install mmcv-full==1.5.0 +pip install timm==0.6.11 mmdet==2.28.1 +``` + +- Install other requirements: + +```bash +pip install opencv-python termcolor yacs pyyaml scipy +``` + +- Install DCNv4 +```bash +pip install DCNv4==latest +``` + +### Data Preparation + +We use standard ImageNet dataset, you can download it from http://image-net.org/. We provide the following two ways to +load data: + +- For standard folder dataset, move validation images to labeled sub-folders. The file structure should look like: + ```bash + $ tree data + imagenet + ├── train + │ ├── class1 + │ │ ├── img1.jpeg + │ │ ├── img2.jpeg + │ │ └── ... + │ ├── class2 + │ │ ├── img3.jpeg + │ │ └── ... + │ └── ... + └── val + ├── class1 + │ ├── img4.jpeg + │ ├── img5.jpeg + │ └── ... + ├── class2 + │ ├── img6.jpeg + │ └── ... + └── ... + + ``` +- To boost the slow speed when reading images from massive small files, we also support zipped ImageNet, which includes + four files: + - `train.zip`, `val.zip`: which store the zipped folder for train and validate splits. + - `train.txt`, `val.txt`: which store the relative path in the corresponding zip file and ground truth + label. Make sure the data folder looks like this: + + ```bash + $ tree data + data + └── ImageNet-Zip + ├── train_map.txt + ├── train.zip + ├── val_map.txt + └── val.zip + + $ head -n 5 meta_data/val.txt + ILSVRC2012_val_00000001.JPEG 65 + ILSVRC2012_val_00000002.JPEG 970 + ILSVRC2012_val_00000003.JPEG 230 + ILSVRC2012_val_00000004.JPEG 809 + ILSVRC2012_val_00000005.JPEG 516 + + $ head -n 5 meta_data/train.txt + n01440764/n01440764_10026.JPEG 0 + n01440764/n01440764_10027.JPEG 0 + n01440764/n01440764_10029.JPEG 0 + n01440764/n01440764_10040.JPEG 0 + n01440764/n01440764_10042.JPEG 0 + ``` +- For ImageNet-22K dataset, make a folder named `fall11_whole` and move all images to labeled sub-folders in this + folder. Then download the train-val split + file ([ILSVRC2011fall_whole_map_train.txt](https://github.com/SwinTransformer/storage/releases/download/v2.0.1/ILSVRC2011fall_whole_map_train.txt) + & [ILSVRC2011fall_whole_map_val.txt](https://github.com/SwinTransformer/storage/releases/download/v2.0.1/ILSVRC2011fall_whole_map_val.txt)) + , and put them in the parent directory of `fall11_whole`. The file structure should look like: + + ```bash + $ tree imagenet22k/ + imagenet22k/ + └── fall11_whole + ├── n00004475 + ├── n00005787 + ├── n00006024 + ├── n00006484 + └── ... + ``` + +### Evaluation + +To evaluate a pretrained `FlashInternImage` on ImageNet val, run: + +```bash +python -m torch.distributed.launch --nproc_per_node --master_port 12345 main.py --eval \ +--cfg --resume --data-path +``` + +For example, to evaluate the `FlashInternImage-B` with a single GPU: + +```bash +python -m torch.distributed.launch --nproc_per_node 1 --master_port 12345 main.py --eval \ +--cfg configs/flash_intern_image_b_1k_224.yaml --resume flash_intern_image_b_1k_224.pth --data-path +``` + +### Training from Scratch on ImageNet-1K + +> The paper results were obtained from models trained with configs in `configs/without_lr_decay`. + +To train an `InternImage` on ImageNet from scratch, run: + +```bash +python -m torch.distributed.launch --nproc_per_node --master_port 12345 main.py \ +--cfg --data-path [--batch-size --output --tag ] +``` + +### Manage Jobs with Slurm. + +For example, to train `FlashInternImage` with 8 GPU on a single node for 300 epochs, run: + +`FlashInternImage-T`: + +```bash +GPUS=8 sh train_in1k.sh configs/flash_intern_image_t_1k_224.yaml --resume flash_intern_image_t_1k_224.pth --eval +``` + +`FlashInternImage-S`: + +```bash +GPUS=8 sh train_in1k.sh configs/flash_intern_image_s_1k_224.yaml --resume flash_intern_image_s_1k_224.pth --eval +``` + + + + + diff --git a/classification/config.py b/classification/config.py new file mode 100644 index 0000000..4d452dd --- /dev/null +++ b/classification/config.py @@ -0,0 +1,302 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + +import os +import yaml +from yacs.config import CfgNode as CN + +_C = CN() + +# Base config files +_C.BASE = [''] + +# ----------------------------------------------------------------------------- +# Data settings +# ----------------------------------------------------------------------------- +_C.DATA = CN() +# Batch size for a single GPU, could be overwritten by command line argument +_C.DATA.BATCH_SIZE = 128 +# Path to dataset, could be overwritten by command line argument +_C.DATA.DATA_PATH = '' +# Dataset name +_C.DATA.DATASET = 'imagenet' +# Input image size +_C.DATA.IMG_SIZE = 224 +# Interpolation to resize image (random, bilinear, bicubic) +_C.DATA.INTERPOLATION = 'bicubic' +# Use zipped dataset instead of folder dataset +# could be overwritten by command line argument +_C.DATA.ZIP_MODE = False +# Cache Data in Memory, could be overwritten by command line argument +_C.DATA.CACHE_MODE = 'part' +# Pin CPU memory in DataLoader for more efficient (sometimes) transfer to GPU. +_C.DATA.PIN_MEMORY = True +# Number of data loading threads +_C.DATA.NUM_WORKERS = 8 +# Load data to memory +_C.DATA.IMG_ON_MEMORY = False + +# ----------------------------------------------------------------------------- +# Model settings +# ----------------------------------------------------------------------------- +_C.MODEL = CN() +# Model type +_C.MODEL.TYPE = 'INTERN_IMAGE' +# Model name +_C.MODEL.NAME = 'intern_image' +# Pretrained weight from checkpoint, could be imagenet22k pretrained weight +# could be overwritten by command line argument +_C.MODEL.PRETRAINED = '' +# Checkpoint to resume, could be overwritten by command line argument +_C.MODEL.RESUME = '' +# Number of classes, overwritten in data preparation +_C.MODEL.NUM_CLASSES = 1000 +# Dropout rate +_C.MODEL.DROP_RATE = 0.0 +# Drop path rate +_C.MODEL.DROP_PATH_RATE = 0.1 +# Drop path type +_C.MODEL.DROP_PATH_TYPE = 'linear' # linear, uniform +# Label Smoothing +_C.MODEL.LABEL_SMOOTHING = 0.1 + +# INTERN_IMAGE parameters +_C.MODEL.INTERN_IMAGE = CN() +_C.MODEL.INTERN_IMAGE.DEPTHS = [4, 4, 18, 4] +_C.MODEL.INTERN_IMAGE.GROUPS = [4, 8, 16, 32] +_C.MODEL.INTERN_IMAGE.CHANNELS = 64 +_C.MODEL.INTERN_IMAGE.LAYER_SCALE = None +_C.MODEL.INTERN_IMAGE.OFFSET_SCALE = 1.0 +_C.MODEL.INTERN_IMAGE.MLP_RATIO = 4.0 +_C.MODEL.INTERN_IMAGE.CORE_OP = 'DCNv3' +_C.MODEL.INTERN_IMAGE.POST_NORM = False +_C.MODEL.INTERN_IMAGE.RES_POST_NORM = False +_C.MODEL.INTERN_IMAGE.DW_KERNEL_SIZE = None +_C.MODEL.INTERN_IMAGE.USE_CLIP_PROJECTOR = False +_C.MODEL.INTERN_IMAGE.LEVEL2_POST_NORM = False +_C.MODEL.INTERN_IMAGE.LEVEL2_POST_NORM_BLOCK_IDS = None +_C.MODEL.INTERN_IMAGE.CENTER_FEATURE_SCALE = False + +# FLASH_INTERN_IMAGE parameters +_C.MODEL.FLASH_INTERN_IMAGE = CN() +_C.MODEL.FLASH_INTERN_IMAGE.DEPTHS = [4, 4, 18, 4] +_C.MODEL.FLASH_INTERN_IMAGE.GROUPS = [4, 8, 16, 32] +_C.MODEL.FLASH_INTERN_IMAGE.CHANNELS = 64 +_C.MODEL.FLASH_INTERN_IMAGE.LAYER_SCALE = None +_C.MODEL.FLASH_INTERN_IMAGE.OFFSET_SCALE = 1.0 +_C.MODEL.FLASH_INTERN_IMAGE.MLP_RATIO = 4.0 +_C.MODEL.FLASH_INTERN_IMAGE.CORE_OP = 'DCNv4' +_C.MODEL.FLASH_INTERN_IMAGE.POST_NORM = False +_C.MODEL.FLASH_INTERN_IMAGE.RES_POST_NORM = False +_C.MODEL.FLASH_INTERN_IMAGE.DW_KERNEL_SIZE = None +_C.MODEL.FLASH_INTERN_IMAGE.USE_CLIP_PROJECTOR = False +_C.MODEL.FLASH_INTERN_IMAGE.LEVEL2_POST_NORM = False +_C.MODEL.FLASH_INTERN_IMAGE.LEVEL2_POST_NORM_BLOCK_IDS = None +_C.MODEL.FLASH_INTERN_IMAGE.CENTER_FEATURE_SCALE = False +_C.MODEL.FLASH_INTERN_IMAGE.MLP_FC2_BIAS = False +_C.MODEL.FLASH_INTERN_IMAGE.DCN_OUTPUT_BIAS = False +# ----------------------------------------------------------------------------- +# Training settings +# ----------------------------------------------------------------------------- +_C.TRAIN = CN() +_C.TRAIN.START_EPOCH = 0 +_C.TRAIN.EPOCHS = 300 +_C.TRAIN.WARMUP_EPOCHS = 20 +_C.TRAIN.WEIGHT_DECAY = 0.05 +_C.TRAIN.BASE_LR = 5e-4 +_C.TRAIN.WARMUP_LR = 5e-7 +_C.TRAIN.MIN_LR = 5e-6 +# Clip gradient norm +_C.TRAIN.CLIP_GRAD = 5.0 +# Auto resume from latest checkpoint +_C.TRAIN.AUTO_RESUME = True +# Gradient accumulation steps +# could be overwritten by command line argument +_C.TRAIN.ACCUMULATION_STEPS = 0 +# Whether to use gradient checkpointing to save memory +# could be overwritten by command line argument +_C.TRAIN.USE_CHECKPOINT = False + +# LR scheduler +_C.TRAIN.LR_SCHEDULER = CN() +_C.TRAIN.LR_SCHEDULER.NAME = 'cosine' +# Epoch interval to decay LR, used in StepLRScheduler +_C.TRAIN.LR_SCHEDULER.DECAY_EPOCHS = 30 +# LR decay rate, used in StepLRScheduler +_C.TRAIN.LR_SCHEDULER.DECAY_RATE = 0.1 + +# Optimizer +_C.TRAIN.OPTIMIZER = CN() +_C.TRAIN.OPTIMIZER.NAME = 'adamw' +# Optimizer Epsilon +_C.TRAIN.OPTIMIZER.EPS = 1e-8 +# Optimizer Betas +_C.TRAIN.OPTIMIZER.BETAS = (0.9, 0.999) +# SGD momentum +_C.TRAIN.OPTIMIZER.MOMENTUM = 0.9 +# ZeRO +_C.TRAIN.OPTIMIZER.USE_ZERO = False +# freeze backbone +_C.TRAIN.OPTIMIZER.FREEZE_BACKBONE = None +# dcn lr +_C.TRAIN.OPTIMIZER.DCN_LR_MUL = None + +# EMA +_C.TRAIN.EMA = CN() +_C.TRAIN.EMA.ENABLE = False +_C.TRAIN.EMA.DECAY = 0.9998 + +# LR_LAYER_DECAY +_C.TRAIN.LR_LAYER_DECAY = False +_C.TRAIN.LR_LAYER_DECAY_RATIO = 0.875 + +# FT head init weights +_C.TRAIN.RAND_INIT_FT_HEAD = False + +# ----------------------------------------------------------------------------- +# Augmentation settings +# ----------------------------------------------------------------------------- +_C.AUG = CN() +# Color jitter factor +_C.AUG.COLOR_JITTER = 0.4 +# Use AutoAugment policy. "v0" or "original" +_C.AUG.AUTO_AUGMENT = 'rand-m9-mstd0.5-inc1' +# Random erase prob +_C.AUG.REPROB = 0.25 +# Random erase mode +_C.AUG.REMODE = 'pixel' +# Random erase count +_C.AUG.RECOUNT = 1 +# Mixup alpha, mixup enabled if > 0 +_C.AUG.MIXUP = 0.8 +# Cutmix alpha, cutmix enabled if > 0 +_C.AUG.CUTMIX = 1.0 +# Cutmix min/max ratio, overrides alpha and enables cutmix if set +_C.AUG.CUTMIX_MINMAX = None +# Probability of performing mixup or cutmix when either/both is enabled +_C.AUG.MIXUP_PROB = 1.0 +# Probability of switching to cutmix when both mixup and cutmix enabled +_C.AUG.MIXUP_SWITCH_PROB = 0.5 +# How to apply mixup/cutmix params. Per "batch", "pair", or "elem" +_C.AUG.MIXUP_MODE = 'batch' +# RandomResizedCrop +_C.AUG.RANDOM_RESIZED_CROP = False +_C.AUG.MEAN = (0.485, 0.456, 0.406) +_C.AUG.STD = (0.229, 0.224, 0.225) + +# ----------------------------------------------------------------------------- +# Testing settings +# ----------------------------------------------------------------------------- +_C.TEST = CN() +# Whether to use center crop when testing +_C.TEST.CROP = True + +# Whether to use SequentialSampler as validation sampler +_C.TEST.SEQUENTIAL = False + +# ----------------------------------------------------------------------------- +# Misc +# ----------------------------------------------------------------------------- +# Mixed precision opt level, if O0, no amp is used ('O0', 'O1', 'O2') +# overwritten by command line argument +_C.AMP_OPT_LEVEL = '' +# Path to output folder, overwritten by command line argument +_C.OUTPUT = '' +# Tag of experiment, overwritten by command line argument +_C.TAG = 'default' +# Frequency to save checkpoint +_C.SAVE_FREQ = 1 +# Frequency to logging info +_C.PRINT_FREQ = 10 +# eval freq +_C.EVAL_FREQ = 1 +# Fixed random seed +_C.SEED = 0 +# Perform evaluation only, overwritten by command line argument +_C.EVAL_MODE = False +# Test throughput only, overwritten by command line argument +_C.THROUGHPUT_MODE = False +# local rank for DistributedDataParallel, given by command line argument +_C.LOCAL_RANK = 0 +_C.EVAL_22K_TO_1K = False + +_C.AMP_TYPE = 'float16' + + +def _update_config_from_file(config, cfg_file): + config.defrost() + with open(cfg_file, 'r') as f: + yaml_cfg = yaml.load(f, Loader=yaml.FullLoader) + + for cfg in yaml_cfg.setdefault('BASE', ['']): + if cfg: + _update_config_from_file( + config, os.path.join(os.path.dirname(cfg_file), cfg)) + print('=> merge config from {}'.format(cfg_file)) + config.merge_from_file(cfg_file) + config.freeze() + + +def update_config(config, args): + _update_config_from_file(config, args.cfg) + + config.defrost() + if hasattr(args, 'opts') and args.opts: + config.merge_from_list(args.opts) + + # merge from specific arguments + if hasattr(args, 'batch_size') and args.batch_size: + config.DATA.BATCH_SIZE = args.batch_size + if hasattr(args, 'dataset') and args.dataset: + config.DATA.DATASET = args.dataset + if hasattr(args, 'data_path') and args.data_path: + config.DATA.DATA_PATH = args.data_path + if hasattr(args, 'zip') and args.zip: + config.DATA.ZIP_MODE = True + if hasattr(args, 'cache_mode') and args.cache_mode: + config.DATA.CACHE_MODE = args.cache_mode + if hasattr(args, 'pretrained') and args.pretrained: + config.MODEL.PRETRAINED = args.pretrained + if hasattr(args, 'resume') and args.resume: + config.MODEL.RESUME = args.resume + if hasattr(args, 'accumulation_steps') and args.accumulation_steps: + config.TRAIN.ACCUMULATION_STEPS = args.accumulation_steps + if hasattr(args, 'use_checkpoint') and args.use_checkpoint: + config.TRAIN.USE_CHECKPOINT = True + if hasattr(args, 'amp_opt_level') and args.amp_opt_level: + config.AMP_OPT_LEVEL = args.amp_opt_level + if hasattr(args, 'output') and args.output: + config.OUTPUT = args.output + if hasattr(args, 'tag') and args.tag: + config.TAG = args.tag + if hasattr(args, 'eval') and args.eval: + config.EVAL_MODE = True + if hasattr(args, 'throughput') and args.throughput: + config.THROUGHPUT_MODE = True + if hasattr(args, 'save_ckpt_num') and args.save_ckpt_num: + config.SAVE_CKPT_NUM = args.save_ckpt_num + if hasattr(args, 'use_zero') and args.use_zero: + config.TRAIN.OPTIMIZER.USE_ZERO = True + # set local rank for distributed training + if hasattr(args, 'local_rank') and args.local_rank: + config.LOCAL_RANK = args.local_rank + + # output folder + config.MODEL.NAME = args.cfg.split('/')[-1].replace('.yaml', '') + config.OUTPUT = os.path.join(config.OUTPUT, config.MODEL.NAME) + # config.OUTPUT = os.path.join(config.OUTPUT, config.MODEL.NAME, config.TAG) + + config.freeze() + + +def get_config(args): + """Get a yacs CfgNode object with default values.""" + # Return a clone so that the defaults will not be altered + # This is for the "local variable" use pattern + config = _C.clone() + update_config(config, args) + + return config diff --git a/classification/configs/flash_intern_image_b_1k_224.yaml b/classification/configs/flash_intern_image_b_1k_224.yaml new file mode 100755 index 0000000..77fb1d1 --- /dev/null +++ b/classification/configs/flash_intern_image_b_1k_224.yaml @@ -0,0 +1,20 @@ +DATA: + IMG_ON_MEMORY: False +MODEL: + TYPE: flash_intern_image + DROP_PATH_RATE: 0.5 + FLASH_INTERN_IMAGE: + CORE_OP: 'DCNv4' + DEPTHS: [4, 4, 21, 4] + GROUPS: [7, 14, 28, 56] + CHANNELS: 112 + LAYER_SCALE: 1e-5 + OFFSET_SCALE: 0.5 + MLP_RATIO: 4.0 + POST_NORM: True + DW_KERNEL_SIZE: 3 +TRAIN: + EMA: + ENABLE: True + DECAY: 0.9999 + BASE_LR: 5e-4 \ No newline at end of file diff --git a/classification/configs/flash_intern_image_l_22kto1k_384.yaml b/classification/configs/flash_intern_image_l_22kto1k_384.yaml new file mode 100755 index 0000000..fe6f005 --- /dev/null +++ b/classification/configs/flash_intern_image_l_22kto1k_384.yaml @@ -0,0 +1,40 @@ +DATA: + IMG_SIZE: 384 + IMG_ON_MEMORY: False +AUG: + MIXUP: 0.0 + CUTMIX: 0.0 + REPROB: 0.0 +MODEL: + TYPE: flash_intern_image + DROP_PATH_RATE: 0.1 + LABEL_SMOOTHING: 0.3 + FLASH_INTERN_IMAGE: + CORE_OP: 'DCNv4' + DEPTHS: [5, 5, 22, 5] + GROUPS: [10, 20, 40, 80] + CHANNELS: 160 + LAYER_SCALE: 1e-5 + OFFSET_SCALE: 2.0 + MLP_RATIO: 4.0 + POST_NORM: True + DW_KERNEL_SIZE: 3 + DCN_OUTPUT_BIAS: True + MLP_FC2_BIAS: True +TRAIN: + EMA: + ENABLE: true + DECAY: 0.9999 + EPOCHS: 20 + WARMUP_EPOCHS: 2 + WEIGHT_DECAY: 0.05 + BASE_LR: 2e-05 # 512 + WARMUP_LR: .0 + MIN_LR: .0 + LR_LAYER_DECAY: true + LR_LAYER_DECAY_RATIO: 0.9 + USE_CHECKPOINT: true + OPTIMIZER: + DCN_LR_MUL: 0.1 +AMP_OPT_LEVEL: O0 +EVAL_FREQ: 1 \ No newline at end of file diff --git a/classification/configs/flash_intern_image_s_1k_224.yaml b/classification/configs/flash_intern_image_s_1k_224.yaml new file mode 100755 index 0000000..403027b --- /dev/null +++ b/classification/configs/flash_intern_image_s_1k_224.yaml @@ -0,0 +1,20 @@ +DATA: + IMG_ON_MEMORY: False +MODEL: + TYPE: flash_intern_image + DROP_PATH_RATE: 0.4 + FLASH_INTERN_IMAGE: + CORE_OP: 'DCNv4' + DEPTHS: [4, 4, 21, 4] + GROUPS: [5, 10, 20, 40] + CHANNELS: 80 + LAYER_SCALE: 1e-5 + OFFSET_SCALE: 1.0 + MLP_RATIO: 4.0 + POST_NORM: True + DW_KERNEL_SIZE: 3 +TRAIN: + EMA: + ENABLE: True + DECAY: 0.9999 + BASE_LR: 5e-4 diff --git a/classification/configs/flash_intern_image_t_1k_224.yaml b/classification/configs/flash_intern_image_t_1k_224.yaml new file mode 100755 index 0000000..4c1e2a8 --- /dev/null +++ b/classification/configs/flash_intern_image_t_1k_224.yaml @@ -0,0 +1,17 @@ +DATA: + IMG_ON_MEMORY: False +MODEL: + TYPE: flash_intern_image + DROP_PATH_RATE: 0.1 + FLASH_INTERN_IMAGE: + CORE_OP: 'DCNv4' + DEPTHS: [4, 4, 18, 4] + GROUPS: [4, 8, 16, 32] + CHANNELS: 64 + OFFSET_SCALE: 1.0 + MLP_RATIO: 4.0 +TRAIN: + EMA: + ENABLE: True + DECAY: 0.9999 + BASE_LR: 5e-4 diff --git a/classification/dataset/__init__.py b/classification/dataset/__init__.py new file mode 100644 index 0000000..6483e16 --- /dev/null +++ b/classification/dataset/__init__.py @@ -0,0 +1,7 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + +from .build import build_loader, build_loader2 \ No newline at end of file diff --git a/classification/dataset/build.py b/classification/dataset/build.py new file mode 100644 index 0000000..d02ca33 --- /dev/null +++ b/classification/dataset/build.py @@ -0,0 +1,286 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + +import os +import torch +import numpy as np +import torch.distributed as dist +from torchvision import transforms +from timm.data import Mixup +from timm.data import create_transform +from .cached_image_folder import ImageCephDataset +from .samplers import SubsetRandomSampler, NodeDistributedSampler + +try: + from torchvision.transforms import InterpolationMode + + def _pil_interp(method): + if method == 'bicubic': + return InterpolationMode.BICUBIC + elif method == 'lanczos': + return InterpolationMode.LANCZOS + elif method == 'hamming': + return InterpolationMode.HAMMING + else: + return InterpolationMode.BILINEAR +except: + from timm.data.transforms import _pil_interp + + +class TTA(torch.nn.Module): + + def __init__(self, size, scales=[1.0, 1.05, 1.1]): + super().__init__() + self.size = size + self.scales = scales + + def forward(self, img): + out = [] + cc = transforms.CenterCrop(self.size) + for scale in self.scales: + size_ = int(scale * self.size) + rs = transforms.Resize(size_, interpolation=_pil_interp('bicubic')) + img_ = rs(img) + img_ = cc(img_) + out.append(img_) + + return out + + def __repr__(self) -> str: + return f"{self.__class__.__name__}(size={self.size}, scale={self.scales})" + + +def build_loader(config): + config.defrost() + dataset_train, config.MODEL.NUM_CLASSES = build_dataset('train', + config=config) + config.freeze() + print(f"local rank {config.LOCAL_RANK} / global rank {dist.get_rank()}" + "successfully build train dataset") + + dataset_val, _ = build_dataset('val', config=config) + print(f"local rank {config.LOCAL_RANK} / global rank {dist.get_rank()}" + "successfully build val dataset") + + dataset_test, _ = build_dataset('test', config=config) + print(f"local rank {config.LOCAL_RANK} / global rank {dist.get_rank()}" + "successfully build test dataset") + + num_tasks = dist.get_world_size() + global_rank = dist.get_rank() + + if dataset_train is not None: + if config.DATA.IMG_ON_MEMORY: + sampler_train = NodeDistributedSampler(dataset_train) + else: + if config.DATA.ZIP_MODE and config.DATA.CACHE_MODE == 'part': + indices = np.arange(dist.get_rank(), len(dataset_train), + dist.get_world_size()) + sampler_train = SubsetRandomSampler(indices) + else: + sampler_train = torch.utils.data.DistributedSampler( + dataset_train, + num_replicas=num_tasks, + rank=global_rank, + shuffle=True) + + if dataset_val is not None: + if config.TEST.SEQUENTIAL: + sampler_val = torch.utils.data.SequentialSampler(dataset_val) + else: + sampler_val = torch.utils.data.distributed.DistributedSampler( + dataset_val, shuffle=False) + + if dataset_test is not None: + if config.TEST.SEQUENTIAL: + sampler_test = torch.utils.data.SequentialSampler(dataset_test) + else: + sampler_test = torch.utils.data.distributed.DistributedSampler( + dataset_test, shuffle=False) + + data_loader_train = torch.utils.data.DataLoader( + dataset_train, + sampler=sampler_train, + batch_size=config.DATA.BATCH_SIZE, + num_workers=config.DATA.NUM_WORKERS, + pin_memory=config.DATA.PIN_MEMORY, + drop_last=True, + persistent_workers=True) if dataset_train is not None else None + + data_loader_val = torch.utils.data.DataLoader( + dataset_val, + sampler=sampler_val, + batch_size=config.DATA.BATCH_SIZE, + shuffle=False, + num_workers=config.DATA.NUM_WORKERS, + pin_memory=config.DATA.PIN_MEMORY, + drop_last=False, + persistent_workers=True) if dataset_val is not None else None + + data_loader_test = torch.utils.data.DataLoader( + dataset_test, + sampler=sampler_test, + batch_size=config.DATA.BATCH_SIZE, + shuffle=False, + num_workers=config.DATA.NUM_WORKERS, + pin_memory=config.DATA.PIN_MEMORY, + drop_last=False, + persistent_workers=True) if dataset_test is not None else None + + # setup mixup / cutmix + mixup_fn = None + mixup_active = config.AUG.MIXUP > 0 or config.AUG.CUTMIX > 0. or config.AUG.CUTMIX_MINMAX is not None + if mixup_active: + mixup_fn = Mixup(mixup_alpha=config.AUG.MIXUP, + cutmix_alpha=config.AUG.CUTMIX, + cutmix_minmax=config.AUG.CUTMIX_MINMAX, + prob=config.AUG.MIXUP_PROB, + switch_prob=config.AUG.MIXUP_SWITCH_PROB, + mode=config.AUG.MIXUP_MODE, + label_smoothing=config.MODEL.LABEL_SMOOTHING, + num_classes=config.MODEL.NUM_CLASSES) + + return dataset_train, dataset_val, dataset_test, data_loader_train, \ + data_loader_val, data_loader_test, mixup_fn + + +def build_loader2(config): + config.defrost() + dataset_train, config.MODEL.NUM_CLASSES = build_dataset('train', + config=config) + config.freeze() + dataset_val, _ = build_dataset('val', config=config) + dataset_test, _ = build_dataset('test', config=config) + + data_loader_train = torch.utils.data.DataLoader( + dataset_train, + shuffle=True, + batch_size=config.DATA.BATCH_SIZE, + num_workers=config.DATA.NUM_WORKERS, + pin_memory=config.DATA.PIN_MEMORY, + drop_last=True, + persistent_workers=True) if dataset_train is not None else None + + data_loader_val = torch.utils.data.DataLoader( + dataset_val, + batch_size=config.DATA.BATCH_SIZE, + shuffle=False, + num_workers=config.DATA.NUM_WORKERS, + pin_memory=config.DATA.PIN_MEMORY, + drop_last=False, + persistent_workers=True) if dataset_val is not None else None + + data_loader_test = torch.utils.data.DataLoader( + dataset_test, + batch_size=config.DATA.BATCH_SIZE, + shuffle=False, + num_workers=config.DATA.NUM_WORKERS, + pin_memory=config.DATA.PIN_MEMORY, + drop_last=False, + persistent_workers=True) if dataset_test is not None else None + + # setup mixup / cutmix + mixup_fn = None + mixup_active = config.AUG.MIXUP > 0 or config.AUG.CUTMIX > 0. or config.AUG.CUTMIX_MINMAX is not None + if mixup_active: + mixup_fn = Mixup(mixup_alpha=config.AUG.MIXUP, + cutmix_alpha=config.AUG.CUTMIX, + cutmix_minmax=config.AUG.CUTMIX_MINMAX, + prob=config.AUG.MIXUP_PROB, + switch_prob=config.AUG.MIXUP_SWITCH_PROB, + mode=config.AUG.MIXUP_MODE, + label_smoothing=config.MODEL.LABEL_SMOOTHING, + num_classes=config.MODEL.NUM_CLASSES) + + return dataset_train, dataset_val, dataset_test, data_loader_train, \ + data_loader_val, data_loader_test, mixup_fn + + +def build_dataset(split, config): + transform = build_transform(split == 'train', config) + dataset = None + nb_classes = None + prefix = split + if config.DATA.DATASET == 'imagenet': + if prefix == 'train' and not config.EVAL_MODE: + root = os.path.join(config.DATA.DATA_PATH, 'train') + dataset = ImageCephDataset(root, + 'train', + transform=transform, + on_memory=config.DATA.IMG_ON_MEMORY) + elif prefix == 'val': + root = os.path.join(config.DATA.DATA_PATH, 'val') + dataset = ImageCephDataset(root, 'val', transform=transform) + nb_classes = 1000 + elif config.DATA.DATASET == 'imagenet22K': + if prefix == 'train': + if not config.EVAL_MODE: + root = config.DATA.DATA_PATH + dataset = ImageCephDataset(root, + 'train', + transform=transform, + on_memory=config.DATA.IMG_ON_MEMORY) + nb_classes = 21841 + elif prefix == 'val': + root = os.path.join(config.DATA.DATA_PATH, 'val') + dataset = ImageCephDataset(root, 'val', transform=transform) + nb_classes = 1000 + else: + raise NotImplementedError( + f'build_dataset does support {config.DATA.DATASET}') + + return dataset, nb_classes + + +def build_transform(is_train, config): + resize_im = config.DATA.IMG_SIZE > 32 + if is_train: + # this should always dispatch to transforms_imagenet_train + transform = create_transform( + input_size=config.DATA.IMG_SIZE, + is_training=True, + color_jitter=config.AUG.COLOR_JITTER + if config.AUG.COLOR_JITTER > 0 else None, + auto_augment=config.AUG.AUTO_AUGMENT + if config.AUG.AUTO_AUGMENT != 'none' else None, + re_prob=config.AUG.REPROB, + re_mode=config.AUG.REMODE, + re_count=config.AUG.RECOUNT, + interpolation=config.DATA.INTERPOLATION, + ) + if not resize_im: + # replace RandomResizedCropAndInterpolation with + # RandomCrop + transform.transforms[0] = transforms.RandomCrop( + config.DATA.IMG_SIZE, padding=4) + + return transform + + t = [] + if resize_im: + if config.TEST.CROP: + size = int(1.0 * config.DATA.IMG_SIZE) + t.append( + transforms.Resize(size, + interpolation=_pil_interp( + config.DATA.INTERPOLATION)), + # to maintain same ratio w.r.t. 224 images + ) + t.append(transforms.CenterCrop(config.DATA.IMG_SIZE)) + elif config.AUG.RANDOM_RESIZED_CROP: + t.append( + transforms.RandomResizedCrop( + (config.DATA.IMG_SIZE, config.DATA.IMG_SIZE), + interpolation=_pil_interp(config.DATA.INTERPOLATION))) + else: + t.append( + transforms.Resize( + (config.DATA.IMG_SIZE, config.DATA.IMG_SIZE), + interpolation=_pil_interp(config.DATA.INTERPOLATION))) + t.append(transforms.ToTensor()) + t.append(transforms.Normalize(config.AUG.MEAN, config.AUG.STD)) + + return transforms.Compose(t) diff --git a/classification/dataset/cached_image_folder.py b/classification/dataset/cached_image_folder.py new file mode 100644 index 0000000..8d95aad --- /dev/null +++ b/classification/dataset/cached_image_folder.py @@ -0,0 +1,538 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + +import io +import os +import re +import time +import json +import math +import mmcv +import torch +import logging +import os.path as osp +from PIL import Image +from tqdm import tqdm, trange +from abc import abstractmethod +import torch.utils.data as data +import torch.distributed as dist +from mmcv.fileio import FileClient +from .zipreader import is_zip_path, ZipReader + +_logger = logging.getLogger(__name__) + +_ERROR_RETRY = 50 + + +def has_file_allowed_extension(filename, extensions): + """Checks if a file is an allowed extension. + Args: + filename (string): path to a file + Returns: + bool: True if the filename ends with a known image extension + """ + filename_lower = filename.lower() + return any(filename_lower.endswith(ext) for ext in extensions) + + +def find_classes(dir): + classes = [ + d for d in os.listdir(dir) if os.path.isdir(os.path.join(dir, d)) + ] + classes.sort() + class_to_idx = {classes[i]: i for i in range(len(classes))} + return classes, class_to_idx + + +def make_dataset(dir, class_to_idx, extensions): + images = [] + dir = os.path.expanduser(dir) + for target in sorted(os.listdir(dir)): + d = os.path.join(dir, target) + if not os.path.isdir(d): + continue + for root, _, fnames in sorted(os.walk(d)): + for fname in sorted(fnames): + if has_file_allowed_extension(fname, extensions): + path = os.path.join(root, fname) + item = (path, class_to_idx[target]) + images.append(item) + + return images + + +def make_dataset_with_ann(ann_file, img_prefix, extensions): + images = [] + with open(ann_file, "r") as f: + contents = f.readlines() + for line_str in contents: + path_contents = [c for c in line_str.split('\t')] + im_file_name = path_contents[0] + class_index = int(path_contents[1]) + assert str.lower(os.path.splitext(im_file_name)[-1]) in extensions + item = (os.path.join(img_prefix, im_file_name), class_index) + images.append(item) + + return images + + +class DatasetFolder(data.Dataset): + """A generic data loader where the samples are arranged in this way: :: + root/class_x/xxx.ext + root/class_x/xxy.ext + root/class_x/xxz.ext + root/class_y/123.ext + root/class_y/nsdf3.ext + root/class_y/asd932_.ext + Args: + root (string): Root directory path. + loader (callable): A function to load a sample given its path. + extensions (list[string]): A list of allowed extensions. + transform (callable, optional): A function/transform that takes in + a sample and returns a transformed version. + E.g, ``transforms.RandomCrop`` for images. + target_transform (callable, optional): A function/transform that takes + in the target and transforms it. + Attributes: + samples (list): List of (sample path, class_index) tuples + """ + + def __init__(self, + root, + loader, + extensions, + ann_file='', + img_prefix='', + transform=None, + target_transform=None, + cache_mode="no"): + # image folder mode + if ann_file == '': + _, class_to_idx = find_classes(root) + samples = make_dataset(root, class_to_idx, extensions) + # zip mode + else: + samples = make_dataset_with_ann(os.path.join(root, ann_file), + os.path.join(root, img_prefix), + extensions) + + if len(samples) == 0: + raise (RuntimeError("Found 0 files in subfolders of: " + root + + "\n" + "Supported extensions are: " + + ",".join(extensions))) + + self.root = root + self.loader = loader + self.extensions = extensions + + self.samples = samples + self.labels = [y_1k for _, y_1k in samples] + self.classes = list(set(self.labels)) + + self.transform = transform + self.target_transform = target_transform + + self.cache_mode = cache_mode + if self.cache_mode != "no": + self.init_cache() + + def init_cache(self): + assert self.cache_mode in ["part", "full"] + n_sample = len(self.samples) + global_rank = dist.get_rank() + world_size = dist.get_world_size() + + samples_bytes = [None for _ in range(n_sample)] + start_time = time.time() + for index in range(n_sample): + if index % (n_sample // 10) == 0: + t = time.time() - start_time + print( + f'global_rank {dist.get_rank()} cached {index}/{n_sample} takes {t:.2f}s per block' + ) + start_time = time.time() + path, target = self.samples[index] + if self.cache_mode == "full": + samples_bytes[index] = (ZipReader.read(path), target) + elif self.cache_mode == "part" and index % world_size == global_rank: + samples_bytes[index] = (ZipReader.read(path), target) + else: + samples_bytes[index] = (path, target) + self.samples = samples_bytes + + def __getitem__(self, index): + """ + Args: + index (int): Index + Returns: + tuple: (sample, target) where target is class_index of the target class. + """ + path, target = self.samples[index] + sample = self.loader(path) + if self.transform is not None: + sample = self.transform(sample) + if self.target_transform is not None: + target = self.target_transform(target) + + return sample, target + + def __len__(self): + return len(self.samples) + + def __repr__(self): + fmt_str = 'Dataset ' + self.__class__.__name__ + '\n' + fmt_str += ' Number of datapoints: {}\n'.format(self.__len__()) + fmt_str += ' Root Location: {}\n'.format(self.root) + tmp = ' Transforms (if any): ' + fmt_str += '{0}{1}\n'.format( + tmp, + self.transform.__repr__().replace('\n', '\n' + ' ' * len(tmp))) + tmp = ' Target Transforms (if any): ' + fmt_str += '{0}{1}'.format( + tmp, + self.target_transform.__repr__().replace('\n', + '\n' + ' ' * len(tmp))) + + return fmt_str + + +IMG_EXTENSIONS = ['.jpg', '.jpeg', '.png', '.ppm', '.bmp', '.pgm', '.tif'] + + +def pil_loader(path): + # open path as file to avoid ResourceWarning (https://github.com/python-pillow/Pillow/issues/835) + if isinstance(path, bytes): + img = Image.open(io.BytesIO(path)) + elif is_zip_path(path): + data = ZipReader.read(path) + img = Image.open(io.BytesIO(data)) + else: + with open(path, 'rb') as f: + img = Image.open(f) + return img.convert('RGB') + + return img.convert('RGB') + + +def accimage_loader(path): + import accimage + try: + return accimage.Image(path) + except IOError: + # Potentially a decoding problem, fall back to PIL.Image + return pil_loader(path) + + +def default_img_loader(path): + from torchvision import get_image_backend + if get_image_backend() == 'accimage': + return accimage_loader(path) + else: + return pil_loader(path) + + +class CachedImageFolder(DatasetFolder): + """A generic data loader where the images are arranged in this way: :: + root/dog/xxx.png + root/dog/xxy.png + root/dog/xxz.png + root/cat/123.png + root/cat/nsdf3.png + root/cat/asd932_.png + Args: + root (string): Root directory path. + transform (callable, optional): A function/transform that takes in an PIL image + and returns a transformed version. E.g, ``transforms.RandomCrop`` + target_transform (callable, optional): A function/transform that takes in the + target and transforms it. + loader (callable, optional): A function to load an image given its path. + Attributes: + imgs (list): List of (image path, class_index) tuples + """ + + def __init__(self, + root, + ann_file='', + img_prefix='', + transform=None, + target_transform=None, + loader=default_img_loader, + cache_mode="no"): + super(CachedImageFolder, + self).__init__(root, + loader, + IMG_EXTENSIONS, + ann_file=ann_file, + img_prefix=img_prefix, + transform=transform, + target_transform=target_transform, + cache_mode=cache_mode) + self.imgs = self.samples + + def __getitem__(self, index): + """ + Args: + index (int): Index + Returns: + tuple: (image, target) where target is class_index of the target class. + """ + path, target = self.samples[index] + image = self.loader(path) + if self.transform is not None: + img = self.transform(image) + else: + img = image + if self.target_transform is not None: + target = self.target_transform(target) + + return img, target + + +class ImageCephDataset(data.Dataset): + + def __init__(self, + root, + split, + parser=None, + transform=None, + target_transform=None, + on_memory=False): + if '22k' in root: + # Imagenet 22k + annotation_root = 'meta/' + else: + # Imagenet + annotation_root = 'meta/' + if parser is None or isinstance(parser, str): + parser = ParserCephImage(root=root, + split=split, + annotation_root=annotation_root, + on_memory=on_memory) + self.parser = parser + self.transform = transform + self.target_transform = target_transform + self._consecutive_errors = 0 + + def __getitem__(self, index): + img, target = self.parser[index] + self._consecutive_errors = 0 + if self.transform is not None: + img = self.transform(img) + if target is None: + target = -1 + elif self.target_transform is not None: + target = self.target_transform(target) + return img, target + + def __len__(self): + return len(self.parser) + + def filename(self, index, basename=False, absolute=False): + return self.parser.filename(index, basename, absolute) + + def filenames(self, basename=False, absolute=False): + return self.parser.filenames(basename, absolute) + + +class Parser: + + def __init__(self): + pass + + @abstractmethod + def _filename(self, index, basename=False, absolute=False): + pass + + def filename(self, index, basename=False, absolute=False): + return self._filename(index, basename=basename, absolute=absolute) + + def filenames(self, basename=False, absolute=False): + return [ + self._filename(index, basename=basename, absolute=absolute) + for index in range(len(self)) + ] + + +class ParserCephImage(Parser): + + def __init__(self, + root, + split, + annotation_root, + on_memory=False, + **kwargs): + super().__init__() + + self.file_client = None + self.kwargs = kwargs + + self.root = root # dataset:s3://imagenet22k + if '22k' in root: + self.io_backend = 'petrel' + with open(osp.join(annotation_root, '22k_class_to_idx.json'), + 'r') as f: + self.class_to_idx = json.loads(f.read()) + with open(osp.join(annotation_root, '22k_label.txt'), 'r') as f: + self.samples = f.read().splitlines() + else: + self.io_backend = 'disk' + self.class_to_idx = None + with open(osp.join(annotation_root, f'{split}.txt'), 'r') as f: + self.samples = f.read().splitlines() + local_rank = None + local_size = None + self._consecutive_errors = 0 + self.on_memory = on_memory + if on_memory: + self.holder = {} + if local_rank is None: + local_rank = int(os.environ.get('LOCAL_RANK', 0)) + if local_size is None: + local_size = int(os.environ.get('LOCAL_SIZE', 1)) + self.local_rank = local_rank + self.local_size = local_size + self.rank = int(os.environ["RANK"]) + self.world_size = int(os.environ['WORLD_SIZE']) + self.num_replicas = int(os.environ['WORLD_SIZE']) + self.num_parts = local_size + self.num_samples = int( + math.ceil(len(self.samples) * 1.0 / self.num_replicas)) + self.total_size = self.num_samples * self.num_replicas + self.total_size_parts = self.num_samples * self.num_replicas // self.num_parts + self.load_onto_memory_v2() + + def load_onto_memory(self): + print("Loading images onto memory...", self.local_rank, + self.local_size) + if self.file_client is None: + self.file_client = FileClient(self.io_backend, **self.kwargs) + for index in trange(len(self.samples)): + if index % self.local_size != self.local_rank: + continue + path, _ = self.samples[index].split(' ') + path = osp.join(self.root, path) + img_bytes = self.file_client.get(path) + self.holder[path] = img_bytes + + print("Loading complete!") + + def load_onto_memory_v2(self): + # print("Loading images onto memory...", self.local_rank, self.local_size) + t = torch.Generator() + t.manual_seed(0) + indices = torch.randperm(len(self.samples), generator=t).tolist() + # indices = range(len(self.samples)) + indices = [i for i in indices if i % self.num_parts == self.local_rank] + # add extra samples to make it evenly divisible + indices += indices[:(self.total_size_parts - len(indices))] + assert len(indices) == self.total_size_parts + + # subsample + indices = indices[self.rank // self.num_parts:self. + total_size_parts:self.num_replicas // self.num_parts] + assert len(indices) == self.num_samples + + if self.file_client is None: + self.file_client = FileClient(self.io_backend, **self.kwargs) + for index in tqdm(indices): + if index % self.local_size != self.local_rank: + continue + path, _ = self.samples[index].split(' ') + path = osp.join(self.root, path) + img_bytes = self.file_client.get(path) + + self.holder[path] = img_bytes + + print("Loading complete!") + + def __getitem__(self, index): + if self.file_client is None: + self.file_client = FileClient(self.io_backend, **self.kwargs) + + filepath, target = self.samples[index].split(' ') + filepath = osp.join(self.root, filepath) + + try: + if self.on_memory: + img_bytes = self.holder[filepath] + else: + # pass + img_bytes = self.file_client.get(filepath) + img = mmcv.imfrombytes(img_bytes)[:, :, ::-1] + except Exception as e: + _logger.warning( + f'Skipped sample (index {index}, file {filepath}). {str(e)}') + self._consecutive_errors += 1 + if self._consecutive_errors < _ERROR_RETRY: + return self.__getitem__((index + 1) % len(self)) + else: + raise e + self._consecutive_errors = 0 + + img = Image.fromarray(img) + try: + if self.class_to_idx is not None: + target = self.class_to_idx[target] + else: + target = int(target) + except: + print('aaaaaaaaaaaa', filepath, target) + exit() + + return img, target + + def __len__(self): + return len(self.samples) + + def _filename(self, index, basename=False, absolute=False): + filename, _ = self.samples[index].split(' ') + filename = osp.join(self.root, filename) + + return filename + + +def get_temporal_info(date, miss_hour=False): + try: + if date: + if miss_hour: + pattern = re.compile(r'(\d*)-(\d*)-(\d*)', re.I) + else: + pattern = re.compile(r'(\d*)-(\d*)-(\d*) (\d*):(\d*):(\d*)', + re.I) + m = pattern.match(date.strip()) + + if m: + year = int(m.group(1)) + month = int(m.group(2)) + day = int(m.group(3)) + x_month = math.sin(2 * math.pi * month / 12) + y_month = math.cos(2 * math.pi * month / 12) + if miss_hour: + x_hour = 0 + y_hour = 0 + else: + hour = int(m.group(4)) + x_hour = math.sin(2 * math.pi * hour / 24) + y_hour = math.cos(2 * math.pi * hour / 24) + return [x_month, y_month, x_hour, y_hour] + else: + return [0, 0, 0, 0] + else: + return [0, 0, 0, 0] + except: + return [0, 0, 0, 0] + + +def get_spatial_info(latitude, longitude): + if latitude and longitude: + latitude = math.radians(latitude) + longitude = math.radians(longitude) + x = math.cos(latitude) * math.cos(longitude) + y = math.cos(latitude) * math.sin(longitude) + z = math.sin(latitude) + return [x, y, z] + else: + return [0, 0, 0] diff --git a/classification/dataset/samplers.py b/classification/dataset/samplers.py new file mode 100644 index 0000000..3f52915 --- /dev/null +++ b/classification/dataset/samplers.py @@ -0,0 +1,114 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + +import torch +import os +import math +from torch.utils.data.sampler import Sampler +import torch.distributed as dist +import numpy as np + + +class SubsetRandomSampler(torch.utils.data.Sampler): + """Samples elements randomly from a given list of indices, without replacement. + + Arguments: + indices (sequence): a sequence of indices + """ + + def __init__(self, indices): + self.epoch = 0 + self.indices = indices + + def __iter__(self): + return (self.indices[i] for i in torch.randperm(len(self.indices))) + + def __len__(self): + return len(self.indices) + + def set_epoch(self, epoch): + self.epoch = epoch + + +class NodeDistributedSampler(Sampler): + """Sampler that restricts data loading to a subset of the dataset. + It is especially useful in conjunction with + :class:`torch.nn.parallel.DistributedDataParallel`. In such case, each + process can pass a DistributedSampler instance as a DataLoader sampler, + and load a subset of the original dataset that is exclusive to it. + .. note:: + Dataset is assumed to be of constant size. + Arguments: + dataset: Dataset used for sampling. + num_replicas (optional): Number of processes participating in + distributed training. + rank (optional): Rank of the current process within num_replicas. + """ + + def __init__(self, + dataset, + num_replicas=None, + rank=None, + local_rank=None, + local_size=None): + if num_replicas is None: + if not dist.is_available(): + raise RuntimeError( + "Requires distributed package to be available") + num_replicas = dist.get_world_size() + if rank is None: + if not dist.is_available(): + raise RuntimeError( + "Requires distributed package to be available") + rank = dist.get_rank() + if local_rank is None: + local_rank = int(os.environ.get('LOCAL_RANK', 0)) + if local_size is None: + local_size = int(os.environ.get('LOCAL_SIZE', 1)) + self.dataset = dataset + self.num_replicas = num_replicas + self.num_parts = local_size + self.rank = rank + self.local_rank = local_rank + self.epoch = 0 + self.num_samples = int( + math.ceil(len(self.dataset) * 1.0 / self.num_replicas)) + self.total_size = self.num_samples * self.num_replicas + + self.total_size_parts = self.num_samples * self.num_replicas // self.num_parts + + def __iter__(self): + # deterministically shuffle based on epoch + g = torch.Generator() + g.manual_seed(self.epoch) + + t = torch.Generator() + t.manual_seed(0) + + indices = torch.randperm(len(self.dataset), generator=t).tolist() + # indices = range(len(self.dataset)) + indices = [i for i in indices if i % self.num_parts == self.local_rank] + + # add extra samples to make it evenly divisible + indices += indices[:(self.total_size_parts - len(indices))] + assert len(indices) == self.total_size_parts + + # subsample + indices = indices[self.rank // self.num_parts:self. + total_size_parts:self.num_replicas // self.num_parts] + + index = torch.randperm(len(indices), generator=g).tolist() + indices = list(np.array(indices)[index]) + + assert len(indices) == self.num_samples + + return iter(indices) + + def __len__(self): + return self.num_samples + + def set_epoch(self, epoch): + self.epoch = epoch diff --git a/classification/dataset/zipreader.py b/classification/dataset/zipreader.py new file mode 100644 index 0000000..e4a03e3 --- /dev/null +++ b/classification/dataset/zipreader.py @@ -0,0 +1,102 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + +import os +import zipfile +import io +import numpy as np +from PIL import Image +from PIL import ImageFile + +ImageFile.LOAD_TRUNCATED_IMAGES = True + + +def is_zip_path(img_or_path): + """judge if this is a zip path""" + return '.zip@' in img_or_path + + +class ZipReader(object): + """A class to read zipped files""" + zip_bank = dict() + + def __init__(self): + super(ZipReader, self).__init__() + + @staticmethod + def get_zipfile(path): + zip_bank = ZipReader.zip_bank + if path not in zip_bank: + zfile = zipfile.ZipFile(path, 'r') + zip_bank[path] = zfile + return zip_bank[path] + + @staticmethod + def split_zip_style_path(path): + pos_at = path.index('@') + assert pos_at != -1, "character '@' is not found from the given path '%s'" % path + + zip_path = path[0:pos_at] + folder_path = path[pos_at + 1:] + folder_path = str.strip(folder_path, '/') + return zip_path, folder_path + + @staticmethod + def list_folder(path): + zip_path, folder_path = ZipReader.split_zip_style_path(path) + + zfile = ZipReader.get_zipfile(zip_path) + folder_list = [] + for file_foler_name in zfile.namelist(): + file_foler_name = str.strip(file_foler_name, '/') + if file_foler_name.startswith(folder_path) and \ + len(os.path.splitext(file_foler_name)[-1]) == 0 and \ + file_foler_name != folder_path: + if len(folder_path) == 0: + folder_list.append(file_foler_name) + else: + folder_list.append(file_foler_name[len(folder_path) + 1:]) + + return folder_list + + @staticmethod + def list_files(path, extension=None): + if extension is None: + extension = ['.*'] + zip_path, folder_path = ZipReader.split_zip_style_path(path) + + zfile = ZipReader.get_zipfile(zip_path) + file_lists = [] + for file_foler_name in zfile.namelist(): + file_foler_name = str.strip(file_foler_name, '/') + if file_foler_name.startswith(folder_path) and \ + str.lower(os.path.splitext(file_foler_name)[-1]) in extension: + if len(folder_path) == 0: + file_lists.append(file_foler_name) + else: + file_lists.append(file_foler_name[len(folder_path) + 1:]) + + return file_lists + + @staticmethod + def read(path): + zip_path, path_img = ZipReader.split_zip_style_path(path) + zfile = ZipReader.get_zipfile(zip_path) + data = zfile.read(path_img) + return data + + @staticmethod + def imread(path): + zip_path, path_img = ZipReader.split_zip_style_path(path) + zfile = ZipReader.get_zipfile(zip_path) + data = zfile.read(path_img) + try: + im = Image.open(io.BytesIO(data)) + except: + print("ERROR IMG LOADED: ", path_img) + random_img = np.random.rand(224, 224, 3) * 255 + im = Image.fromarray(np.uint8(random_img)) + return im diff --git a/classification/ddp_hooks.py b/classification/ddp_hooks.py new file mode 100644 index 0000000..27dcf86 --- /dev/null +++ b/classification/ddp_hooks.py @@ -0,0 +1,183 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + +from typing import Any, Callable + +import torch +import torch.distributed as dist + + +def _allreduce_fut(process_group: dist.ProcessGroup, + tensor: torch.Tensor) -> torch.futures.Future[torch.Tensor]: + "Averages the input gradient tensor by allreduce and returns a future." + group_to_use = process_group if process_group is not None else dist.group.WORLD + + # Apply the division first to avoid overflow, especially for FP16. + tensor.div_(group_to_use.size()) + + return (dist.all_reduce( + tensor, group=group_to_use, + async_op=True).get_future().then(lambda fut: fut.value()[0])) + + +def allreduce_hook( + process_group: dist.ProcessGroup, + bucket: dist.GradBucket) -> torch.futures.Future[torch.Tensor]: + """ + This DDP communication hook just calls ``allreduce`` using ``GradBucket`` + tensors. Once gradient tensors are aggregated across all workers, its ``then`` + callback takes the mean and returns the result. If user registers this hook, + DDP results is expected to be same as the case where no hook was registered. + Hence, this won't change behavior of DDP and user can use this as a reference + or modify this hook to log useful information or any other purposes while + unaffecting DDP behavior. + + Example:: + >>> ddp_model.register_comm_hook(process_group, allreduce_hook) + """ + return _allreduce_fut(process_group, bucket.buffer()) + + +def fp16_compress_hook( + process_group: dist.ProcessGroup, + bucket: dist.GradBucket) -> torch.futures.Future[torch.Tensor]: + """ + This DDP communication hook implements a simple gradient compression + approach that casts ``GradBucket`` tensor to half-precision floating-point format (``torch.float16``) + and then divides it by the process group size. + It allreduces those ``float16`` gradient tensors. Once compressed gradient + tensors are allreduced, the chained callback ``decompress`` casts it back to the input data type (such as ``float32``). + + Example:: + >>> ddp_model.register_comm_hook(process_group, fp16_compress_hook) + """ + group_to_use = process_group if process_group is not None else dist.group.WORLD + world_size = group_to_use.size() + + compressed_tensor = bucket.buffer().to(torch.float16).div_(world_size) + + fut = dist.all_reduce(compressed_tensor, group=group_to_use, + async_op=True).get_future() + + def decompress(fut): + decompressed_tensor = bucket.buffer() + # Decompress in place to reduce the peak memory. + # See: https://github.com/pytorch/pytorch/issues/45968 + decompressed_tensor.copy_(fut.value()[0]) + return decompressed_tensor + + return fut.then(decompress) + + +# TODO: create an internal helper function and extract the duplicate code in FP16_compress and BF16_compress. + + +def bf16_compress_hook( + process_group: dist.ProcessGroup, + bucket: dist.GradBucket) -> torch.futures.Future[torch.Tensor]: + """ + Warning: This API is experimental, and it requires NCCL version later than 2.9.6. + + This DDP communication hook implements a simple gradient compression + approach that casts ``GradBucket`` tensor to half-precision + `Brain floating point format `_ (``torch.bfloat16``) + and then divides it by the process group size. + It allreduces those ``bfloat16`` gradient tensors. Once compressed gradient + tensors are allreduced, the chained callback ``decompress`` casts it back to the input data type (such as ``float32``). + + Example:: + >>> ddp_model.register_comm_hook(process_group, bf16_compress_hook) + """ + group_to_use = process_group if process_group is not None else dist.group.WORLD + world_size = group_to_use.size() + + compressed_tensor = bucket.buffer().to(torch.bfloat16).div_(world_size) + + fut = dist.all_reduce(compressed_tensor, group=group_to_use, + async_op=True).get_future() + + def decompress(fut): + decompressed_tensor = bucket.buffer() + # Decompress in place to reduce the peak memory. + # See: https://github.com/pytorch/pytorch/issues/45968 + decompressed_tensor.copy_(fut.value()[0]) + return decompressed_tensor + + return fut.then(decompress) + + +def fp16_compress_wrapper( + hook: Callable[[Any, dist.GradBucket], torch.futures.Future[torch.Tensor]] +) -> Callable[[Any, dist.GradBucket], torch.futures.Future[torch.Tensor]]: + """ + This wrapper casts the input gradient tensor of a given DDP communication hook to half-precision + floating point format (``torch.float16``), and casts the resulting tensor of the given hook back to + the input data type, such as ``float32``. + + Therefore, ``fp16_compress_hook`` is equivalent to ``fp16_compress_wrapper(allreduce_hook)``. + + Example:: + >>> state = PowerSGDState(process_group=process_group, matrix_approximation_rank=1, start_powerSGD_iter=10) + >>> ddp_model.register_comm_hook(state, fp16_compress_wrapper(powerSGD_hook)) + """ + + def fp16_compress_wrapper_hook( + hook_state, + bucket: dist.GradBucket) -> torch.futures.Future[torch.Tensor]: + # Cast bucket tensor to FP16. + bucket.set_buffer(bucket.buffer().to(torch.float16)) + + fut = hook(hook_state, bucket) + + def decompress(fut): + decompressed_tensor = bucket.buffer() + # Decompress in place to reduce the peak memory. + # See: https://github.com/pytorch/pytorch/issues/45968 + decompressed_tensor.copy_(fut.value()) + return decompressed_tensor + + # Decompress after hook has run. + return fut.then(decompress) + + return fp16_compress_wrapper_hook + + +def bf16_compress_wrapper( + hook: Callable[[Any, dist.GradBucket], torch.futures.Future[torch.Tensor]] +) -> Callable[[Any, dist.GradBucket], torch.futures.Future[torch.Tensor]]: + """ + Warning: This API is experimental, and it requires NCCL version later than 2.9.6. + + This wrapper casts the input gradient tensor of a given DDP communication hook to half-precision + `Brain floating point format `_ (``torch.bfloat16``), + and casts the resulting tensor of the given hook back to the input data type, such as ``float32``. + + Therefore, ``bf16_compress_hook`` is equivalent to ``bf16_compress_wrapper(allreduce_hook)``. + + Example:: + >>> state = PowerSGDState(process_group=process_group, matrix_approximation_rank=1, start_powerSGD_iter=10) + >>> ddp_model.register_comm_hook(state, bf16_compress_wrapper(powerSGD_hook)) + """ + + def bf16_compress_wrapper_hook( + hook_state, + bucket: dist.GradBucket) -> torch.futures.Future[torch.Tensor]: + # Cast bucket tensor to BF16. + bucket.set_buffer(bucket.buffer().to(torch.bfloat16)) + + fut = hook(hook_state, bucket) + + def decompress(fut): + decompressed_tensor = bucket.buffer() + # Decompress in place to reduce the peak memory. + # See: https://github.com/pytorch/pytorch/issues/45968 + decompressed_tensor.copy_(fut.value()) + return decompressed_tensor + + # Decompress after hook has run. + return fut.then(decompress) + + return bf16_compress_wrapper_hook diff --git a/classification/ema_deepspeed.py b/classification/ema_deepspeed.py new file mode 100644 index 0000000..1b2b58d --- /dev/null +++ b/classification/ema_deepspeed.py @@ -0,0 +1,99 @@ +import torch +import torch.nn as nn +import deepspeed +from deepspeed.runtime.zero import GatheredParameters +from contextlib import contextmanager + + +class EMADeepspeed(nn.Module): + """ migrated from https://github.com/microsoft/DeepSpeed/issues/2056 + """ + + def __init__(self, model, decay=0.9999, use_num_updates=True): + super().__init__() + if decay < 0.0 or decay > 1.0: + raise ValueError('Decay must be between 0 and 1') + + self.m_name2s_name = {} + self.decay = decay + self.num_updates = 0 if use_num_updates else -1 + + with GatheredParameters(model.parameters(), fwd_module=self): + for name, p in model.named_parameters(): + if p.requires_grad: + # remove as '.'-character is not allowed in buffers + s_name = name.replace('.', '') + self.m_name2s_name.update({name: s_name}) + self.register_buffer(s_name, p.clone().detach().data) + # remove as '.'-character is not allowed in buffers + self.collected_params = [] + + def forward(self, model): + decay = self.decay + + if self.num_updates >= 0: + self.num_updates += 1 + decay = min(self.decay, (1 + self.num_updates) / (10 + self.num_updates)) + + one_minus_decay = 1.0 - decay + shadow_params = dict(self.named_buffers()) + + with torch.no_grad(): + with GatheredParameters(model.parameters()): + if deepspeed.comm.get_rank() == 0: + m_param = dict(model.named_parameters()) + + for key in m_param: + if m_param[key].requires_grad: + sname = self.m_name2s_name[key] + shadow_params[sname] = shadow_params[sname].type_as(m_param[key]) + shadow_params[sname].sub_(one_minus_decay * (shadow_params[sname] - m_param[key])) + else: + assert not key in self.m_name2s_name + + def copy_to(self, model): + shadow_params = dict(self.named_buffers()) + with GatheredParameters(model.parameters(), modifier_rank=0): + if deepspeed.comm.get_rank() == 0: + m_param = dict(model.named_parameters()) + for key in m_param: + if m_param[key].requires_grad: + m_param[key].data.copy_(shadow_params[self.m_name2s_name[key]].data) + else: + assert not key in self.m_name2s_name + + def store(self, model): + """ + Save the current parameters for restoring later. + Args: + model: A model that parameters will be stored + """ + with GatheredParameters(model.parameters()): + if deepspeed.comm.get_rank() == 0: + parameters = model.parameters() + self.collected_params = [param.clone() for param in parameters] + + def restore(self, model): + """ + Restore the parameters stored with the `store` method. + Useful to validate the model with EMA parameters without affecting the + original optimization process. Store the parameters before the + `copy_to` method. After validation (or model saving), use this to + restore the former parameters. + Args: + model: A model that to restore its parameters. + """ + with GatheredParameters(model.parameters(), modifier_rank=0): + if deepspeed.comm.get_rank() == 0: + parameters = model.parameters() + for c_param, param in zip(self.collected_params, parameters): + param.data.copy_(c_param.data) + + @contextmanager + def activate(self, model): + try: + self.store(model) + self.copy_to(model) + yield + finally: + self.restore(model) diff --git a/classification/eval.sh b/classification/eval.sh new file mode 100644 index 0000000..40c7204 --- /dev/null +++ b/classification/eval.sh @@ -0,0 +1,2 @@ +python -m torch.distributed.launch --nproc_per_node 1 --master_port 12345 main.py --eval \ +--cfg configs/flash_intern_image_l_22k_384.yaml --data-path /path/to/imagenet1k diff --git a/classification/export.py b/classification/export.py new file mode 100644 index 0000000..3b0b189 --- /dev/null +++ b/classification/export.py @@ -0,0 +1,122 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + + +import os +import time +import argparse + +import torch +from tqdm import tqdm + +from config import get_config +from models import build_model + +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument('--model_name', type=str, + default='internimage_t_1k_224') + parser.add_argument('--ckpt_dir', type=str, + default='/mnt/petrelfs/share_data/huangzhenhang/code/internimage/checkpoint_dir/new/cls') + parser.add_argument('--onnx', default=False, action='store_true') + parser.add_argument('--trt', default=False, action='store_true') + + args = parser.parse_args() + args.cfg = os.path.join('./configs', f'{args.model_name}.yaml') + args.ckpt = os.path.join(args.ckpt_dir, f'{args.model_name}.pth') + args.size = int(args.model_name.split('.')[0].split('_')[-1]) + + cfg = get_config(args) + return args, cfg + +def get_model(args, cfg): + model = build_model(cfg) + ckpt = torch.load(args.ckpt, map_location='cpu')['model'] + + model.load_state_dict(ckpt) + return model + +def speed_test(model, input): + # warmup + for _ in tqdm(range(100)): + _ = model(input) + + # speed test + torch.cuda.synchronize() + start = time.time() + for _ in tqdm(range(100)): + _ = model(input) + end = time.time() + th = 100 / (end - start) + print(f"using time: {end - start}, throughput {th}") + +def torch2onnx(args, cfg): + model = get_model(args, cfg).cuda() + + # speed_test(model) + + onnx_name = f'{args.model_name}.onnx' + torch.onnx.export(model, + torch.rand(1, 3, args.size, args.size).cuda(), + onnx_name, + input_names=['input'], + output_names=['output']) + + return model + +def onnx2trt(args): + from mmdeploy.backend.tensorrt import from_onnx + + onnx_name = f'{args.model_name}.onnx' + from_onnx( + onnx_name, + args.model_name, + dict( + input=dict( + min_shape=[1, 3, args.size, args.size], + opt_shape=[1, 3, args.size, args.size], + max_shape=[1, 3, args.size, args.size], + ) + ), + max_workspace_size=2**30, + ) + +def check(args, cfg): + from mmdeploy.backend.tensorrt.wrapper import TRTWrapper + + model = get_model(args, cfg).cuda() + model.eval() + trt_model = TRTWrapper(f'{args.model_name}.engine', + ['output']) + + x = torch.randn(1, 3, args.size, args.size).cuda() + + torch_out = model(x) + trt_out = trt_model(dict(input=x))['output'] + + print('torch out shape:', torch_out.shape) + print('trt out shape:', trt_out.shape) + + print('max delta:', (torch_out - trt_out).abs().max()) + print('mean delta:', (torch_out - trt_out).abs().mean()) + + speed_test(model, x) + speed_test(trt_model, dict(input=x)) + +def main(): + args, cfg = get_args() + + if args.onnx or args.trt: + torch2onnx(args, cfg) + print('torch -> onnx: succeess') + + if args.trt: + onnx2trt(args) + print('onnx -> trt: success') + check(args, cfg) + +if __name__ == '__main__': + main() diff --git a/classification/extract_feature.py b/classification/extract_feature.py new file mode 100644 index 0000000..acb4f45 --- /dev/null +++ b/classification/extract_feature.py @@ -0,0 +1,128 @@ +import functools +from collections import OrderedDict + + +# using wonder's beautiful simplification: +# https://stackoverflow.com/questions/31174295/getattr-and-setattr-on-nested-objects/31174427?noredirect=1#comment86638618_31174427 +def rgetattr(obj, attr, *args): + def _getattr(obj, attr): + return getattr(obj, attr, *args) + + return functools.reduce(_getattr, [obj] + attr.split('.')) + + +class IntermediateLayerGetter: + def __init__(self, model, return_layers, keep_output=True): + """Wraps a Pytorch module to get intermediate values + + Arguments: + model {nn.module} -- The Pytorch module to call + return_layers {dict} -- Dictionary with the selected submodules + to return the output (format: {[current_module_name]: [desired_output_name]}, + current_module_name can be a nested submodule, e.g. submodule1.submodule2.submodule3) + + Keyword Arguments: + keep_output {bool} -- If True model_output contains the final model's output + in the other case model_output is None (default: {True}) + + Returns: + (mid_outputs {OrderedDict}, model_output {any}) -- mid_outputs keys are + your desired_output_name (s) and their values are the returned tensors + of those submodules (OrderedDict([(desired_output_name,tensor(...)), ...). + See keep_output argument for model_output description. + In case a submodule is called more than one time, all it's outputs are + stored in a list. + """ + self._model = model + self.return_layers = return_layers + self.keep_output = keep_output + + def __call__(self, *args, **kwargs): + ret = OrderedDict() + handles = [] + for name, new_name in self.return_layers.items(): + layer = rgetattr(self._model, name) + + def hook(module, input, output, new_name=new_name): + if new_name in ret: + if type(ret[new_name]) is list: + ret[new_name].append(output) + else: + ret[new_name] = [ret[new_name], output] + else: + ret[new_name] = output + + try: + h = layer.register_forward_hook(hook) + except AttributeError as e: + raise AttributeError(f'Module {name} not found') + handles.append(h) + + if self.keep_output: + output = self._model(*args, **kwargs) + else: + self._model(*args, **kwargs) + output = None + + for h in handles: + h.remove() + + return ret, output + + +def main(args, config): + from models import build_model + import torchvision.transforms as T + from PIL import Image + + model = build_model(config) + checkpoint = torch.load(config.MODEL.RESUME, map_location='cpu') + model.load_state_dict(checkpoint['model'], strict=False) + model.cuda() + + # examples: + # return_layers = { + # 'patch_embed': 'patch_embed', + # 'levels.0.downsample': 'levels.0.downsample', + # 'levels.0.blocks.0.dcn': 'levels.0.blocks.0.dcn', + # } + return_layers = {k: k for k in args.keys} + mid_getter = IntermediateLayerGetter(model, return_layers=return_layers, keep_output=True) + + image = Image.open(args.img) + + transforms = T.Compose([ + T.Resize(config.DATA.IMG_SIZE), + T.ToTensor(), + T.Normalize(config.AUG.MEAN, config.AUG.STD) + ]) + image = transforms(image) + image = image.unsqueeze(0) + image = image.cuda() + + mid_outputs, model_output = mid_getter(image) + + for k, v in mid_outputs.items(): + print(k, v.shape) + + return mid_outputs, model_output + + +if __name__ == '__main__': + import argparse + import torch + from config import get_config + + parser = argparse.ArgumentParser('Get Intermediate Layer Output') + parser.add_argument('--cfg', type=str, required=True, metavar="FILE", help='Path to config file') + parser.add_argument('--img', type=str, required=True, metavar="FILE", help='Path to img file') + parser.add_argument("--keys", default=None, nargs='+', help="The intermediate layer's keys you want to save.") + parser.add_argument('--resume', help='resume from checkpoint') + parser.add_argument('--save', action='store_true', help='Save the results.') + args = parser.parse_args() + config = get_config(args) + + mid_outputs, model_output = main(args, config) + + if args.save: + torch.save(mid_outputs, args.img[:-3] + '.pth') \ No newline at end of file diff --git a/classification/logger.py b/classification/logger.py new file mode 100644 index 0000000..55cd27c --- /dev/null +++ b/classification/logger.py @@ -0,0 +1,44 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + +import os +import sys +import logging +import functools +from termcolor import colored + + +@functools.lru_cache() +def create_logger(output_dir, dist_rank=0, name=''): + # create logger + logger = logging.getLogger(name) + logger.setLevel(logging.DEBUG) + logger.propagate = False + + # create formatter + fmt = '[%(asctime)s %(name)s] (%(filename)s %(lineno)d): %(levelname)s %(message)s' + color_fmt = colored('[%(asctime)s %(name)s]', 'green') + \ + colored('(%(filename)s %(lineno)d)', 'yellow') + \ + ': %(levelname)s %(message)s' + + # create console handlers for master process + if dist_rank == 0: + console_handler = logging.StreamHandler(sys.stdout) + console_handler.setLevel(logging.DEBUG) + console_handler.setFormatter( + logging.Formatter(fmt=color_fmt, datefmt='%Y-%m-%d %H:%M:%S')) + logger.addHandler(console_handler) + + # create file handlers + file_handler = logging.FileHandler(os.path.join( + output_dir, f'log_rank{dist_rank}.txt'), + mode='a') + file_handler.setLevel(logging.DEBUG) + file_handler.setFormatter( + logging.Formatter(fmt=fmt, datefmt='%Y-%m-%d %H:%M:%S')) + logger.addHandler(file_handler) + + return logger diff --git a/classification/lr_scheduler.py b/classification/lr_scheduler.py new file mode 100644 index 0000000..f184f15 --- /dev/null +++ b/classification/lr_scheduler.py @@ -0,0 +1,112 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + + +import torch +from timm.scheduler.cosine_lr import CosineLRScheduler +from timm.scheduler.step_lr import StepLRScheduler +from timm.scheduler.scheduler import Scheduler + + +def build_scheduler(config, optimizer, n_iter_per_epoch): + num_steps = int(config.TRAIN.EPOCHS * n_iter_per_epoch) + warmup_steps = int(config.TRAIN.WARMUP_EPOCHS * n_iter_per_epoch) + decay_steps = int(config.TRAIN.LR_SCHEDULER.DECAY_EPOCHS * + n_iter_per_epoch) + + lr_scheduler = None + if config.TRAIN.LR_SCHEDULER.NAME == 'cosine': + lr_scheduler = CosineLRScheduler( + optimizer, + t_initial=num_steps, + # t_mul=1., + lr_min=config.TRAIN.MIN_LR, + warmup_lr_init=config.TRAIN.WARMUP_LR, + warmup_t=warmup_steps, + cycle_limit=1, + t_in_epochs=False, + ) + elif config.TRAIN.LR_SCHEDULER.NAME == 'linear': + lr_scheduler = LinearLRScheduler( + optimizer, + t_initial=num_steps, + lr_min_rate=0.01, + warmup_lr_init=config.TRAIN.WARMUP_LR, + warmup_t=warmup_steps, + t_in_epochs=False, + ) + elif config.TRAIN.LR_SCHEDULER.NAME == 'step': + lr_scheduler = StepLRScheduler( + optimizer, + decay_t=decay_steps, + decay_rate=config.TRAIN.LR_SCHEDULER.DECAY_RATE, + warmup_lr_init=config.TRAIN.WARMUP_LR, + warmup_t=warmup_steps, + t_in_epochs=False, + ) + + return lr_scheduler + + +class LinearLRScheduler(Scheduler): + + def __init__( + self, + optimizer: torch.optim.Optimizer, + t_initial: int, + lr_min_rate: float, + warmup_t=0, + warmup_lr_init=0., + t_in_epochs=True, + noise_range_t=None, + noise_pct=0.67, + noise_std=1.0, + noise_seed=42, + initialize=True, + ) -> None: + super().__init__(optimizer, + param_group_field="lr", + noise_range_t=noise_range_t, + noise_pct=noise_pct, + noise_std=noise_std, + noise_seed=noise_seed, + initialize=initialize) + + self.t_initial = t_initial + self.lr_min_rate = lr_min_rate + self.warmup_t = warmup_t + self.warmup_lr_init = warmup_lr_init + self.t_in_epochs = t_in_epochs + if self.warmup_t: + self.warmup_steps = [(v - warmup_lr_init) / self.warmup_t + for v in self.base_values] + super().update_groups(self.warmup_lr_init) + else: + self.warmup_steps = [1 for _ in self.base_values] + + def _get_lr(self, t): + if t < self.warmup_t: + lrs = [self.warmup_lr_init + t * s for s in self.warmup_steps] + else: + t = t - self.warmup_t + total_t = self.t_initial - self.warmup_t + lrs = [ + v - ((v - v * self.lr_min_rate) * (t / total_t)) + for v in self.base_values + ] + return lrs + + def get_epoch_values(self, epoch: int): + if self.t_in_epochs: + return self._get_lr(epoch) + else: + return None + + def get_update_values(self, num_updates: int): + if not self.t_in_epochs: + return self._get_lr(num_updates) + else: + return None diff --git a/classification/main.py b/classification/main.py new file mode 100644 index 0000000..0ed32cb --- /dev/null +++ b/classification/main.py @@ -0,0 +1,671 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + + +import os +import time +import random +import argparse +import datetime +import numpy as np +import subprocess + +import torch +import torch.backends.cudnn as cudnn +import torch.distributed as dist +from timm.utils import ModelEma, ApexScaler +from timm.loss import LabelSmoothingCrossEntropy, SoftTargetCrossEntropy +from timm.utils import accuracy, AverageMeter + +from config import get_config +from models import build_model +from dataset import build_loader +from lr_scheduler import build_scheduler +from optimizer import build_optimizer +from logger import create_logger +from utils import NativeScalerWithGradNormCount as NativeScaler +from utils import (load_checkpoint, load_pretrained, save_checkpoint, + get_grad_norm, auto_resume_helper, reduce_tensor, + load_ema_checkpoint, MyAverageMeter) + +from contextlib import suppress +from ddp_hooks import fp16_compress_hook + +try: + from apex import amp + has_apex = True +except ImportError: + has_apex = False +# assert not has_apex, "The code is modified based on native amp" + +has_native_amp = False +try: + if getattr(torch.cuda.amp, 'autocast') is not None: + has_native_amp = True +except AttributeError: + pass + +TORCH_VERSION = tuple(int(x) for x in torch.__version__.split('.')[:2]) + + +def obsolete_torch_version(torch_version, version_threshold): + return torch_version == 'parrots' or torch_version <= version_threshold + + +def parse_option(): + parser = argparse.ArgumentParser( + 'InternImage training and evaluation script', add_help=False) + parser.add_argument('--cfg', + type=str, + required=True, + metavar="FILE", + help='path to config file') + parser.add_argument( + "--opts", + help="Modify config options by adding 'KEY VALUE' pairs. ", + default=None, + nargs='+') + + # easy config modification + parser.add_argument('--batch-size', + type=int, + help="batch size for single GPU") + parser.add_argument('--dataset', + type=str, + help='dataset name', + default=None) + parser.add_argument('--data-path', type=str, help='path to dataset') + parser.add_argument('--zip', + action='store_true', + help='use zipped dataset instead of folder dataset') + parser.add_argument( + '--cache-mode', + type=str, + default='part', + choices=['no', 'full', 'part'], + help='no: no cache, ' + 'full: cache all data, ' + 'part: sharding the dataset into nonoverlapping pieces and only cache one piece' + ) + parser.add_argument( + '--pretrained', + help= + 'pretrained weight from checkpoint, could be imagenet22k pretrained weight' + ) + parser.add_argument('--resume', help='resume from checkpoint') + parser.add_argument('--accumulation-steps', + type=int, + default=1, + help="gradient accumulation steps") + parser.add_argument( + '--use-checkpoint', + action='store_true', + help="whether to use gradient checkpointing to save memory") + parser.add_argument( + '--amp-opt-level', + type=str, + default='O1', + choices=['O0', 'O1', 'O2'], + help='mixed precision opt level, if O0, no amp is used') + parser.add_argument( + '--output', + default='output', + type=str, + metavar='PATH', + help= + 'root of output folder, the full path is // (default: output)' + ) + parser.add_argument('--tag', help='tag of experiment') + parser.add_argument('--eval', + action='store_true', + help='Perform evaluation only') + parser.add_argument('--throughput', + action='store_true', + help='Test throughput only') + parser.add_argument('--save-ckpt-num', default=1, type=int) + parser.add_argument( + '--use-zero', + action='store_true', + help="whether to use ZeroRedundancyOptimizer (ZeRO) to save memory") + + # distributed training + parser.add_argument("--local-rank", + type=int, + default=0, + help='local rank for DistributedDataParallel') + + args, unparsed = parser.parse_known_args() + + if 'LOCAL_RANK' not in os.environ: + os.environ['LOCAL_RANK'] = str(args.local_rank) + + + config = get_config(args) + + config.defrost() + config.LOCAL_RANK = int(os.environ['LOCAL_RANK']) + config.freeze() + return args, config + + +@torch.no_grad() +def throughput(data_loader, model, logger): + model.eval() + + for idx, (images, _) in enumerate(data_loader): + images = images.cuda(non_blocking=True) + batch_size = images.shape[0] + for i in range(50): + model(images) + torch.cuda.synchronize() + logger.info(f"throughput averaged with 30 times") + tic1 = time.time() + for i in range(30): + model(images) + torch.cuda.synchronize() + tic2 = time.time() + logger.info( + f"batch_size {batch_size} throughput {30 * batch_size / (tic2 - tic1)}" + ) + return + + +def main(config): + # prepare data loaders + dataset_train, dataset_val, dataset_test, data_loader_train, \ + data_loader_val, data_loader_test, mixup_fn = build_loader(config) + + # build runner + logger.info(f"Creating model:{config.MODEL.TYPE}/{config.MODEL.NAME}") + model = build_model(config) + model.cuda() + logger.info(str(model)) + + # build optimizer + optimizer = build_optimizer(config, model) + + if config.AMP_OPT_LEVEL != "O0": + config.defrost() + if has_native_amp: + config.native_amp = True + use_amp = 'native' + elif has_apex: + config.apex_amp = True + use_amp = 'apex' + else: + use_amp = None + logger.warning( + "Neither APEX or native Torch AMP is available, using float32. " + "Install NVIDA apex or upgrade to PyTorch 1.6") + config.freeze() + + # setup automatic mixed-precision (AMP) loss scaling and op casting + amp_autocast = suppress # do nothing + loss_scaler = None + if config.AMP_OPT_LEVEL != "O0": + if use_amp == 'apex': + model, optimizer = amp.initialize(model, + optimizer, + opt_level=config.AMP_OPT_LEVEL) + loss_scaler = ApexScaler() + if config.LOCAL_RANK == 0: + logger.info( + 'Using NVIDIA APEX AMP. Training in mixed precision.') + if use_amp == 'native': + amp_autocast = torch.cuda.amp.autocast + loss_scaler = NativeScaler() + if config.LOCAL_RANK == 0: + logger.info( + 'Using native Torch AMP. Training in mixed precision.') + else: + if config.LOCAL_RANK == 0: + logger.info('AMP not enabled. Training in float32.') + + # put model on gpus + model = torch.nn.parallel.DistributedDataParallel( + model, device_ids=[config.LOCAL_RANK], broadcast_buffers=False) + + try: + model.register_comm_hook(state=None, hook=fp16_compress_hook) + logger.info('using fp16_compress_hook!') + except: + logger.info("cannot register fp16_compress_hook!") + + model_without_ddp = model.module + + n_parameters = sum(p.numel() for p in model.parameters() + if p.requires_grad) + logger.info(f"number of params: {n_parameters}") + if hasattr(model_without_ddp, 'flops'): + flops = model_without_ddp.flops() + logger.info(f"number of GFLOPs: {flops / 1e9}") + + # build learning rate scheduler + lr_scheduler = build_scheduler(config, optimizer, len(data_loader_train)) \ + if not config.EVAL_MODE else None + + # build criterion + if config.AUG.MIXUP > 0.: + # smoothing is handled with mixup label transform + criterion = SoftTargetCrossEntropy() + elif config.MODEL.LABEL_SMOOTHING > 0.: + criterion = LabelSmoothingCrossEntropy( + smoothing=config.MODEL.LABEL_SMOOTHING) + else: + criterion = torch.nn.CrossEntropyLoss() + + max_accuracy = 0.0 + max_ema_accuracy = 0.0 + # set auto resume + if config.MODEL.RESUME == '' and config.TRAIN.AUTO_RESUME: + resume_file = auto_resume_helper(config.OUTPUT) + if resume_file: + if config.MODEL.RESUME: + logger.warning( + f"auto-resume changing resume file from {config.MODEL.RESUME} to {resume_file}" + ) + config.defrost() + config.MODEL.RESUME = resume_file + config.freeze() + logger.info(f'auto resuming from {resume_file}') + else: + logger.info( + f'no checkpoint found in {config.OUTPUT}, ignoring auto resume' + ) + + # set resume and pretrain + if config.MODEL.RESUME: + max_accuracy = load_checkpoint(config, model_without_ddp, optimizer, + lr_scheduler, loss_scaler, logger) + if data_loader_val is not None: + acc1, acc5, loss = validate(config, data_loader_val, model, amp_autocast=amp_autocast) + logger.info( + f"Accuracy of the network on the {len(dataset_val)} test images: {acc1:.1f}%" + ) + elif config.MODEL.PRETRAINED: + load_pretrained(config, model_without_ddp, logger) + if data_loader_val is not None: + acc1, acc5, loss = validate(config, data_loader_val, model, amp_autocast=amp_autocast) + logger.info( + f"Accuracy of the network on the {len(dataset_val)} test images: {acc1:.1f}%" + ) + + # evaluate EMA + model_ema = None + if config.TRAIN.EMA.ENABLE: + # Important to create EMA model after cuda(), DP wrapper, and AMP but before SyncBN and DDP wrapper + model_ema = ModelEma(model, decay=config.TRAIN.EMA.DECAY) + print("Using EMA with decay = %.8f" % config.TRAIN.EMA.DECAY) + if config.MODEL.RESUME: + load_ema_checkpoint(config, model_ema, logger) + acc1, acc5, loss = validate(config, data_loader_val, model_ema.ema, amp_autocast=amp_autocast) + logger.info( + f"Accuracy of the ema network on the {len(dataset_val)} test images: {acc1:.1f}%" + ) + + if config.THROUGHPUT_MODE: + throughput(data_loader_val, model, logger) + + if config.EVAL_MODE: + return + + # train + logger.info("Start training") + start_time = time.time() + for epoch in range(config.TRAIN.START_EPOCH, config.TRAIN.EPOCHS): + data_loader_train.sampler.set_epoch(epoch) + + train_one_epoch(config, + model, + criterion, + data_loader_train, + optimizer, + epoch, + mixup_fn, + lr_scheduler, + amp_autocast, + loss_scaler, + model_ema=model_ema) + if (epoch % config.SAVE_FREQ == 0 or epoch == (config.TRAIN.EPOCHS - 1)) and \ + config.TRAIN.OPTIMIZER.USE_ZERO: + optimizer.consolidate_state_dict(to=0) + if dist.get_rank() == 0 and (epoch % config.SAVE_FREQ == 0 + or epoch == (config.TRAIN.EPOCHS - 1)): + save_checkpoint(config, + epoch, + model_without_ddp, + max_accuracy, + optimizer, + lr_scheduler, + loss_scaler, + logger, + model_ema=model_ema) + if data_loader_val is not None and epoch % config.EVAL_FREQ == 0: + acc1, acc5, loss = validate(config, data_loader_val, model, epoch, amp_autocast) + logger.info( + f"Accuracy of the network on the {len(dataset_val)} test images: {acc1:.1f}%" + ) + if dist.get_rank() == 0 and acc1 > max_accuracy: + save_checkpoint(config, + epoch, + model_without_ddp, + max_accuracy, + optimizer, + lr_scheduler, + loss_scaler, + logger, + model_ema=model_ema, + best='best') + max_accuracy = max(max_accuracy, acc1) + logger.info(f'Max accuracy: {max_accuracy:.2f}%') + + if config.TRAIN.EMA.ENABLE: + acc1, acc5, loss = validate(config, data_loader_val, + model_ema.ema, epoch, amp_autocast) + logger.info( + f"Accuracy of the ema network on the {len(dataset_val)} test images: {acc1:.1f}%" + ) + if dist.get_rank() == 0 and acc1 > max_ema_accuracy: + save_checkpoint(config, + epoch, + model_without_ddp, + max_accuracy, + optimizer, + lr_scheduler, + loss_scaler, + logger, + model_ema=model_ema, + best='ema_best') + max_ema_accuracy = max(max_ema_accuracy, acc1) + logger.info(f'Max ema accuracy: {max_ema_accuracy:.2f}%') + + total_time = time.time() - start_time + total_time_str = str(datetime.timedelta(seconds=int(total_time))) + logger.info('Training time {}'.format(total_time_str)) + + +def train_one_epoch(config, + model, + criterion, + data_loader, + optimizer, + epoch, + mixup_fn, + lr_scheduler, + amp_autocast=suppress, + loss_scaler=None, + model_ema=None): + model.train() + optimizer.zero_grad() + + num_steps = len(data_loader) + batch_time = AverageMeter() + model_time = AverageMeter() + loss_meter = AverageMeter() + norm_meter = MyAverageMeter(300) + + start = time.time() + end = time.time() + + amp_type = torch.float16 if config.AMP_TYPE == 'float16' else torch.bfloat16 + for idx, (samples, targets) in enumerate(data_loader): + iter_begin_time = time.time() + samples = samples.cuda(non_blocking=True) + targets = targets.cuda(non_blocking=True) + + if mixup_fn is not None: + samples, targets = mixup_fn(samples, targets) + + if not obsolete_torch_version(TORCH_VERSION, + (1, 9)) and config.AMP_OPT_LEVEL != "O0": + with amp_autocast(dtype=amp_type): + outputs = model(samples) + else: + with amp_autocast(): + outputs = model(samples) + + if config.TRAIN.ACCUMULATION_STEPS > 1: + if not obsolete_torch_version( + TORCH_VERSION, (1, 9)) and config.AMP_OPT_LEVEL != "O0": + with amp_autocast(dtype=amp_type): + loss = criterion(outputs, targets) + loss = loss / config.TRAIN.ACCUMULATION_STEPS + else: + with amp_autocast(): + loss = criterion(outputs, targets) + loss = loss / config.TRAIN.ACCUMULATION_STEPS + if config.AMP_OPT_LEVEL != "O0": + is_second_order = hasattr(optimizer, 'is_second_order') and \ + optimizer.is_second_order + grad_norm = loss_scaler(loss, + optimizer, + clip_grad=config.TRAIN.CLIP_GRAD, + parameters=model.parameters(), + create_graph=is_second_order, + update_grad=(idx + 1) % + config.TRAIN.ACCUMULATION_STEPS == 0) + if (idx + 1) % config.TRAIN.ACCUMULATION_STEPS == 0: + optimizer.zero_grad() + if model_ema is not None: + model_ema.update(model) + else: + loss.backward() + if config.TRAIN.CLIP_GRAD: + grad_norm = torch.nn.utils.clip_grad_norm_( + model.parameters(), config.TRAIN.CLIP_GRAD) + else: + grad_norm = get_grad_norm(model.parameters()) + if (idx + 1) % config.TRAIN.ACCUMULATION_STEPS == 0: + optimizer.step() + optimizer.zero_grad() + if model_ema is not None: + model_ema.update(model) + if (idx + 1) % config.TRAIN.ACCUMULATION_STEPS == 0: + lr_scheduler.step_update(epoch * num_steps + idx) + else: + if not obsolete_torch_version( + TORCH_VERSION, (1, 9)) and config.AMP_OPT_LEVEL != "O0": + with amp_autocast(dtype=amp_type): + loss = criterion(outputs, targets) + else: + with amp_autocast(): + loss = criterion(outputs, targets) + optimizer.zero_grad() + if config.AMP_OPT_LEVEL != "O0": + is_second_order = hasattr(optimizer, 'is_second_order') and \ + optimizer.is_second_order + grad_norm = loss_scaler(loss, + optimizer, + clip_grad=config.TRAIN.CLIP_GRAD, + parameters=model.parameters(), + create_graph=is_second_order, + update_grad=(idx + 1) % + config.TRAIN.ACCUMULATION_STEPS == 0) + if model_ema is not None: + model_ema.update(model) + else: + loss.backward() + if config.TRAIN.CLIP_GRAD: + grad_norm = torch.nn.utils.clip_grad_norm_( + model.parameters(), config.TRAIN.CLIP_GRAD) + else: + grad_norm = get_grad_norm(model.parameters()) + optimizer.step() + if model_ema is not None: + model_ema.update(model) + + lr_scheduler.step_update(epoch * num_steps + idx) + + torch.cuda.synchronize() + + loss_meter.update(loss.item(), targets.size(0)) + if grad_norm is not None: + norm_meter.update(grad_norm.item()) + batch_time.update(time.time() - end) + model_time.update(time.time() - iter_begin_time) + end = time.time() + + if idx % config.PRINT_FREQ == 0: + lr = optimizer.param_groups[0]['lr'] + memory_used = torch.cuda.max_memory_allocated() / (1024.0 * 1024.0) + etas = batch_time.avg * (num_steps - idx) + logger.info( + f'Train: [{epoch}/{config.TRAIN.EPOCHS}][{idx}/{num_steps}]\t' + f'eta {datetime.timedelta(seconds=int(etas))} lr {lr:.6f}\t' + f'time {batch_time.val:.4f} ({batch_time.avg:.4f})\t' + f'model_time {model_time.val:.4f} ({model_time.avg:.4f})\t' + f'loss {loss_meter.val:.4f} ({loss_meter.avg:.4f})\t' + f'grad_norm {norm_meter.val:.4f} ({norm_meter.avg:.4f}/{norm_meter.var:.4f})\t' + f'mem {memory_used:.0f}MB') + epoch_time = time.time() - start + logger.info( + f"EPOCH {epoch} training takes {datetime.timedelta(seconds=int(epoch_time))}" + ) + + +@torch.no_grad() +def validate(config, data_loader, model, epoch=None, amp_autocast=None): + criterion = torch.nn.CrossEntropyLoss() + model.eval() + + batch_time = AverageMeter() + loss_meter = AverageMeter() + acc1_meter = AverageMeter() + acc5_meter = AverageMeter() + + end = time.time() + for idx, (images, target) in enumerate(data_loader): + images = images.cuda(non_blocking=True) + target = target.cuda(non_blocking=True) + with amp_autocast(): + output = model(images) + + # convert 22k to 1k to evaluate + if output.size(-1) == 21841: + convert_file = './meta_data/map22kto1k.txt' + with open(convert_file, 'r') as f: + convert_list = [int(line) for line in f.readlines()] + output = output[:, convert_list] + + # measure accuracy and record loss + loss = criterion(output, target) + acc1, acc5 = accuracy(output, target, topk=(1, 5)) + + acc1 = reduce_tensor(acc1) + acc5 = reduce_tensor(acc5) + loss = reduce_tensor(loss) + + loss_meter.update(loss.item(), target.size(0)) + acc1_meter.update(acc1.item(), target.size(0)) + acc5_meter.update(acc5.item(), target.size(0)) + + # measure elapsed time + batch_time.update(time.time() - end) + end = time.time() + + if idx % config.PRINT_FREQ == 0: + memory_used = torch.cuda.max_memory_allocated() / (1024.0 * 1024.0) + logger.info(f'Test: [{idx}/{len(data_loader)}]\t' + f'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' + f'Loss {loss_meter.val:.4f} ({loss_meter.avg:.4f})\t' + f'Acc@1 {acc1_meter.val:.3f} ({acc1_meter.avg:.3f})\t' + f'Acc@5 {acc5_meter.val:.3f} ({acc5_meter.avg:.3f})\t' + f'Mem {memory_used:.0f}MB') + if epoch is not None: + logger.info( + f'[Epoch:{epoch}] * Acc@1 {acc1_meter.avg:.3f} Acc@5 {acc5_meter.avg:.3f}' + ) + else: + logger.info( + f' * Acc@1 {acc1_meter.avg:.3f} Acc@5 {acc5_meter.avg:.3f}') + + return acc1_meter.avg, acc5_meter.avg, loss_meter.avg + + +if __name__ == '__main__': + _, config = parse_option() + + if config.AMP_OPT_LEVEL != "O0": + assert has_native_amp, "Please update pytorch(1.6+) to support amp!" + + # init distributed env + if 'SLURM_PROCID' in os.environ and int(os.environ['SLURM_NNODES']) != 1: + print("\nDist init: SLURM") + rank = int(os.environ['SLURM_PROCID']) + gpu = rank % torch.cuda.device_count() + config.defrost() + config.LOCAL_RANK = gpu + config.freeze() + + world_size = int(os.environ["SLURM_NTASKS"]) + if "MASTER_PORT" not in os.environ: + os.environ["MASTER_PORT"] = "29501" + node_list = os.environ["SLURM_NODELIST"] + addr = subprocess.getoutput( + f"scontrol show hostname {node_list} | head -n1") + if "MASTER_ADDR" not in os.environ: + os.environ["MASTER_ADDR"] = addr + + os.environ['RANK'] = str(rank) + os.environ['LOCAL_RANK'] = str(gpu) + os.environ['LOCAL_SIZE'] = str(torch.cuda.device_count()) + os.environ['WORLD_SIZE'] = str(world_size) + if 'RANK' in os.environ and 'WORLD_SIZE' in os.environ: + rank = int(os.environ["RANK"]) + world_size = int(os.environ['WORLD_SIZE']) + print(f"RANK and WORLD_SIZE in environ: {rank}/{world_size}") + else: + rank = -1 + world_size = -1 + torch.cuda.set_device(config.LOCAL_RANK) + torch.distributed.init_process_group(backend='nccl', + init_method='env://', + world_size=world_size, + rank=rank) + torch.distributed.barrier() + + seed = config.SEED + dist.get_rank() + torch.manual_seed(seed) + torch.cuda.manual_seed(seed) + np.random.seed(seed) + random.seed(seed) + cudnn.benchmark = True + + # linear scale the learning rate according to total batch size, may not be optimal + linear_scaled_lr = config.TRAIN.BASE_LR * \ + config.DATA.BATCH_SIZE * dist.get_world_size() / 512.0 + linear_scaled_warmup_lr = config.TRAIN.WARMUP_LR * \ + config.DATA.BATCH_SIZE * dist.get_world_size() / 512.0 + linear_scaled_min_lr = config.TRAIN.MIN_LR * \ + config.DATA.BATCH_SIZE * dist.get_world_size() / 512.0 + # gradient accumulation also need to scale the learning rate + if config.TRAIN.ACCUMULATION_STEPS > 1: + linear_scaled_lr = linear_scaled_lr * config.TRAIN.ACCUMULATION_STEPS + linear_scaled_warmup_lr = linear_scaled_warmup_lr * config.TRAIN.ACCUMULATION_STEPS + linear_scaled_min_lr = linear_scaled_min_lr * config.TRAIN.ACCUMULATION_STEPS + config.defrost() + config.TRAIN.BASE_LR = linear_scaled_lr + config.TRAIN.WARMUP_LR = linear_scaled_warmup_lr + config.TRAIN.MIN_LR = linear_scaled_min_lr + print(config.AMP_OPT_LEVEL, _.amp_opt_level) + + config.freeze() + + os.makedirs(config.OUTPUT, exist_ok=True) + logger = create_logger(output_dir=config.OUTPUT, + dist_rank=dist.get_rank(), + name=f"{config.MODEL.NAME}") + + if dist.get_rank() == 0: + path = os.path.join(config.OUTPUT, "config.json") + with open(path, "w") as f: + f.write(config.dump()) + logger.info(f"Full config saved to {path}") + + # print config + logger.info(config.dump()) + + main(config) diff --git a/classification/main_accelerate.py b/classification/main_accelerate.py new file mode 100644 index 0000000..6b7828c --- /dev/null +++ b/classification/main_accelerate.py @@ -0,0 +1,380 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + + +import datetime +import argparse +import os +import time +import logging +import random + +import torch +import torch.backends.cudnn as cudnn +import numpy as np +from accelerate import Accelerator +from accelerate import GradScalerKwargs +from accelerate.logging import get_logger +from timm.loss import LabelSmoothingCrossEntropy, SoftTargetCrossEntropy +from timm.utils import AverageMeter, accuracy, ModelEma +from tqdm import tqdm +import warnings + +from config import get_config +from models import build_model +from dataset import build_loader2 +from lr_scheduler import build_scheduler +from optimizer import build_optimizer +from utils import load_pretrained, load_ema_checkpoint +from ddp_hooks import fp16_compress_hook + +logger = get_logger(__name__) +warnings.filterwarnings('ignore') + + +def parse_option(): + parser = argparse.ArgumentParser( + 'InternImage training and evaluation script', add_help=False) + parser.add_argument('--cfg', type=str, required=True, metavar="FILE", help='path to config file') + parser.add_argument("--opts", help="Modify config options by adding 'KEY VALUE' pairs. ", default=None, nargs='+') + + # easy config modification + parser.add_argument('--batch-size', type=int, help="batch size for single GPU") + parser.add_argument('--dataset', type=str, help='dataset name', default=None) + parser.add_argument('--data-path', type=str, help='path to dataset') + parser.add_argument('--zip', action='store_true', help='use zipped dataset instead of folder dataset') + parser.add_argument('--cache-mode', type=str, default='part', choices=['no', 'full', 'part'], + help='no: no cache, ' + 'full: cache all data, ' + 'part: sharding the dataset into nonoverlapping pieces and only cache one piece' + ) + parser.add_argument('--pretrained', help='pretrained weight from checkpoint, could be imagenet22k pretrained weight') + parser.add_argument('--resume', help='resume from checkpoint') + parser.add_argument('--output', default='output', type=str, metavar='PATH', + help='root of output folder, the full path is // (default: output)' + ) + parser.add_argument('--eval', action='store_true', help='Perform evaluation only') + parser.add_argument('--throughput', action='store_true', help='Test throughput only') + parser.add_argument('--save-ckpt-num', default=1, type=int) + parser.add_argument('--accumulation-steps', type=int, default=1, help="gradient accumulation steps") + parser.add_argument('--disable-grad-scalar', action='store_true', help='disable Grad Scalar') + parser.add_argument( + "--logger", + type=str, + default="tensorboard", + choices=["tensorboard", "wandb"], + help=( + "Whether to use [tensorboard](https://www.tensorflow.org/tensorboard) or [wandb](https://www.wandb.ai)" + " for experiment tracking and logging of model metrics and model checkpoints" + ), + ) + + args, unparsed = parser.parse_known_args() + config = get_config(args) + config.defrost() + config.TRAIN.OPTIMIZER.USE_ZERO = False + config.OUTPUT += '_deepspeed' + config.DATA.IMG_ON_MEMORY = False + config.freeze() + return args, config + + +def seed_everything(seed, rank): + seed = seed + rank + torch.manual_seed(seed) + torch.cuda.manual_seed(seed) + np.random.seed(seed) + random.seed(seed) + cudnn.benchmark = True + + +def save_config(config): + path = os.path.join(config.OUTPUT, "config.json") + with open(path, "w") as f: + f.write(config.dump()) + logger.info(f"Full config saved to {path}") + + +def build_criterion(config): + if config.AUG.MIXUP > 0.: + # smoothing is handled with mixup label transform + criterion = SoftTargetCrossEntropy() + elif config.MODEL.LABEL_SMOOTHING > 0.: + criterion = LabelSmoothingCrossEntropy( + smoothing=config.MODEL.LABEL_SMOOTHING) + else: + criterion = torch.nn.CrossEntropyLoss() + return criterion + + +def scale_learning_rate(config, num_processes): + # linear scale the learning rate according to total batch size, may not be optimal + linear_scaled_lr = config.TRAIN.BASE_LR * \ + config.DATA.BATCH_SIZE * num_processes / 512.0 + linear_scaled_warmup_lr = config.TRAIN.WARMUP_LR * \ + config.DATA.BATCH_SIZE * num_processes / 512.0 + linear_scaled_min_lr = config.TRAIN.MIN_LR * \ + config.DATA.BATCH_SIZE * num_processes / 512.0 + # gradient accumulation also need to scale the learning rate + if config.TRAIN.ACCUMULATION_STEPS > 1: + linear_scaled_lr = linear_scaled_lr * config.TRAIN.ACCUMULATION_STEPS + linear_scaled_warmup_lr = linear_scaled_warmup_lr * config.TRAIN.ACCUMULATION_STEPS + linear_scaled_min_lr = linear_scaled_min_lr * config.TRAIN.ACCUMULATION_STEPS + config.defrost() + config.TRAIN.BASE_LR = linear_scaled_lr + config.TRAIN.WARMUP_LR = linear_scaled_warmup_lr + config.TRAIN.MIN_LR = linear_scaled_min_lr + config.freeze() + + logger.info('BASE_LR={}'.format(config.TRAIN.BASE_LR)) + logger.info('WARMUP_LR={}'.format(config.TRAIN.WARMUP_LR)) + logger.info('MIN_LR={}'.format(config.TRAIN.MIN_LR)) + + +def setup_autoresume(config): + if config.MODEL.RESUME == '' and config.TRAIN.AUTO_RESUME: + last_checkpoint = os.path.join(config.OUTPUT, 'last') + resume_file = last_checkpoint if os.path.exists(last_checkpoint) else None + + if resume_file: + if config.MODEL.RESUME: + logger.warning(f"auto-resume changing resume file from {config.MODEL.RESUME} to {resume_file}") + config.defrost() + config.MODEL.RESUME = resume_file + config.freeze() + logger.info(f'auto resuming from {resume_file}') + else: + logger.info(f'no checkpoint found in {config.OUTPUT}, ignoring auto resume') + + +def load_model_checkpoint(config, model, accelerator): + if config.MODEL.RESUME: + try: + checkpoint = torch.load(config.MODEL.RESUME)['model'] + checkpoint = {k.replace('module.', ''): v for k, v in checkpoint.items()} + model.load_state_dict(checkpoint) + except: + accelerator.load_state(config.MODEL.RESUME) + elif config.MODEL.PRETRAINED: + try: + load_pretrained(config, model, logger) + except: + accelerator.load_state(config.MODEL.PRETRAINED) + return model + + +def save_checkpoint(save_dir, accelerator, epoch, max_acc, config, lr_scheduler=None): + # let accelerator handle the model and optimizer state for ddp and deepspeed. + accelerator.save_state(save_dir) + + if accelerator.is_main_process: + save_state = { + 'lr_scheduler': lr_scheduler.state_dict(), + 'max_acc': max_acc, + 'epoch': epoch, + 'config': config + } + torch.save(save_state, os.path.join(save_dir, 'additional_state.pth')) + + +def load_checkpoint_if_needed(accelerator, config, lr_scheduler=None): + setup_autoresume(config) + save_dir = config.MODEL.RESUME + if not save_dir: + return 0.0 + accelerator.load_state(save_dir) + checkpoint = torch.load(os.path.join(save_dir, 'additional_state.pth'), map_location='cpu') + if lr_scheduler is not None: + logger.info('resuming lr_scheduler') + lr_scheduler.load_state_dict(checkpoint['lr_scheduler']) + config.defrost() + config.TRAIN.START_EPOCH = checkpoint['epoch'] + 1 + config.freeze() + max_acc = checkpoint.get('max_acc', 0.0) + logger.info(f"=> loaded successfully {config.MODEL.RESUME} (epoch {checkpoint['epoch']})") + return max_acc + + +def log_model_statistic(model_wo_ddp): + n_parameters = sum(p.numel() for p in model_wo_ddp.parameters() + if p.requires_grad) + logger.info(f"number of params: {n_parameters}") + if hasattr(model_wo_ddp, 'flops'): + flops = model_wo_ddp.flops() + logger.info(f"number of GFLOPs: {flops / 1e9}") + + +def train_epoch(*, model, optimizer, data_loader, scheduler, criterion, mixup_fn, + accelerator: Accelerator, epoch, config): + model.train() + + num_steps = len(data_loader) + batch_time = AverageMeter() + model_time = AverageMeter() + loss_meter = AverageMeter() + + end = time.time() + + gradient_accumulation_steps = config.TRAIN.ACCUMULATION_STEPS + + for step, (samples, targets) in enumerate(data_loader): + iter_begin_time = time.time() + + if mixup_fn is not None: + samples, targets = mixup_fn(samples, targets) + + with accelerator.accumulate(model): + outputs = model(samples) + loss = criterion(outputs, targets) + accelerator.backward(loss) + if accelerator.sync_gradients: + accelerator.clip_grad_norm_(model.parameters(), config.TRAIN.CLIP_GRAD) + optimizer.step() + optimizer.zero_grad() + + accelerator.wait_for_everyone() + + if (step + 1) % gradient_accumulation_steps == 0: + if scheduler is not None: + scheduler.step_update((epoch * num_steps + step) // gradient_accumulation_steps) + + batch_time.update(time.time() - end) + model_time.update(time.time() - iter_begin_time) + loss_meter.update(loss.item()) + end = time.time() + + if accelerator.is_main_process and step % config.PRINT_FREQ == 0: + lr = optimizer.param_groups[0]['lr'] + memory_used = torch.cuda.max_memory_allocated() / (1024.0 * 1024.0) + etas = batch_time.avg * (num_steps - step) + + logger.info( + f'Train: [{epoch}/{config.TRAIN.EPOCHS}][{step}/{num_steps}]\t' + f'eta {datetime.timedelta(seconds=int(etas))} lr {lr:.10f}\t' + f'time {batch_time.val:.4f} ({batch_time.avg:.4f})\t' + f'model_time {model_time.val:.4f} ({model_time.avg:.4f})\t' + f'loss {loss_meter.val:.8f} ({loss_meter.avg:.4f})\t' + f'mem {memory_used:.0f}MB') + + +@torch.no_grad() +def eval_epoch(*, config, data_loader, model, accelerator: Accelerator): + model.eval() + + acc1_meter = AverageMeter() + acc5_meter = AverageMeter() + + for idx, (images, target) in enumerate(tqdm(data_loader, disable=accelerator.is_main_process)): + output = model(images) + + # convert 22k to 1k to evaluate + if output.size(-1) == 21841: + convert_file = './meta_data/map22kto1k.txt' + with open(convert_file, 'r') as f: + convert_list = [int(line) for line in f.readlines()] + output = output[:, convert_list] + + acc1, acc5 = accuracy(output, target, topk=(1, 5)) + acc1 = accelerator.gather(acc1).mean(0) + acc5 = accelerator.gather(acc5).mean(0) + + acc1_meter.update(acc1.item(), target.size(0)) + acc5_meter.update(acc5.item(), target.size(0)) + + if (idx + 1) % config.PRINT_FREQ == 0 or idx + 1 == len(data_loader): + logger.info(f'Test: [{idx+1}/{len(data_loader)}]\t' + f'Acc@1 {acc1_meter.val:.3f} ({acc1_meter.avg:.3f})\t' + f'Acc@5 {acc5_meter.val:.3f} ({acc5_meter.avg:.3f})\t' + ) + return acc1_meter.avg + + +def eval(config, accelerator: Accelerator): + _, _, _, _, validate_dataloader, _, _ = build_loader2(config) + model = build_model(config) + model, validate_dataloader = accelerator.prepare(model, validate_dataloader) + model = load_model_checkpoint(config, model, accelerator) + log_model_statistic(accelerator.unwrap_model(model)) + eval_epoch(config=config, data_loader=validate_dataloader, model=model, accelerator=accelerator) + + +def train(config, accelerator: Accelerator): + _, _, _, training_dataloader, validate_dataloader, _, mixup_fn = build_loader2(config) + model = build_model(config) + optimizer = build_optimizer(config, model) + criterion = build_criterion(config) + + model, optimizer, training_dataloader, validate_dataloader = accelerator.prepare( + model, optimizer, training_dataloader, validate_dataloader) + + effective_update_steps_per_epoch = len(training_dataloader) // config.TRAIN.ACCUMULATION_STEPS + lr_scheduler = build_scheduler(config, optimizer, effective_update_steps_per_epoch) + + try: + model.register_comm_hook(state=None, hook=fp16_compress_hook) + logger.info('using fp16_compress_hook!') + except: + logger.info("cannot register fp16_compress_hook!") + + max_acc = load_checkpoint_if_needed(accelerator, config, lr_scheduler) + + logger.info(f"Created model:{config.MODEL.TYPE}/{config.MODEL.NAME}") + logger.info(str(model)) + logger.info("Effective Optimizer Steps: {}".format(effective_update_steps_per_epoch)) + logger.info("Start training") + logger.info("Max accuracy: {}".format(max_acc)) + log_model_statistic(accelerator.unwrap_model(model)) + + for epoch in range(config.TRAIN.START_EPOCH, config.TRAIN.EPOCHS): + train_epoch(model=model, optimizer=optimizer, data_loader=training_dataloader, + scheduler=lr_scheduler, criterion=criterion, mixup_fn=mixup_fn, + accelerator=accelerator, epoch=epoch, config=config) + acc = eval_epoch(config=config, data_loader=validate_dataloader, model=model, + accelerator=accelerator) + + accelerator.wait_for_everyone() + if acc > max_acc: + max_acc = acc + save_checkpoint(os.path.join(config.OUTPUT, 'best'), accelerator, epoch, max_acc, config, lr_scheduler) + logger.info(f'Max Acc@1 {max_acc:.3f}') + save_checkpoint(os.path.join(config.OUTPUT, 'last'), accelerator, epoch, max_acc, config, lr_scheduler) + + +def main(): + args, config = parse_option() + os.makedirs(config.OUTPUT, exist_ok=True) + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + filename=os.path.join(config.OUTPUT, 'run.log'), + level=logging.INFO, + ) + + loggers = ['tensorboard'] + accelerator = Accelerator( + log_with=loggers, + project_dir=config.OUTPUT, + gradient_accumulation_steps=config.TRAIN.ACCUMULATION_STEPS, + # When use deepspeed, you could not comment this out + # even if you set loss scale to 1.0 in deepspeed config. + kwargs_handlers=[GradScalerKwargs(enabled=not args.disable_grad_scalar)], + ) + logger.info(accelerator.state, main_process_only=False) + + scale_learning_rate(config, accelerator.num_processes) + seed_everything(config.SEED, accelerator.process_index) + save_config(config) + + logger.info(config.dump()) + + if config.EVAL_MODE: + eval(config, accelerator) + else: + train(config, accelerator) + + +if __name__ == '__main__': + main() diff --git a/classification/main_deepspeed.py b/classification/main_deepspeed.py new file mode 100644 index 0000000..66a9e2f --- /dev/null +++ b/classification/main_deepspeed.py @@ -0,0 +1,531 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + + +import os +import time +import random +import argparse +import datetime +import numpy as np +import subprocess + +import torch +import torch.backends.cudnn as cudnn +import torch.distributed as dist +import deepspeed +from timm.loss import LabelSmoothingCrossEntropy, SoftTargetCrossEntropy +from timm.utils import accuracy, AverageMeter + +from config import get_config +from models import build_model +from dataset import build_loader +from lr_scheduler import build_scheduler +from optimizer import set_weight_decay_and_lr +from logger import create_logger +from utils import load_pretrained, reduce_tensor, MyAverageMeter +from ddp_hooks import fp16_compress_hook +from ema_deepspeed import EMADeepspeed + +def parse_option(): + parser = argparse.ArgumentParser( + 'InternImage training and evaluation script', add_help=False) + parser.add_argument('--cfg', type=str, required=True, metavar="FILE", help='path to config file') + parser.add_argument("--opts", help="Modify config options by adding 'KEY VALUE' pairs. ", default=None, nargs='+') + + # easy config modification + parser.add_argument('--batch-size', type=int, help="batch size for single GPU") + parser.add_argument('--dataset', type=str, help='dataset name', default=None) + parser.add_argument('--data-path', type=str, help='path to dataset') + parser.add_argument('--zip', action='store_true', help='use zipped dataset instead of folder dataset') + parser.add_argument('--cache-mode', type=str, default='part', choices=['no', 'full', 'part'], + help='no: no cache, ' + 'full: cache all data, ' + 'part: sharding the dataset into nonoverlapping pieces and only cache one piece' + ) + parser.add_argument('--pretrained', help='pretrained weight from checkpoint, could be imagenet22k pretrained weight') + parser.add_argument('--resume', help='resume from checkpoint') + parser.add_argument('--output', default='output', type=str, metavar='PATH', + help='root of output folder, the full path is // (default: output)' + ) + + parser.add_argument('--eval', action='store_true', help='Perform evaluation only') + parser.add_argument('--throughput', action='store_true', help='Test throughput only') + parser.add_argument('--save-ckpt-num', default=1, type=int) + parser.add_argument('--accumulation-steps', type=int, default=1, help="gradient accumulation steps") + + # distributed training + parser.add_argument("--local-rank", type=int, required=True, help='local rank for DistributedDataParallel') + parser.add_argument('--disable-grad-scalar', action='store_true', help='disable Grad Scalar') + + args, unparsed = parser.parse_known_args() + config = get_config(args) + + return args, config + + +def seed_everything(seed, rank): + seed = seed + rank + torch.manual_seed(seed) + torch.cuda.manual_seed(seed) + np.random.seed(seed) + random.seed(seed) + cudnn.benchmark = True + + +def save_config(config): + path = os.path.join(config.OUTPUT, "config.json") + with open(path, "w") as f: + f.write(config.dump()) + logger.info(f"Full config saved to {path}") + + +def build_criterion(config): + if config.AUG.MIXUP > 0.: + # smoothing is handled with mixup label transform + criterion = SoftTargetCrossEntropy() + elif config.MODEL.LABEL_SMOOTHING > 0.: + criterion = LabelSmoothingCrossEntropy( + smoothing=config.MODEL.LABEL_SMOOTHING) + else: + criterion = torch.nn.CrossEntropyLoss() + return criterion + + +def scale_learning_rate(config, num_processes): + # linear scale the learning rate according to total batch size, may not be optimal + linear_scaled_lr = config.TRAIN.BASE_LR * \ + config.DATA.BATCH_SIZE * num_processes / 512.0 + linear_scaled_warmup_lr = config.TRAIN.WARMUP_LR * \ + config.DATA.BATCH_SIZE * num_processes / 512.0 + linear_scaled_min_lr = config.TRAIN.MIN_LR * \ + config.DATA.BATCH_SIZE * num_processes / 512.0 + # gradient accumulation also need to scale the learning rate + if config.TRAIN.ACCUMULATION_STEPS > 1: + linear_scaled_lr = linear_scaled_lr * config.TRAIN.ACCUMULATION_STEPS + linear_scaled_warmup_lr = linear_scaled_warmup_lr * config.TRAIN.ACCUMULATION_STEPS + linear_scaled_min_lr = linear_scaled_min_lr * config.TRAIN.ACCUMULATION_STEPS + config.defrost() + config.TRAIN.BASE_LR = linear_scaled_lr + config.TRAIN.WARMUP_LR = linear_scaled_warmup_lr + config.TRAIN.MIN_LR = linear_scaled_min_lr + config.freeze() + + logger.info('BASE_LR={}'.format(config.TRAIN.BASE_LR)) + logger.info('WARMUP_LR={}'.format(config.TRAIN.WARMUP_LR)) + logger.info('MIN_LR={}'.format(config.TRAIN.MIN_LR)) + + +def log_model_statistic(model_wo_ddp): + n_parameters = sum(p.numel() for p in model_wo_ddp.parameters() + if p.requires_grad) + logger.info(f"number of params: {n_parameters/1e6} M") + if hasattr(model_wo_ddp, 'flops'): + flops = model_wo_ddp.flops() + logger.info(f"number of GFLOPs: {flops / 1e9}") + + +def get_parameter_groups(model, config): + skip = {} + skip_keywords = {} + if hasattr(model, 'no_weight_decay'): + skip = model.no_weight_decay() + if hasattr(model, 'no_weight_decay_keywords'): + skip_keywords = model.no_weight_decay_keywords() + + parameters = set_weight_decay_and_lr( + model, + config.TRAIN.WEIGHT_DECAY, + config.TRAIN.BASE_LR, + skip, + skip_keywords, + lr_layer_decay=config.TRAIN.LR_LAYER_DECAY, + lr_layer_decay_ratio=config.TRAIN.LR_LAYER_DECAY_RATIO, + freeze_backbone=config.TRAIN.OPTIMIZER.FREEZE_BACKBONE, + dcn_lr_mul=config.TRAIN.OPTIMIZER.DCN_LR_MUL, + ) + return parameters + + +def get_optimizer_state_str(optimizer): + states = [] + for param_group in optimizer.param_groups: + states.append(f'name={param_group["name"]} lr={param_group["lr"]} weight_decay={param_group["weight_decay"]}') + return '\n'.join(states) + + +def build_ds_config(config, args): + opt_lower = config.TRAIN.OPTIMIZER.NAME.lower() + if opt_lower == 'adamw': + optimizer = { + "type": "AdamW", + "params": { + "lr": config.TRAIN.BASE_LR, + "eps": config.TRAIN.OPTIMIZER.EPS, + "betas": config.TRAIN.OPTIMIZER.BETAS, + "weight_decay": config.TRAIN.WEIGHT_DECAY + } + } + else: + return NotImplemented + + ds_config = { + "train_micro_batch_size_per_gpu": config.DATA.BATCH_SIZE, + "optimizer": optimizer, + "fp16": { + "enabled": True, + "auto_cast": True, + "loss_scale": 1 if args.disable_grad_scalar else 0 + }, + "zero_optimization": { + "stage": 1, + }, + "steps_per_print": 1e10, + "gradient_accumulation_steps": config.TRAIN.ACCUMULATION_STEPS, + "gradient_clipping": config.TRAIN.CLIP_GRAD, + } + return ds_config + + +@torch.no_grad() +def throughput(data_loader, model, logger): + model.eval() + + for idx, (images, _) in enumerate(data_loader): + images = images.cuda(non_blocking=True) + batch_size = images.shape[0] + for i in range(50): + model(images) + torch.cuda.synchronize() + logger.info(f"throughput averaged with 30 times") + tic1 = time.time() + for i in range(30): + model(images) + torch.cuda.synchronize() + tic2 = time.time() + logger.info( + f"batch_size {batch_size} throughput {30 * batch_size / (tic2 - tic1)}" + ) + return + + +def train_epoch(config, model, criterion, data_loader, optimizer, epoch, mixup_fn, lr_scheduler, model_ema=None): + model.train() + + num_steps = len(data_loader) + batch_time = AverageMeter() + model_time = AverageMeter() + loss_meter = AverageMeter() + norm_meter = MyAverageMeter(300) + + start = time.time() + end = time.time() + + for idx, (samples, targets) in enumerate(data_loader): + iter_begin_time = time.time() + samples = samples.cuda(non_blocking=True) + targets = targets.cuda(non_blocking=True) + + if mixup_fn is not None: + samples, targets = mixup_fn(samples, targets) + + outputs = model(samples) + loss = criterion(outputs, targets) + + model.backward(loss) + model.step() + + if model_ema is not None: + model_ema(model) + + if (idx + 1) % config.TRAIN.ACCUMULATION_STEPS == 0: + lr_scheduler.step_update(epoch * num_steps + idx) + + torch.cuda.synchronize() + loss_meter.update(loss.item(), targets.size(0)) + norm_meter.update(optimizer._global_grad_norm) + batch_time.update(time.time() - end) + model_time.update(time.time() - iter_begin_time) + end = time.time() + + if idx % config.PRINT_FREQ == 0: + lr = optimizer.param_groups[0]['lr'] + memory_used = torch.cuda.max_memory_allocated() / (1024.0 * 1024.0) + etas = batch_time.avg * (num_steps - idx) + logger.info( + f'Train: [{epoch}/{config.TRAIN.EPOCHS}][{idx}/{num_steps}]\t' + f'eta {datetime.timedelta(seconds=int(etas))} lr {lr:.6f}\t' + f'time {batch_time.val:.4f} ({batch_time.avg:.4f})\t' + f'model_time {model_time.val:.4f} ({model_time.avg:.4f})\t' + f'loss {loss_meter.val:.4f} ({loss_meter.avg:.4f})\t' + f'grad_norm {norm_meter.val:.4f} ({norm_meter.avg:.4f}/{norm_meter.var:.4f})\t' + f'mem {memory_used:.0f}MB') + + epoch_time = time.time() - start + logger.info(f"EPOCH {epoch} training takes {datetime.timedelta(seconds=int(epoch_time))}") + + +@torch.no_grad() +def eval_epoch(config, data_loader, model, epoch=None): + criterion = torch.nn.CrossEntropyLoss() + model.eval() + + batch_time = AverageMeter() + loss_meter = AverageMeter() + acc1_meter = AverageMeter() + acc5_meter = AverageMeter() + + end = time.time() + for idx, (images, target) in enumerate(data_loader): + images = images.cuda(non_blocking=True) + target = target.cuda(non_blocking=True) + output = model(images) + + # convert 22k to 1k to evaluate + if output.size(-1) == 21841: + convert_file = './meta_data/map22kto1k.txt' + with open(convert_file, 'r') as f: + convert_list = [int(line) for line in f.readlines()] + output = output[:, convert_list] + + # measure accuracy and record loss + loss = criterion(output, target) + acc1, acc5 = accuracy(output, target, topk=(1, 5)) + + acc1 = reduce_tensor(acc1) + acc5 = reduce_tensor(acc5) + loss = reduce_tensor(loss) + + loss_meter.update(loss.item(), target.size(0)) + acc1_meter.update(acc1.item(), target.size(0)) + acc5_meter.update(acc5.item(), target.size(0)) + + # measure elapsed time + batch_time.update(time.time() - end) + end = time.time() + + if idx % config.PRINT_FREQ == 0: + memory_used = torch.cuda.max_memory_allocated() / (1024.0 * 1024.0) + logger.info(f'Test: [{idx}/{len(data_loader)}]\t' + f'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' + f'Loss {loss_meter.val:.4f} ({loss_meter.avg:.4f})\t' + f'Acc@1 {acc1_meter.val:.3f} ({acc1_meter.avg:.3f})\t' + f'Acc@5 {acc5_meter.val:.3f} ({acc5_meter.avg:.3f})\t' + f'Mem {memory_used:.0f}MB') + if epoch is not None: + logger.info(f'[Epoch:{epoch}] * Acc@1 {acc1_meter.avg:.3f} Acc@5 {acc5_meter.avg:.3f}') + else: + logger.info(f' * Acc@1 {acc1_meter.avg:.3f} Acc@5 {acc5_meter.avg:.3f}') + + return acc1_meter.avg, acc5_meter.avg, loss_meter.avg + + +def train(config, ds_config): + # -------------- build ---------------- # + + _, dataset_val, _, data_loader_train, data_loader_val, _, mixup_fn = build_loader(config) + model = build_model(config) + model.cuda() + + if config.MODEL.PRETRAINED: + load_pretrained(config, model, logger) + + logger.info(ds_config) + model, optimizer, _, _ = deepspeed.initialize( + config=ds_config, + model=model, + model_parameters=get_parameter_groups(model, config), + dist_init_required=False, + ) + + try: + model.register_comm_hook(state=None, hook=fp16_compress_hook) + logger.info('using fp16_compress_hook!') + except: + logger.info("cannot register fp16_compress_hook!") + + model_without_ddp = model.module + + lr_scheduler = build_scheduler(config, optimizer, len(data_loader_train)) + criterion = build_criterion(config) + + model_ema = None + if config.TRAIN.EMA.ENABLE: + model_ema = EMADeepspeed(model, config.TRAIN.EMA.DECAY) + + # -------------- resume ---------------- # + + max_accuracy = 0.0 + max_accuracy_ema = 0.0 + client_state = {} + if config.MODEL.RESUME == '' and config.TRAIN.AUTO_RESUME: + if os.path.exists(os.path.join(config.OUTPUT, 'latest')): + config.defrost() + config.MODEL.RESUME = config.OUTPUT + config.freeze() + tag = None + elif config.MODEL.RESUME: + config.MODEL.RESUME = os.path.dirname(config.MODEL.RESUME) + tag = os.path.basename(config.MODEL.RESUME) + if config.MODEL.RESUME: + logger.info('loading checkpoint from {}'.format(config.MODEL.RESUME)) + _, client_state = model.load_checkpoint(load_dir=config.MODEL.RESUME, tag=tag) + logger.info(f'client_state={client_state.keys()}') + lr_scheduler.load_state_dict(client_state['custom_lr_scheduler']) + max_accuracy = client_state['max_accuracy'] + + if model_ema is not None: + max_accuracy_ema = client_state.get('max_accuracy_ema', 0.0) + model_ema.load_state_dict((client_state['model_ema'])) + + # -------------- training ---------------- # + + logger.info(f"Creating model:{config.MODEL.TYPE}/{config.MODEL.NAME}") + logger.info(str(model)) + logger.info(get_optimizer_state_str(optimizer)) + logger.info("Start training") + logger.info('max_accuracy: {}'.format(max_accuracy)) + log_model_statistic(model_without_ddp) + + start_time = time.time() + start_epoch = client_state['epoch'] + 1 if 'epoch' in client_state else config.TRAIN.START_EPOCH + for epoch in range(start_epoch, config.TRAIN.EPOCHS): + data_loader_train.sampler.set_epoch(epoch) + train_epoch(config, model, criterion, data_loader_train, optimizer, epoch, mixup_fn, lr_scheduler, + model_ema=model_ema) + + if epoch % config.SAVE_FREQ == 0 or epoch == config.TRAIN.EPOCHS - 1: + model.save_checkpoint( + save_dir=config.OUTPUT, + tag=f'epoch{epoch}', + client_state={ + 'custom_lr_scheduler': lr_scheduler.state_dict(), + 'max_accuracy': max_accuracy, + 'epoch': epoch, + 'config': config, + 'max_accuracy_ema': max_accuracy_ema if model_ema is not None else 0.0, + 'model_ema': model_ema.state_dict() if model_ema is not None else None, + } + ) + + if epoch % config.EVAL_FREQ == 0: + acc1, _, _ = eval_epoch(config, data_loader_val, model, epoch) + logger.info(f"Accuracy of the network on the {len(dataset_val)} test images: {acc1:.1f}%") + + if acc1 > max_accuracy: + model.save_checkpoint( + save_dir=config.OUTPUT, + tag='best', + client_state={ + 'custom_lr_scheduler': lr_scheduler.state_dict(), + 'max_accuracy': max_accuracy, + 'epoch': epoch, + 'config': config, + 'max_accuracy_ema': max_accuracy_ema if model_ema is not None else 0.0, + 'model_ema': model_ema.state_dict() if model_ema is not None else None, + } + ) + + max_accuracy = max(max_accuracy, acc1) + logger.info(f'Max accuracy: {max_accuracy:.2f}%') + + if model_ema is not None: + with model_ema.activate(model): + acc1_ema, _, _ = eval_epoch(config, data_loader_val, model, epoch) + logger.info(f"[EMA] Accuracy of the network on the {len(dataset_val)} test images: {acc1_ema:.1f}%") + max_accuracy_ema = max(max_accuracy_ema, acc1_ema) + logger.info(f'[EMA] Max accuracy: {max_accuracy_ema:.2f}%') + + total_time = time.time() - start_time + total_time_str = str(datetime.timedelta(seconds=int(total_time))) + logger.info('Training time {}'.format(total_time_str)) + + +def eval(config): + _, _, _, _, data_loader_val, _, _ = build_loader(config) + model = build_model(config) + model.cuda() + model = torch.nn.parallel.DistributedDataParallel( + model, device_ids=[config.LOCAL_RANK], broadcast_buffers=False) + + model_wo_ddp = model.module + if config.MODEL.RESUME: + try: + checkpoint = torch.load(config.MODEL.RESUME, map_location='cpu') + msg = model_wo_ddp.load_state_dict(checkpoint['model'], strict=False) + logger.info(msg) + except: + try: + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + ckpt_dir = os.path.dirname(config.MODEL.RESUME) + tag = os.path.basename(config.MODEL.RESUME) + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir=ckpt_dir, tag=tag) + model_wo_ddp.load_state_dict(state_dict) + except: + checkpoint = torch.load(os.path.join(config.MODEL.RESUME, 'mp_rank_00_model_states.pt'), map_location='cpu') + model_wo_ddp.load_state_dict(checkpoint['module']) + elif config.MODEL.PRETRAINED: + load_pretrained(config, model_wo_ddp, logger) + + if config.THROUGHPUT_MODE: + throughput(data_loader_val, model, logger) + + eval_epoch(config, data_loader_val, model) + + +if __name__ == '__main__': + args, config = parse_option() + + # init distributed env + if 'SLURM_PROCID' in os.environ and int(os.environ['SLURM_TASKS_PER_NODE']) != 1: + print("\nDist init: SLURM") + rank = int(os.environ['SLURM_PROCID']) + gpu = rank % torch.cuda.device_count() + config.defrost() + config.LOCAL_RANK = gpu + config.freeze() + + world_size = int(os.environ["SLURM_NTASKS"]) + if "MASTER_PORT" not in os.environ: + os.environ["MASTER_PORT"] = "29501" + node_list = os.environ["SLURM_NODELIST"] + addr = subprocess.getoutput( + f"scontrol show hostname {node_list} | head -n1") + if "MASTER_ADDR" not in os.environ: + os.environ["MASTER_ADDR"] = addr + + os.environ['RANK'] = str(rank) + os.environ['LOCAL_RANK'] = str(gpu) + os.environ['LOCAL_SIZE'] = str(torch.cuda.device_count()) + os.environ['WORLD_SIZE'] = str(world_size) + if 'RANK' in os.environ and 'WORLD_SIZE' in os.environ: + rank = int(os.environ["RANK"]) + world_size = int(os.environ['WORLD_SIZE']) + print(f"RANK and WORLD_SIZE in environ: {rank}/{world_size}") + else: + rank = -1 + world_size = -1 + torch.cuda.set_device(config.LOCAL_RANK) + torch.distributed.init_process_group(backend='nccl', + init_method='env://', + world_size=world_size, + rank=rank) + torch.distributed.barrier() + + os.makedirs(config.OUTPUT, exist_ok=True) + logger = create_logger(output_dir=config.OUTPUT, + dist_rank=dist.get_rank(), + name=f"{config.MODEL.NAME}") + logger.info(config.dump()) + + if dist.get_rank() == 0: save_config(config) + scale_learning_rate(config, dist.get_world_size()) + seed_everything(config.SEED, dist.get_rank()) + + if config.EVAL_MODE: + eval(config) + else: + train(config, build_ds_config(config, args)) diff --git a/classification/meta_data/22k_class_to_idx.json b/classification/meta_data/22k_class_to_idx.json new file mode 100644 index 0000000..221492e --- /dev/null +++ b/classification/meta_data/22k_class_to_idx.json @@ -0,0 +1 @@ +{"n00004475": 0, "n00005787": 1, "n00006024": 2, "n00006484": 3, "n00007846": 4, "n00015388": 5, "n00017222": 6, "n00021265": 7, "n00021939": 8, "n00120010": 9, "n00141669": 10, "n00288000": 11, "n00288190": 12, "n00288384": 13, "n00324978": 14, "n00326094": 15, "n00433458": 16, "n00433661": 17, "n00433802": 18, "n00434075": 19, "n00439826": 20, "n00440039": 21, "n00440218": 22, "n00440382": 23, "n00440509": 24, "n00440643": 25, "n00440747": 26, "n00440941": 27, "n00441073": 28, "n00441824": 29, "n00442115": 30, "n00442437": 31, "n00442847": 32, "n00442981": 33, "n00443231": 34, "n00443375": 35, "n00443517": 36, "n00443692": 37, "n00443803": 38, "n00443917": 39, "n00444142": 40, "n00444340": 41, "n00444490": 42, "n00444651": 43, "n00444846": 44, "n00444937": 45, "n00445055": 46, "n00445226": 47, "n00445351": 48, "n00445685": 49, "n00445802": 50, "n00446311": 51, "n00446411": 52, "n00446493": 53, "n00446632": 54, "n00446804": 55, "n00446980": 56, "n00447073": 57, "n00447221": 58, "n00447361": 59, "n00447463": 60, "n00447540": 61, "n00447957": 62, "n00448126": 63, "n00448232": 64, "n00448466": 65, "n00448640": 66, "n00448748": 67, "n00448872": 68, "n00448958": 69, "n00449054": 70, "n00449168": 71, "n00449295": 72, "n00449517": 73, "n00449695": 74, "n00449796": 75, "n00449892": 76, "n00449977": 77, "n00450070": 78, "n00450335": 79, "n00450700": 80, "n00450866": 81, "n00450998": 82, "n00451186": 83, "n00451370": 84, "n00451563": 85, "n00451635": 86, "n00451768": 87, "n00451866": 88, "n00452034": 89, "n00452152": 90, "n00452293": 91, "n00452734": 92, "n00452864": 93, "n00453126": 94, "n00453313": 95, "n00453396": 96, "n00453478": 97, "n00453631": 98, "n00453935": 99, "n00454237": 100, "n00454395": 101, "n00454493": 102, "n00454624": 103, "n00454855": 104, "n00454983": 105, "n00455076": 106, "n00455173": 107, "n00456465": 108, "n00463246": 109, "n00463543": 110, "n00464277": 111, "n00464478": 112, "n00464651": 113, "n00464894": 114, "n00466273": 115, "n00466377": 116, "n00466524": 117, "n00466630": 118, "n00466712": 119, "n00466880": 120, "n00467320": 121, "n00467536": 122, "n00467719": 123, "n00467995": 124, "n00468299": 125, "n00468480": 126, "n00469651": 127, "n00470554": 128, "n00470682": 129, "n00470830": 130, "n00470966": 131, "n00471437": 132, "n00471613": 133, "n00474568": 134, "n00474657": 135, "n00474769": 136, "n00474881": 137, "n00475014": 138, "n00475142": 139, "n00475273": 140, "n00475403": 141, "n00475535": 142, "n00475661": 143, "n00475787": 144, "n00476140": 145, "n00476235": 146, "n00476389": 147, "n00477392": 148, "n00477639": 149, "n00477827": 150, "n00478262": 151, "n00479076": 152, "n00479440": 153, "n00479616": 154, "n00479734": 155, "n00479887": 156, "n00480211": 157, "n00480366": 158, "n00480508": 159, "n00480885": 160, "n00480993": 161, "n00481803": 162, "n00481938": 163, "n00482122": 164, "n00482298": 165, "n00483205": 166, "n00483313": 167, "n00483409": 168, "n00483508": 169, "n00483605": 170, "n00483705": 171, "n00483848": 172, "n00523513": 173, "n00812526": 174, "n00825773": 175, "n00887544": 176, "n01035504": 177, "n01035667": 178, "n01055165": 179, "n01314388": 180, "n01314663": 181, "n01314781": 182, "n01314910": 183, "n01315213": 184, "n01315330": 185, "n01315581": 186, "n01315805": 187, "n01316422": 188, "n01316579": 189, "n01316734": 190, "n01316949": 191, "n01317089": 192, "n01317294": 193, "n01317391": 194, "n01317541": 195, "n01317813": 196, "n01317916": 197, "n01318053": 198, "n01318279": 199, "n01318381": 200, "n01318478": 201, "n01318660": 202, "n01318894": 203, "n01319001": 204, "n01319187": 205, "n01319467": 206, "n01319685": 207, "n01320872": 208, "n01321123": 209, "n01321230": 210, "n01321456": 211, "n01321579": 212, "n01321770": 213, "n01321854": 214, "n01322221": 215, "n01322343": 216, "n01322508": 217, "n01322604": 218, "n01322685": 219, "n01322898": 220, "n01322983": 221, "n01323068": 222, "n01323155": 223, "n01323261": 224, "n01323355": 225, "n01323493": 226, "n01323599": 227, "n01323781": 228, "n01324305": 229, "n01324431": 230, "n01324610": 231, "n01324799": 232, "n01324916": 233, "n01325060": 234, "n01326291": 235, "n01327909": 236, "n01329186": 237, "n01330126": 238, "n01330497": 239, "n01332181": 240, "n01333082": 241, "n01333483": 242, "n01333610": 243, "n01334217": 244, "n01334690": 245, "n01335218": 246, "n01337191": 247, "n01337734": 248, "n01338685": 249, "n01339083": 250, "n01339336": 251, "n01339471": 252, "n01339801": 253, "n01340014": 254, "n01340522": 255, "n01340785": 256, "n01340935": 257, "n01341090": 258, "n01342269": 259, "n01347583": 260, "n01349735": 261, "n01350226": 262, "n01350701": 263, "n01351170": 264, "n01351315": 265, "n01357328": 266, "n01357507": 267, "n01358572": 268, "n01359762": 269, "n01362336": 270, "n01363719": 271, "n01365474": 272, "n01365885": 273, "n01366700": 274, "n01367772": 275, "n01368672": 276, "n01369358": 277, "n01369484": 278, "n01374703": 279, "n01374846": 280, "n01375204": 281, "n01376237": 282, "n01376437": 283, "n01376543": 284, "n01377278": 285, "n01377510": 286, "n01377694": 287, "n01378545": 288, "n01379389": 289, "n01380610": 290, "n01380754": 291, "n01381044": 292, "n01382033": 293, "n01384084": 294, "n01384164": 295, "n01384687": 296, "n01385017": 297, "n01385330": 298, "n01386007": 299, "n01386182": 300, "n01386354": 301, "n01387065": 302, "n01389507": 303, "n01390123": 304, "n01390763": 305, "n01392275": 306, "n01392380": 307, "n01393486": 308, "n01394040": 309, "n01394492": 310, "n01394771": 311, "n01395254": 312, "n01396048": 313, "n01396617": 314, "n01397114": 315, "n01397690": 316, "n01397871": 317, "n01400247": 318, "n01400391": 319, "n01402600": 320, "n01403457": 321, "n01404365": 322, "n01404495": 323, "n01405007": 324, "n01405616": 325, "n01407798": 326, "n01410457": 327, "n01411450": 328, "n01412694": 329, "n01413457": 330, "n01414216": 331, "n01415626": 332, "n01415920": 333, "n01416213": 334, "n01418498": 335, "n01418620": 336, "n01419332": 337, "n01419573": 338, "n01419888": 339, "n01421333": 340, "n01421807": 341, "n01422185": 342, "n01422335": 343, "n01422450": 344, "n01423302": 345, "n01423617": 346, "n01424420": 347, "n01425223": 348, "n01427399": 349, "n01429172": 350, "n01438208": 351, "n01438581": 352, "n01439121": 353, "n01439514": 354, "n01439808": 355, "n01440160": 356, "n01440242": 357, "n01440467": 358, "n01440764": 359, "n01441117": 360, "n01441272": 361, "n01441425": 362, "n01441910": 363, "n01442450": 364, "n01442710": 365, "n01442972": 366, "n01443243": 367, "n01443537": 368, "n01443831": 369, "n01444339": 370, "n01444783": 371, "n01445429": 372, "n01445593": 373, "n01445857": 374, "n01446152": 375, "n01446589": 376, "n01446760": 377, "n01447139": 378, "n01447331": 379, "n01447658": 380, "n01447946": 381, "n01448291": 382, "n01448594": 383, "n01448951": 384, "n01449374": 385, "n01449712": 386, "n01449980": 387, "n01450661": 388, "n01450950": 389, "n01451115": 390, "n01451295": 391, "n01451426": 392, "n01451863": 393, "n01452345": 394, "n01453087": 395, "n01453475": 396, "n01453742": 397, "n01454545": 398, "n01454856": 399, "n01455317": 400, "n01455461": 401, "n01455778": 402, "n01456137": 403, "n01456454": 404, "n01456756": 405, "n01457082": 406, "n01457407": 407, "n01457852": 408, "n01458746": 409, "n01458842": 410, "n01459791": 411, "n01460303": 412, "n01461315": 413, "n01461646": 414, "n01462042": 415, "n01462544": 416, "n01462803": 417, "n01464844": 418, "n01466257": 419, "n01467336": 420, "n01467804": 421, "n01468238": 422, "n01468712": 423, "n01469103": 424, "n01469723": 425, "n01470145": 426, "n01470479": 427, "n01470733": 428, "n01470895": 429, "n01471682": 430, "n01472303": 431, "n01472502": 432, "n01473806": 433, "n01474283": 434, "n01474864": 435, "n01475232": 436, "n01475940": 437, "n01476418": 438, "n01477080": 439, "n01477525": 440, "n01477875": 441, "n01478511": 442, "n01478969": 443, "n01479213": 444, "n01479820": 445, "n01480106": 446, "n01480516": 447, "n01480880": 448, "n01481331": 449, "n01481498": 450, "n01482071": 451, "n01482330": 452, "n01483021": 453, "n01483522": 454, "n01483830": 455, "n01484097": 456, "n01484285": 457, "n01484447": 458, "n01484562": 459, "n01484850": 460, "n01485479": 461, "n01486010": 462, "n01486540": 463, "n01486838": 464, "n01487506": 465, "n01488038": 466, "n01488918": 467, "n01489501": 468, "n01489709": 469, "n01489920": 470, "n01490112": 471, "n01490360": 472, "n01490670": 473, "n01491006": 474, "n01491361": 475, "n01491661": 476, "n01491874": 477, "n01492357": 478, "n01492569": 479, "n01492708": 480, "n01492860": 481, "n01493146": 482, "n01493541": 483, "n01493829": 484, "n01494041": 485, "n01494475": 486, "n01494757": 487, "n01494882": 488, "n01495006": 489, "n01495493": 490, "n01495701": 491, "n01496331": 492, "n01497118": 493, "n01497413": 494, "n01497738": 495, "n01498041": 496, "n01498406": 497, "n01498699": 498, "n01498989": 499, "n01499396": 500, "n01499732": 501, "n01500091": 502, "n01500476": 503, "n01500854": 504, "n01501160": 505, "n01501641": 506, "n01501777": 507, "n01501948": 508, "n01502101": 509, "n01503061": 510, "n01503976": 511, "n01504179": 512, "n01504344": 513, "n01514668": 514, "n01514752": 515, "n01514859": 516, "n01514926": 517, "n01515078": 518, "n01515217": 519, "n01515303": 520, "n01516212": 521, "n01517389": 522, "n01517565": 523, "n01517966": 524, "n01518878": 525, "n01519563": 526, "n01519873": 527, "n01520576": 528, "n01521399": 529, "n01521756": 530, "n01522450": 531, "n01523105": 532, "n01524359": 533, "n01524761": 534, "n01525720": 535, "n01526521": 536, "n01526766": 537, "n01527194": 538, "n01527347": 539, "n01527617": 540, "n01527917": 541, "n01528396": 542, "n01528654": 543, "n01528845": 544, "n01529672": 545, "n01530439": 546, "n01530575": 547, "n01531178": 548, "n01531344": 549, "n01531512": 550, "n01531639": 551, "n01531811": 552, "n01531971": 553, "n01532325": 554, "n01532511": 555, "n01532829": 556, "n01533000": 557, "n01533339": 558, "n01533481": 559, "n01533651": 560, "n01533893": 561, "n01534155": 562, "n01534433": 563, "n01534582": 564, "n01534762": 565, "n01535140": 566, "n01535469": 567, "n01535690": 568, "n01536035": 569, "n01536186": 570, "n01536334": 571, "n01536644": 572, "n01536780": 573, "n01537134": 574, "n01537544": 575, "n01537895": 576, "n01538059": 577, "n01538200": 578, "n01538362": 579, "n01538630": 580, "n01538955": 581, "n01539272": 582, "n01539573": 583, "n01539925": 584, "n01540090": 585, "n01540233": 586, "n01540566": 587, "n01540832": 588, "n01541102": 589, "n01541386": 590, "n01541760": 591, "n01541922": 592, "n01542168": 593, "n01542433": 594, "n01542786": 595, "n01543175": 596, "n01543383": 597, "n01543632": 598, "n01543936": 599, "n01544208": 600, "n01544389": 601, "n01544704": 602, "n01545574": 603, "n01546039": 604, "n01546506": 605, "n01546921": 606, "n01547832": 607, "n01548301": 608, "n01548492": 609, "n01548694": 610, "n01548865": 611, "n01549053": 612, "n01549430": 613, "n01549641": 614, "n01549886": 615, "n01550172": 616, "n01550761": 617, "n01551080": 618, "n01551300": 619, "n01551711": 620, "n01552034": 621, "n01552333": 622, "n01552813": 623, "n01553142": 624, "n01553527": 625, "n01553762": 626, "n01554017": 627, "n01554448": 628, "n01555004": 629, "n01555305": 630, "n01555809": 631, "n01556182": 632, "n01556514": 633, "n01557185": 634, "n01557962": 635, "n01558149": 636, "n01558307": 637, "n01558461": 638, "n01558594": 639, "n01558765": 640, "n01558993": 641, "n01559160": 642, "n01559477": 643, "n01559639": 644, "n01559804": 645, "n01560105": 646, "n01560280": 647, "n01560419": 648, "n01560636": 649, "n01560793": 650, "n01560935": 651, "n01561181": 652, "n01561452": 653, "n01561732": 654, "n01562014": 655, "n01562265": 656, "n01562451": 657, "n01563128": 658, "n01563449": 659, "n01563746": 660, "n01563945": 661, "n01564101": 662, "n01564217": 663, "n01564394": 664, "n01564773": 665, "n01564914": 666, "n01565078": 667, "n01565345": 668, "n01565599": 669, "n01565930": 670, "n01566207": 671, "n01566645": 672, "n01567133": 673, "n01567678": 674, "n01567879": 675, "n01568132": 676, "n01568294": 677, "n01568720": 678, "n01568892": 679, "n01569060": 680, "n01569262": 681, "n01569423": 682, "n01569566": 683, "n01569836": 684, "n01569971": 685, "n01570267": 686, "n01570421": 687, "n01570676": 688, "n01570839": 689, "n01571410": 690, "n01571904": 691, "n01572328": 692, "n01572489": 693, "n01572654": 694, "n01572782": 695, "n01573074": 696, "n01573240": 697, "n01573360": 698, "n01573627": 699, "n01573898": 700, "n01574045": 701, "n01574390": 702, "n01574560": 703, "n01574801": 704, "n01575117": 705, "n01575401": 706, "n01575745": 707, "n01576076": 708, "n01576358": 709, "n01576695": 710, "n01577035": 711, "n01577458": 712, "n01577659": 713, "n01577941": 714, "n01578180": 715, "n01578575": 716, "n01579028": 717, "n01579149": 718, "n01579260": 719, "n01579410": 720, "n01579578": 721, "n01579729": 722, "n01580077": 723, "n01580379": 724, "n01580490": 725, "n01580772": 726, "n01580870": 727, "n01581166": 728, "n01581434": 729, "n01581730": 730, "n01581874": 731, "n01581984": 732, "n01582220": 733, "n01582398": 734, "n01582498": 735, "n01582856": 736, "n01583209": 737, "n01583495": 738, "n01583828": 739, "n01584225": 740, "n01584695": 741, "n01584853": 742, "n01585121": 743, "n01585287": 744, "n01585422": 745, "n01585715": 746, "n01586020": 747, "n01586374": 748, "n01586941": 749, "n01587278": 750, "n01587526": 751, "n01587834": 752, "n01588002": 753, "n01588431": 754, "n01588725": 755, "n01588996": 756, "n01589286": 757, "n01589718": 758, "n01589893": 759, "n01590220": 760, "n01591005": 761, "n01591123": 762, "n01591301": 763, "n01591697": 764, "n01592084": 765, "n01592257": 766, "n01592387": 767, "n01592540": 768, "n01592694": 769, "n01593028": 770, "n01593282": 771, "n01593553": 772, "n01594004": 773, "n01594372": 774, "n01594787": 775, "n01594968": 776, "n01595168": 777, "n01595450": 778, "n01595624": 779, "n01595974": 780, "n01596273": 781, "n01596608": 782, "n01597022": 783, "n01597336": 784, "n01597737": 785, "n01597906": 786, "n01598074": 787, "n01598271": 788, "n01598588": 789, "n01598988": 790, "n01599159": 791, "n01599269": 792, "n01599388": 793, "n01599556": 794, "n01599741": 795, "n01600085": 796, "n01600341": 797, "n01600657": 798, "n01601068": 799, "n01601410": 800, "n01601694": 801, "n01602080": 802, "n01602209": 803, "n01602630": 804, "n01602832": 805, "n01603000": 806, "n01603152": 807, "n01603600": 808, "n01603812": 809, "n01603953": 810, "n01604330": 811, "n01604968": 812, "n01605630": 813, "n01606097": 814, "n01606177": 815, "n01606522": 816, "n01606672": 817, "n01606809": 818, "n01606978": 819, "n01607309": 820, "n01607429": 821, "n01607600": 822, "n01607812": 823, "n01607962": 824, "n01608265": 825, "n01608432": 826, "n01608814": 827, "n01609062": 828, "n01609391": 829, "n01609751": 830, "n01609956": 831, "n01610100": 832, "n01610226": 833, "n01610552": 834, "n01610955": 835, "n01611472": 836, "n01611674": 837, "n01611800": 838, "n01611969": 839, "n01612122": 840, "n01612275": 841, "n01612476": 842, "n01612628": 843, "n01612955": 844, "n01613177": 845, "n01613294": 846, "n01613615": 847, "n01613807": 848, "n01614038": 849, "n01614343": 850, "n01614556": 851, "n01614925": 852, "n01615121": 853, "n01615303": 854, "n01615458": 855, "n01615703": 856, "n01616086": 857, "n01616318": 858, "n01616551": 859, "n01616764": 860, "n01617095": 861, "n01617443": 862, "n01617766": 863, "n01618082": 864, "n01618503": 865, "n01618922": 866, "n01619310": 867, "n01619536": 868, "n01619835": 869, "n01620135": 870, "n01620414": 871, "n01620735": 872, "n01621127": 873, "n01621635": 874, "n01622120": 875, "n01622352": 876, "n01622483": 877, "n01622779": 878, "n01622959": 879, "n01623110": 880, "n01623425": 881, "n01623615": 882, "n01623706": 883, "n01623880": 884, "n01624115": 885, "n01624212": 886, "n01624305": 887, "n01624537": 888, "n01624833": 889, "n01625121": 890, "n01625562": 891, "n01627424": 892, "n01628331": 893, "n01628770": 894, "n01629276": 895, "n01629819": 896, "n01629962": 897, "n01630148": 898, "n01630284": 899, "n01630670": 900, "n01630901": 901, "n01631175": 902, "n01631354": 903, "n01631512": 904, "n01631663": 905, "n01632047": 906, "n01632308": 907, "n01632458": 908, "n01632601": 909, "n01632777": 910, "n01632952": 911, "n01633406": 912, "n01633781": 913, "n01634227": 914, "n01634522": 915, "n01635027": 916, "n01635176": 917, "n01635480": 918, "n01636127": 919, "n01636352": 920, "n01636510": 921, "n01636829": 922, "n01637112": 923, "n01637338": 924, "n01637615": 925, "n01637932": 926, "n01638194": 927, "n01638329": 928, "n01638722": 929, "n01639187": 930, "n01639765": 931, "n01640846": 932, "n01641206": 933, "n01641391": 934, "n01641577": 935, "n01641739": 936, "n01641930": 937, "n01642097": 938, "n01642257": 939, "n01642391": 940, "n01642539": 941, "n01642943": 942, "n01643255": 943, "n01643507": 944, "n01643896": 945, "n01644373": 946, "n01644900": 947, "n01645466": 948, "n01645776": 949, "n01646292": 950, "n01646388": 951, "n01646555": 952, "n01646648": 953, "n01646802": 954, "n01646902": 955, "n01647033": 956, "n01647180": 957, "n01647303": 958, "n01647466": 959, "n01647640": 960, "n01648139": 961, "n01648356": 962, "n01648620": 963, "n01649170": 964, "n01649412": 965, "n01649556": 966, "n01649726": 967, "n01650167": 968, "n01650690": 969, "n01650901": 970, "n01651059": 971, "n01651285": 972, "n01651487": 973, "n01651641": 974, "n01651778": 975, "n01652026": 976, "n01652297": 977, "n01653026": 978, "n01653223": 979, "n01653509": 980, "n01653773": 981, "n01654083": 982, "n01654637": 983, "n01654863": 984, "n01655344": 985, "n01661091": 986, "n01661592": 987, "n01661818": 988, "n01662060": 989, "n01662622": 990, "n01662784": 991, "n01663401": 992, "n01663782": 993, "n01664065": 994, "n01664369": 995, "n01664492": 996, "n01664674": 997, "n01664990": 998, "n01665541": 999, "n01665932": 1000, "n01666228": 1001, "n01666585": 1002, "n01667114": 1003, "n01667432": 1004, "n01667778": 1005, "n01668091": 1006, "n01668436": 1007, "n01668665": 1008, "n01668892": 1009, "n01669191": 1010, "n01669372": 1011, "n01669654": 1012, "n01670092": 1013, "n01670535": 1014, "n01670802": 1015, "n01671125": 1016, "n01671479": 1017, "n01671705": 1018, "n01672032": 1019, "n01672432": 1020, "n01672611": 1021, "n01673282": 1022, "n01674216": 1023, "n01674464": 1024, "n01674990": 1025, "n01675352": 1026, "n01675722": 1027, "n01676755": 1028, "n01677366": 1029, "n01677747": 1030, "n01678043": 1031, "n01678343": 1032, "n01678657": 1033, "n01679005": 1034, "n01679307": 1035, "n01679626": 1036, "n01679962": 1037, "n01680264": 1038, "n01680478": 1039, "n01680655": 1040, "n01680813": 1041, "n01680983": 1042, "n01681328": 1043, "n01681653": 1044, "n01681940": 1045, "n01682172": 1046, "n01682435": 1047, "n01682714": 1048, "n01683201": 1049, "n01683558": 1050, "n01684133": 1051, "n01684578": 1052, "n01684741": 1053, "n01685439": 1054, "n01685808": 1055, "n01686044": 1056, "n01686220": 1057, "n01686403": 1058, "n01686609": 1059, "n01686808": 1060, "n01687128": 1061, "n01687290": 1062, "n01687665": 1063, "n01687978": 1064, "n01688243": 1065, "n01688961": 1066, "n01689081": 1067, "n01689411": 1068, "n01689811": 1069, "n01690149": 1070, "n01690466": 1071, "n01691217": 1072, "n01691652": 1073, "n01691951": 1074, "n01692333": 1075, "n01692523": 1076, "n01692864": 1077, "n01693175": 1078, "n01693334": 1079, "n01693783": 1080, "n01694178": 1081, "n01694311": 1082, "n01694709": 1083, "n01694955": 1084, "n01695060": 1085, "n01696633": 1086, "n01697178": 1087, "n01697457": 1088, "n01697611": 1089, "n01697749": 1090, "n01697978": 1091, "n01698434": 1092, "n01698640": 1093, "n01698782": 1094, "n01699040": 1095, "n01699254": 1096, "n01699675": 1097, "n01701551": 1098, "n01701859": 1099, "n01702256": 1100, "n01702479": 1101, "n01703011": 1102, "n01703161": 1103, "n01703569": 1104, "n01704103": 1105, "n01704323": 1106, "n01704626": 1107, "n01705010": 1108, "n01705591": 1109, "n01705934": 1110, "n01707294": 1111, "n01708106": 1112, "n01708998": 1113, "n01709484": 1114, "n01709876": 1115, "n01710177": 1116, "n01711160": 1117, "n01712008": 1118, "n01712752": 1119, "n01713170": 1120, "n01713764": 1121, "n01714231": 1122, "n01715888": 1123, "n01717016": 1124, "n01717229": 1125, "n01717467": 1126, "n01718096": 1127, "n01718414": 1128, "n01719403": 1129, "n01721174": 1130, "n01721898": 1131, "n01722670": 1132, "n01722998": 1133, "n01723579": 1134, "n01724231": 1135, "n01724840": 1136, "n01725086": 1137, "n01725713": 1138, "n01726203": 1139, "n01726692": 1140, "n01727646": 1141, "n01728266": 1142, "n01728572": 1143, "n01728920": 1144, "n01729322": 1145, "n01729672": 1146, "n01729977": 1147, "n01730185": 1148, "n01730307": 1149, "n01730563": 1150, "n01730812": 1151, "n01730960": 1152, "n01731137": 1153, "n01731277": 1154, "n01731545": 1155, "n01731764": 1156, "n01731941": 1157, "n01732093": 1158, "n01732244": 1159, "n01732614": 1160, "n01732789": 1161, "n01732989": 1162, "n01733214": 1163, "n01733466": 1164, "n01733757": 1165, "n01733957": 1166, "n01734104": 1167, "n01734418": 1168, "n01734637": 1169, "n01734808": 1170, "n01735189": 1171, "n01735439": 1172, "n01735577": 1173, "n01735728": 1174, "n01736032": 1175, "n01736375": 1176, "n01736796": 1177, "n01737021": 1178, "n01737472": 1179, "n01737728": 1180, "n01737875": 1181, "n01738065": 1182, "n01738306": 1183, "n01738601": 1184, "n01738731": 1185, "n01739094": 1186, "n01739381": 1187, "n01739647": 1188, "n01739871": 1189, "n01740131": 1190, "n01740551": 1191, "n01740885": 1192, "n01741232": 1193, "n01741442": 1194, "n01741562": 1195, "n01741943": 1196, "n01742172": 1197, "n01742447": 1198, "n01742821": 1199, "n01743086": 1200, "n01743605": 1201, "n01743936": 1202, "n01744100": 1203, "n01744270": 1204, "n01744401": 1205, "n01744555": 1206, "n01745125": 1207, "n01745484": 1208, "n01745902": 1209, "n01746191": 1210, "n01746359": 1211, "n01746952": 1212, "n01747285": 1213, "n01747589": 1214, "n01747885": 1215, "n01748264": 1216, "n01748389": 1217, "n01748686": 1218, "n01748906": 1219, "n01749244": 1220, "n01749582": 1221, "n01749742": 1222, "n01749939": 1223, "n01750167": 1224, "n01750437": 1225, "n01750743": 1226, "n01751036": 1227, "n01751215": 1228, "n01751472": 1229, "n01751748": 1230, "n01752165": 1231, "n01752585": 1232, "n01752736": 1233, "n01753032": 1234, "n01753180": 1235, "n01753488": 1236, "n01753959": 1237, "n01754370": 1238, "n01754533": 1239, "n01754876": 1240, "n01755581": 1241, "n01755740": 1242, "n01755952": 1243, "n01756089": 1244, "n01756291": 1245, "n01756508": 1246, "n01756733": 1247, "n01756916": 1248, "n01757115": 1249, "n01757343": 1250, "n01757677": 1251, "n01757901": 1252, "n01758141": 1253, "n01758757": 1254, "n01758895": 1255, "n01767661": 1256, "n01768244": 1257, "n01769347": 1258, "n01770081": 1259, "n01770393": 1260, "n01770795": 1261, "n01771100": 1262, "n01771417": 1263, "n01771766": 1264, "n01772222": 1265, "n01772664": 1266, "n01773157": 1267, "n01773549": 1268, "n01773797": 1269, "n01774097": 1270, "n01774384": 1271, "n01774750": 1272, "n01775062": 1273, "n01775370": 1274, "n01775730": 1275, "n01776192": 1276, "n01776313": 1277, "n01776705": 1278, "n01777304": 1279, "n01777467": 1280, "n01777649": 1281, "n01777909": 1282, "n01778217": 1283, "n01778487": 1284, "n01778621": 1285, "n01778801": 1286, "n01779148": 1287, "n01779463": 1288, "n01779629": 1289, "n01779939": 1290, "n01780142": 1291, "n01780426": 1292, "n01780696": 1293, "n01781071": 1294, "n01781570": 1295, "n01781698": 1296, "n01781875": 1297, "n01782209": 1298, "n01782516": 1299, "n01783017": 1300, "n01783706": 1301, "n01784293": 1302, "n01784675": 1303, "n01785667": 1304, "n01786646": 1305, "n01787006": 1306, "n01787191": 1307, "n01787835": 1308, "n01788291": 1309, "n01788579": 1310, "n01788864": 1311, "n01789386": 1312, "n01789740": 1313, "n01790171": 1314, "n01790304": 1315, "n01790398": 1316, "n01790557": 1317, "n01790711": 1318, "n01790812": 1319, "n01791107": 1320, "n01791314": 1321, "n01791388": 1322, "n01791463": 1323, "n01791625": 1324, "n01791954": 1325, "n01792042": 1326, "n01792158": 1327, "n01792429": 1328, "n01792530": 1329, "n01792640": 1330, "n01792808": 1331, "n01792955": 1332, "n01793085": 1333, "n01793159": 1334, "n01793249": 1335, "n01793340": 1336, "n01793435": 1337, "n01793565": 1338, "n01793715": 1339, "n01794158": 1340, "n01794344": 1341, "n01794651": 1342, "n01795088": 1343, "n01795545": 1344, "n01795735": 1345, "n01795900": 1346, "n01796019": 1347, "n01796105": 1348, "n01796340": 1349, "n01796519": 1350, "n01796729": 1351, "n01797020": 1352, "n01797307": 1353, "n01797601": 1354, "n01797886": 1355, "n01798168": 1356, "n01798484": 1357, "n01798706": 1358, "n01798839": 1359, "n01798979": 1360, "n01799302": 1361, "n01799679": 1362, "n01800195": 1363, "n01800424": 1364, "n01800633": 1365, "n01801088": 1366, "n01801479": 1367, "n01801672": 1368, "n01801876": 1369, "n01802159": 1370, "n01802721": 1371, "n01803078": 1372, "n01803362": 1373, "n01803641": 1374, "n01803893": 1375, "n01804163": 1376, "n01804478": 1377, "n01804653": 1378, "n01804921": 1379, "n01805070": 1380, "n01805321": 1381, "n01805801": 1382, "n01806061": 1383, "n01806143": 1384, "n01806297": 1385, "n01806364": 1386, "n01806467": 1387, "n01806567": 1388, "n01806847": 1389, "n01807105": 1390, "n01807496": 1391, "n01807828": 1392, "n01808140": 1393, "n01808291": 1394, "n01808596": 1395, "n01809106": 1396, "n01809371": 1397, "n01809752": 1398, "n01810268": 1399, "n01810700": 1400, "n01811243": 1401, "n01811909": 1402, "n01812187": 1403, "n01812337": 1404, "n01812662": 1405, "n01812866": 1406, "n01813088": 1407, "n01813385": 1408, "n01813532": 1409, "n01813658": 1410, "n01813948": 1411, "n01814217": 1412, "n01814370": 1413, "n01814549": 1414, "n01814620": 1415, "n01814755": 1416, "n01814921": 1417, "n01815036": 1418, "n01815270": 1419, "n01815601": 1420, "n01816017": 1421, "n01816140": 1422, "n01816474": 1423, "n01816887": 1424, "n01817263": 1425, "n01817346": 1426, "n01817953": 1427, "n01818299": 1428, "n01818515": 1429, "n01818832": 1430, "n01819115": 1431, "n01819313": 1432, "n01819465": 1433, "n01819734": 1434, "n01820052": 1435, "n01820348": 1436, "n01820546": 1437, "n01820801": 1438, "n01821076": 1439, "n01821203": 1440, "n01821554": 1441, "n01821869": 1442, "n01822300": 1443, "n01822602": 1444, "n01823013": 1445, "n01823414": 1446, "n01823740": 1447, "n01824035": 1448, "n01824344": 1449, "n01824575": 1450, "n01824749": 1451, "n01825278": 1452, "n01825930": 1453, "n01826364": 1454, "n01826680": 1455, "n01826844": 1456, "n01827403": 1457, "n01827793": 1458, "n01828096": 1459, "n01828556": 1460, "n01828970": 1461, "n01829413": 1462, "n01829869": 1463, "n01830042": 1464, "n01830479": 1465, "n01830915": 1466, "n01831360": 1467, "n01831712": 1468, "n01832167": 1469, "n01832493": 1470, "n01832813": 1471, "n01833112": 1472, "n01833415": 1473, "n01833805": 1474, "n01834177": 1475, "n01834540": 1476, "n01835276": 1477, "n01835769": 1478, "n01835918": 1479, "n01836087": 1480, "n01836673": 1481, "n01837072": 1482, "n01837526": 1483, "n01838038": 1484, "n01838598": 1485, "n01839086": 1486, "n01839330": 1487, "n01839598": 1488, "n01839750": 1489, "n01839949": 1490, "n01840120": 1491, "n01840412": 1492, "n01840775": 1493, "n01841102": 1494, "n01841288": 1495, "n01841441": 1496, "n01841679": 1497, "n01841943": 1498, "n01842235": 1499, "n01842504": 1500, "n01842788": 1501, "n01843065": 1502, "n01843383": 1503, "n01843719": 1504, "n01844231": 1505, "n01844551": 1506, "n01844746": 1507, "n01844917": 1508, "n01845132": 1509, "n01845477": 1510, "n01846331": 1511, "n01847000": 1512, "n01847089": 1513, "n01847170": 1514, "n01847253": 1515, "n01847407": 1516, "n01847806": 1517, "n01847978": 1518, "n01848123": 1519, "n01848323": 1520, "n01848453": 1521, "n01848555": 1522, "n01848648": 1523, "n01848840": 1524, "n01848976": 1525, "n01849157": 1526, "n01849466": 1527, "n01849676": 1528, "n01849863": 1529, "n01850192": 1530, "n01850373": 1531, "n01850553": 1532, "n01850873": 1533, "n01851038": 1534, "n01851207": 1535, "n01851375": 1536, "n01851573": 1537, "n01851731": 1538, "n01851895": 1539, "n01852142": 1540, "n01852329": 1541, "n01852400": 1542, "n01852671": 1543, "n01852861": 1544, "n01853195": 1545, "n01853498": 1546, "n01853666": 1547, "n01853870": 1548, "n01854415": 1549, "n01854700": 1550, "n01854838": 1551, "n01855032": 1552, "n01855188": 1553, "n01855476": 1554, "n01855672": 1555, "n01856072": 1556, "n01856155": 1557, "n01856380": 1558, "n01856553": 1559, "n01856890": 1560, "n01857079": 1561, "n01857325": 1562, "n01857512": 1563, "n01857632": 1564, "n01857851": 1565, "n01858281": 1566, "n01858441": 1567, "n01858780": 1568, "n01858845": 1569, "n01858906": 1570, "n01859190": 1571, "n01859325": 1572, "n01859496": 1573, "n01859689": 1574, "n01859852": 1575, "n01860002": 1576, "n01860187": 1577, "n01860497": 1578, "n01860864": 1579, "n01861148": 1580, "n01861330": 1581, "n01861778": 1582, "n01862399": 1583, "n01871265": 1584, "n01871543": 1585, "n01871875": 1586, "n01872401": 1587, "n01872772": 1588, "n01873310": 1589, "n01874434": 1590, "n01874928": 1591, "n01875313": 1592, "n01875610": 1593, "n01876034": 1594, "n01876326": 1595, "n01876667": 1596, "n01877134": 1597, "n01877606": 1598, "n01877812": 1599, "n01878061": 1600, "n01878335": 1601, "n01878639": 1602, "n01878929": 1603, "n01879217": 1604, "n01879509": 1605, "n01879837": 1606, "n01880152": 1607, "n01880473": 1608, "n01880716": 1609, "n01880813": 1610, "n01881171": 1611, "n01881564": 1612, "n01881857": 1613, "n01882125": 1614, "n01882714": 1615, "n01883070": 1616, "n01883513": 1617, "n01883920": 1618, "n01884104": 1619, "n01884203": 1620, "n01884476": 1621, "n01884834": 1622, "n01885158": 1623, "n01885498": 1624, "n01886045": 1625, "n01886756": 1626, "n01887474": 1627, "n01887623": 1628, "n01887787": 1629, "n01887896": 1630, "n01888045": 1631, "n01888181": 1632, "n01888264": 1633, "n01888411": 1634, "n01889074": 1635, "n01889520": 1636, "n01889849": 1637, "n01890144": 1638, "n01890564": 1639, "n01890860": 1640, "n01891013": 1641, "n01891274": 1642, "n01891633": 1643, "n01892030": 1644, "n01892145": 1645, "n01892385": 1646, "n01892551": 1647, "n01892744": 1648, "n01893021": 1649, "n01893164": 1650, "n01893399": 1651, "n01893825": 1652, "n01894207": 1653, "n01894522": 1654, "n01894956": 1655, "n01896844": 1656, "n01897257": 1657, "n01897426": 1658, "n01897536": 1659, "n01897667": 1660, "n01898593": 1661, "n01899894": 1662, "n01900150": 1663, "n01903234": 1664, "n01903346": 1665, "n01903498": 1666, "n01904029": 1667, "n01904806": 1668, "n01904886": 1669, "n01905321": 1670, "n01905661": 1671, "n01906749": 1672, "n01907287": 1673, "n01907738": 1674, "n01908042": 1675, "n01908958": 1676, "n01909422": 1677, "n01909788": 1678, "n01909906": 1679, "n01910252": 1680, "n01910747": 1681, "n01911063": 1682, "n01911403": 1683, "n01911839": 1684, "n01912152": 1685, "n01912454": 1686, "n01912809": 1687, "n01913166": 1688, "n01913346": 1689, "n01913440": 1690, "n01914163": 1691, "n01914609": 1692, "n01914830": 1693, "n01915700": 1694, "n01915811": 1695, "n01916187": 1696, "n01916388": 1697, "n01916481": 1698, "n01916588": 1699, "n01916925": 1700, "n01917289": 1701, "n01917611": 1702, "n01917882": 1703, "n01918744": 1704, "n01919385": 1705, "n01920051": 1706, "n01920438": 1707, "n01921059": 1708, "n01922303": 1709, "n01922717": 1710, "n01922948": 1711, "n01923025": 1712, "n01923404": 1713, "n01923890": 1714, "n01924800": 1715, "n01924916": 1716, "n01925270": 1717, "n01925695": 1718, "n01925916": 1719, "n01926379": 1720, "n01926689": 1721, "n01927159": 1722, "n01927456": 1723, "n01927928": 1724, "n01928215": 1725, "n01928517": 1726, "n01928865": 1727, "n01929186": 1728, "n01930112": 1729, "n01930852": 1730, "n01931140": 1731, "n01931520": 1732, "n01931714": 1733, "n01932151": 1734, "n01932936": 1735, "n01933151": 1736, "n01933478": 1737, "n01933988": 1738, "n01934440": 1739, "n01934844": 1740, "n01935176": 1741, "n01935395": 1742, "n01936391": 1743, "n01936671": 1744, "n01936858": 1745, "n01937579": 1746, "n01937909": 1747, "n01938454": 1748, "n01938735": 1749, "n01940736": 1750, "n01941223": 1751, "n01941340": 1752, "n01942177": 1753, "n01942869": 1754, "n01943087": 1755, "n01943541": 1756, "n01943899": 1757, "n01944118": 1758, "n01944390": 1759, "n01944812": 1760, "n01944955": 1761, "n01945143": 1762, "n01945340": 1763, "n01945685": 1764, "n01945845": 1765, "n01946277": 1766, "n01946630": 1767, "n01946827": 1768, "n01947139": 1769, "n01947396": 1770, "n01947997": 1771, "n01948446": 1772, "n01948573": 1773, "n01949085": 1774, "n01949499": 1775, "n01949973": 1776, "n01950731": 1777, "n01951274": 1778, "n01951613": 1779, "n01952029": 1780, "n01952712": 1781, "n01953361": 1782, "n01953594": 1783, "n01953762": 1784, "n01954516": 1785, "n01955084": 1786, "n01955933": 1787, "n01956344": 1788, "n01956481": 1789, "n01956764": 1790, "n01957335": 1791, "n01958038": 1792, "n01958346": 1793, "n01958435": 1794, "n01958531": 1795, "n01959029": 1796, "n01959492": 1797, "n01959985": 1798, "n01960177": 1799, "n01960459": 1800, "n01961234": 1801, "n01961600": 1802, "n01961985": 1803, "n01962506": 1804, "n01962788": 1805, "n01963317": 1806, "n01963479": 1807, "n01963571": 1808, "n01964049": 1809, "n01964271": 1810, "n01964441": 1811, "n01964957": 1812, "n01965252": 1813, "n01965529": 1814, "n01965889": 1815, "n01966377": 1816, "n01966586": 1817, "n01967094": 1818, "n01967308": 1819, "n01967963": 1820, "n01968315": 1821, "n01968897": 1822, "n01969726": 1823, "n01970164": 1824, "n01970667": 1825, "n01971094": 1826, "n01971280": 1827, "n01971620": 1828, "n01971850": 1829, "n01972131": 1830, "n01972541": 1831, "n01973148": 1832, "n01974773": 1833, "n01975687": 1834, "n01976146": 1835, "n01976868": 1836, "n01976957": 1837, "n01977485": 1838, "n01978010": 1839, "n01978136": 1840, "n01978287": 1841, "n01978455": 1842, "n01978587": 1843, "n01978930": 1844, "n01979269": 1845, "n01979526": 1846, "n01979874": 1847, "n01980166": 1848, "n01980655": 1849, "n01981276": 1850, "n01981702": 1851, "n01982068": 1852, "n01982347": 1853, "n01982650": 1854, "n01983048": 1855, "n01983481": 1856, "n01983674": 1857, "n01983829": 1858, "n01984245": 1859, "n01984695": 1860, "n01985128": 1861, "n01985493": 1862, "n01985797": 1863, "n01986214": 1864, "n01986806": 1865, "n01987076": 1866, "n01987545": 1867, "n01987727": 1868, "n01988203": 1869, "n01988701": 1870, "n01988869": 1871, "n01989516": 1872, "n01989869": 1873, "n01990007": 1874, "n01990516": 1875, "n01990800": 1876, "n01991028": 1877, "n01991520": 1878, "n01992262": 1879, "n01992423": 1880, "n01992773": 1881, "n01993525": 1882, "n01993830": 1883, "n01994910": 1884, "n01995514": 1885, "n01995686": 1886, "n01996280": 1887, "n01996585": 1888, "n01997119": 1889, "n01997825": 1890, "n01998183": 1891, "n01998741": 1892, "n01999186": 1893, "n01999767": 1894, "n02000954": 1895, "n02002075": 1896, "n02002556": 1897, "n02002724": 1898, "n02003037": 1899, "n02003204": 1900, "n02003577": 1901, "n02003839": 1902, "n02004131": 1903, "n02004492": 1904, "n02004855": 1905, "n02005399": 1906, "n02005790": 1907, "n02006063": 1908, "n02006364": 1909, "n02006656": 1910, "n02006985": 1911, "n02007284": 1912, "n02007558": 1913, "n02008041": 1914, "n02008497": 1915, "n02008643": 1916, "n02008796": 1917, "n02009229": 1918, "n02009380": 1919, "n02009508": 1920, "n02009750": 1921, "n02009912": 1922, "n02010272": 1923, "n02010453": 1924, "n02010728": 1925, "n02011016": 1926, "n02011281": 1927, "n02011460": 1928, "n02011805": 1929, "n02011943": 1930, "n02012185": 1931, "n02012849": 1932, "n02013177": 1933, "n02013567": 1934, "n02013706": 1935, "n02014237": 1936, "n02014524": 1937, "n02014941": 1938, "n02015357": 1939, "n02015554": 1940, "n02015797": 1941, "n02016066": 1942, "n02016358": 1943, "n02016659": 1944, "n02016816": 1945, "n02016956": 1946, "n02017213": 1947, "n02017475": 1948, "n02017725": 1949, "n02018027": 1950, "n02018207": 1951, "n02018368": 1952, "n02018795": 1953, "n02019190": 1954, "n02019438": 1955, "n02019929": 1956, "n02020219": 1957, "n02020578": 1958, "n02021050": 1959, "n02021281": 1960, "n02021795": 1961, "n02022684": 1962, "n02023341": 1963, "n02023855": 1964, "n02023992": 1965, "n02024185": 1966, "n02024479": 1967, "n02024763": 1968, "n02025043": 1969, "n02025239": 1970, "n02025389": 1971, "n02026059": 1972, "n02026629": 1973, "n02026948": 1974, "n02027075": 1975, "n02027357": 1976, "n02027492": 1977, "n02027897": 1978, "n02028035": 1979, "n02028175": 1980, "n02028342": 1981, "n02028451": 1982, "n02028727": 1983, "n02028900": 1984, "n02029087": 1985, "n02029378": 1986, "n02029706": 1987, "n02030035": 1988, "n02030224": 1989, "n02030287": 1990, "n02030568": 1991, "n02030837": 1992, "n02030996": 1993, "n02031298": 1994, "n02031585": 1995, "n02031934": 1996, "n02032222": 1997, "n02032355": 1998, "n02032480": 1999, "n02032769": 2000, "n02033041": 2001, "n02033208": 2002, "n02033324": 2003, "n02033561": 2004, "n02033779": 2005, "n02033882": 2006, "n02034129": 2007, "n02034295": 2008, "n02034661": 2009, "n02034971": 2010, "n02035210": 2011, "n02035402": 2012, "n02035656": 2013, "n02036053": 2014, "n02036228": 2015, "n02036711": 2016, "n02037110": 2017, "n02037464": 2018, "n02037869": 2019, "n02038141": 2020, "n02038466": 2021, "n02038993": 2022, "n02039171": 2023, "n02039497": 2024, "n02039780": 2025, "n02040266": 2026, "n02040505": 2027, "n02041085": 2028, "n02041246": 2029, "n02041678": 2030, "n02041875": 2031, "n02042046": 2032, "n02042180": 2033, "n02042472": 2034, "n02042759": 2035, "n02043063": 2036, "n02043333": 2037, "n02043808": 2038, "n02044178": 2039, "n02044517": 2040, "n02044778": 2041, "n02044908": 2042, "n02045369": 2043, "n02045596": 2044, "n02045864": 2045, "n02046171": 2046, "n02046759": 2047, "n02046939": 2048, "n02047045": 2049, "n02047260": 2050, "n02047411": 2051, "n02047517": 2052, "n02047614": 2053, "n02047975": 2054, "n02048115": 2055, "n02048353": 2056, "n02048698": 2057, "n02049088": 2058, "n02049532": 2059, "n02050004": 2060, "n02050313": 2061, "n02050442": 2062, "n02050586": 2063, "n02050809": 2064, "n02051059": 2065, "n02051474": 2066, "n02051845": 2067, "n02052204": 2068, "n02052365": 2069, "n02052775": 2070, "n02053083": 2071, "n02053425": 2072, "n02053584": 2073, "n02054036": 2074, "n02054502": 2075, "n02054711": 2076, "n02055107": 2077, "n02055658": 2078, "n02055803": 2079, "n02056228": 2080, "n02056570": 2081, "n02056728": 2082, "n02057035": 2083, "n02057330": 2084, "n02057731": 2085, "n02057898": 2086, "n02058221": 2087, "n02058594": 2088, "n02058747": 2089, "n02059162": 2090, "n02059541": 2091, "n02059852": 2092, "n02060133": 2093, "n02060411": 2094, "n02060569": 2095, "n02060889": 2096, "n02061217": 2097, "n02061560": 2098, "n02061853": 2099, "n02062017": 2100, "n02062430": 2101, "n02062744": 2102, "n02063224": 2103, "n02063662": 2104, "n02064000": 2105, "n02064338": 2106, "n02064816": 2107, "n02065026": 2108, "n02065263": 2109, "n02065407": 2110, "n02065726": 2111, "n02066245": 2112, "n02066707": 2113, "n02067240": 2114, "n02067603": 2115, "n02067768": 2116, "n02068206": 2117, "n02068541": 2118, "n02068974": 2119, "n02069412": 2120, "n02069701": 2121, "n02069974": 2122, "n02070174": 2123, "n02070430": 2124, "n02070624": 2125, "n02070776": 2126, "n02071028": 2127, "n02071294": 2128, "n02071636": 2129, "n02072040": 2130, "n02072493": 2131, "n02072798": 2132, "n02073250": 2133, "n02073831": 2134, "n02074367": 2135, "n02074726": 2136, "n02075296": 2137, "n02075612": 2138, "n02075927": 2139, "n02076196": 2140, "n02076402": 2141, "n02076779": 2142, "n02077152": 2143, "n02077384": 2144, "n02077658": 2145, "n02077787": 2146, "n02077923": 2147, "n02078292": 2148, "n02078574": 2149, "n02078738": 2150, "n02079005": 2151, "n02079389": 2152, "n02079851": 2153, "n02080146": 2154, "n02080415": 2155, "n02080713": 2156, "n02081060": 2157, "n02081571": 2158, "n02081798": 2159, "n02081927": 2160, "n02082056": 2161, "n02082190": 2162, "n02082791": 2163, "n02083346": 2164, "n02083672": 2165, "n02083780": 2166, "n02084071": 2167, "n02084732": 2168, "n02084861": 2169, "n02085019": 2170, "n02085118": 2171, "n02085272": 2172, "n02085374": 2173, "n02085620": 2174, "n02085782": 2175, "n02085936": 2176, "n02086079": 2177, "n02086240": 2178, "n02086346": 2179, "n02086478": 2180, "n02086646": 2181, "n02086753": 2182, "n02086910": 2183, "n02087046": 2184, "n02087122": 2185, "n02087314": 2186, "n02087394": 2187, "n02087551": 2188, "n02088094": 2189, "n02088238": 2190, "n02088364": 2191, "n02088466": 2192, "n02088632": 2193, "n02088745": 2194, "n02088839": 2195, "n02088992": 2196, "n02089078": 2197, "n02089232": 2198, "n02089468": 2199, "n02089555": 2200, "n02089725": 2201, "n02089867": 2202, "n02089973": 2203, "n02090129": 2204, "n02090253": 2205, "n02090379": 2206, "n02090475": 2207, "n02090622": 2208, "n02090721": 2209, "n02090827": 2210, "n02091032": 2211, "n02091134": 2212, "n02091244": 2213, "n02091467": 2214, "n02091635": 2215, "n02091831": 2216, "n02092002": 2217, "n02092173": 2218, "n02092339": 2219, "n02092468": 2220, "n02093056": 2221, "n02093256": 2222, "n02093428": 2223, "n02093647": 2224, "n02093754": 2225, "n02093859": 2226, "n02093991": 2227, "n02094114": 2228, "n02094258": 2229, "n02094433": 2230, "n02094562": 2231, "n02094721": 2232, "n02094931": 2233, "n02095050": 2234, "n02095212": 2235, "n02095314": 2236, "n02095412": 2237, "n02095570": 2238, "n02095727": 2239, "n02095889": 2240, "n02096051": 2241, "n02096177": 2242, "n02096294": 2243, "n02096437": 2244, "n02096585": 2245, "n02096756": 2246, "n02097047": 2247, "n02097130": 2248, "n02097209": 2249, "n02097298": 2250, "n02097474": 2251, "n02097658": 2252, "n02097786": 2253, "n02097967": 2254, "n02098105": 2255, "n02098286": 2256, "n02098413": 2257, "n02098550": 2258, "n02098806": 2259, "n02098906": 2260, "n02099029": 2261, "n02099267": 2262, "n02099429": 2263, "n02099601": 2264, "n02099712": 2265, "n02099849": 2266, "n02099997": 2267, "n02100236": 2268, "n02100399": 2269, "n02100583": 2270, "n02100735": 2271, "n02100877": 2272, "n02101006": 2273, "n02101108": 2274, "n02101388": 2275, "n02101556": 2276, "n02101670": 2277, "n02101861": 2278, "n02102040": 2279, "n02102177": 2280, "n02102318": 2281, "n02102480": 2282, "n02102605": 2283, "n02102806": 2284, "n02102973": 2285, "n02103181": 2286, "n02103406": 2287, "n02103841": 2288, "n02104029": 2289, "n02104184": 2290, "n02104280": 2291, "n02104365": 2292, "n02104523": 2293, "n02104882": 2294, "n02105056": 2295, "n02105162": 2296, "n02105251": 2297, "n02105412": 2298, "n02105505": 2299, "n02105641": 2300, "n02105855": 2301, "n02106030": 2302, "n02106166": 2303, "n02106382": 2304, "n02106550": 2305, "n02106662": 2306, "n02106854": 2307, "n02106966": 2308, "n02107142": 2309, "n02107312": 2310, "n02107420": 2311, "n02107574": 2312, "n02107683": 2313, "n02107908": 2314, "n02108000": 2315, "n02108089": 2316, "n02108254": 2317, "n02108422": 2318, "n02108551": 2319, "n02108672": 2320, "n02108915": 2321, "n02109047": 2322, "n02109150": 2323, "n02109256": 2324, "n02109391": 2325, "n02109525": 2326, "n02109687": 2327, "n02109811": 2328, "n02109961": 2329, "n02110063": 2330, "n02110185": 2331, "n02110341": 2332, "n02110532": 2333, "n02110627": 2334, "n02110806": 2335, "n02110958": 2336, "n02111129": 2337, "n02111277": 2338, "n02111500": 2339, "n02111626": 2340, "n02111889": 2341, "n02112018": 2342, "n02112137": 2343, "n02112350": 2344, "n02112497": 2345, "n02112706": 2346, "n02112826": 2347, "n02113023": 2348, "n02113186": 2349, "n02113335": 2350, "n02113624": 2351, "n02113712": 2352, "n02113799": 2353, "n02113892": 2354, "n02113978": 2355, "n02114100": 2356, "n02114367": 2357, "n02114548": 2358, "n02114712": 2359, "n02114855": 2360, "n02115012": 2361, "n02115096": 2362, "n02115335": 2363, "n02115641": 2364, "n02115913": 2365, "n02116185": 2366, "n02116450": 2367, "n02116738": 2368, "n02117135": 2369, "n02117512": 2370, "n02117646": 2371, "n02117900": 2372, "n02118176": 2373, "n02118333": 2374, "n02118643": 2375, "n02118707": 2376, "n02119022": 2377, "n02119247": 2378, "n02119359": 2379, "n02119477": 2380, "n02119634": 2381, "n02119789": 2382, "n02120079": 2383, "n02120278": 2384, "n02120505": 2385, "n02120997": 2386, "n02121620": 2387, "n02121808": 2388, "n02122298": 2389, "n02122430": 2390, "n02122510": 2391, "n02122580": 2392, "n02122725": 2393, "n02122810": 2394, "n02122878": 2395, "n02122948": 2396, "n02123045": 2397, "n02123159": 2398, "n02123242": 2399, "n02123394": 2400, "n02123478": 2401, "n02123597": 2402, "n02123785": 2403, "n02123917": 2404, "n02124075": 2405, "n02124157": 2406, "n02124313": 2407, "n02124484": 2408, "n02124623": 2409, "n02125010": 2410, "n02125081": 2411, "n02125311": 2412, "n02125494": 2413, "n02125689": 2414, "n02125872": 2415, "n02126028": 2416, "n02126139": 2417, "n02126317": 2418, "n02126640": 2419, "n02126787": 2420, "n02127052": 2421, "n02127292": 2422, "n02127381": 2423, "n02127482": 2424, "n02127586": 2425, "n02127678": 2426, "n02127808": 2427, "n02128385": 2428, "n02128598": 2429, "n02128669": 2430, "n02128757": 2431, "n02128925": 2432, "n02129165": 2433, "n02129463": 2434, "n02129530": 2435, "n02129604": 2436, "n02129837": 2437, "n02129923": 2438, "n02129991": 2439, "n02130086": 2440, "n02130308": 2441, "n02130545": 2442, "n02130925": 2443, "n02131653": 2444, "n02132136": 2445, "n02132320": 2446, "n02132466": 2447, "n02132580": 2448, "n02132788": 2449, "n02133161": 2450, "n02133400": 2451, "n02133704": 2452, "n02134084": 2453, "n02134418": 2454, "n02134971": 2455, "n02135220": 2456, "n02135610": 2457, "n02135844": 2458, "n02136103": 2459, "n02136285": 2460, "n02136452": 2461, "n02136794": 2462, "n02137015": 2463, "n02137302": 2464, "n02137549": 2465, "n02137722": 2466, "n02137888": 2467, "n02138169": 2468, "n02138441": 2469, "n02138647": 2470, "n02138777": 2471, "n02139199": 2472, "n02139671": 2473, "n02140049": 2474, "n02140179": 2475, "n02140268": 2476, "n02140491": 2477, "n02140858": 2478, "n02141306": 2479, "n02141611": 2480, "n02141713": 2481, "n02142407": 2482, "n02142734": 2483, "n02142898": 2484, "n02143142": 2485, "n02143439": 2486, "n02143891": 2487, "n02144251": 2488, "n02144593": 2489, "n02144936": 2490, "n02145424": 2491, "n02145910": 2492, "n02146201": 2493, "n02146371": 2494, "n02146700": 2495, "n02146879": 2496, "n02147173": 2497, "n02147328": 2498, "n02147591": 2499, "n02147947": 2500, "n02148088": 2501, "n02148512": 2502, "n02148835": 2503, "n02148991": 2504, "n02149420": 2505, "n02149653": 2506, "n02149861": 2507, "n02150134": 2508, "n02150482": 2509, "n02150885": 2510, "n02151230": 2511, "n02152740": 2512, "n02152881": 2513, "n02152991": 2514, "n02153109": 2515, "n02153203": 2516, "n02153809": 2517, "n02156732": 2518, "n02156871": 2519, "n02157206": 2520, "n02157285": 2521, "n02159955": 2522, "n02160947": 2523, "n02161225": 2524, "n02161338": 2525, "n02161457": 2526, "n02161588": 2527, "n02162561": 2528, "n02163008": 2529, "n02163297": 2530, "n02164464": 2531, "n02165105": 2532, "n02165456": 2533, "n02165877": 2534, "n02166229": 2535, "n02166567": 2536, "n02166826": 2537, "n02167151": 2538, "n02167505": 2539, "n02167820": 2540, "n02167944": 2541, "n02168245": 2542, "n02168427": 2543, "n02168699": 2544, "n02169023": 2545, "n02169218": 2546, "n02169497": 2547, "n02169705": 2548, "n02169974": 2549, "n02170400": 2550, "n02170599": 2551, "n02170738": 2552, "n02170993": 2553, "n02171164": 2554, "n02171453": 2555, "n02171869": 2556, "n02172182": 2557, "n02172518": 2558, "n02172678": 2559, "n02172761": 2560, "n02172870": 2561, "n02173113": 2562, "n02173373": 2563, "n02173784": 2564, "n02174001": 2565, "n02174355": 2566, "n02174659": 2567, "n02175014": 2568, "n02175569": 2569, "n02175916": 2570, "n02176261": 2571, "n02176439": 2572, "n02176747": 2573, "n02176916": 2574, "n02177196": 2575, "n02177506": 2576, "n02177775": 2577, "n02177972": 2578, "n02178411": 2579, "n02178717": 2580, "n02179012": 2581, "n02179192": 2582, "n02179340": 2583, "n02179891": 2584, "n02180233": 2585, "n02180427": 2586, "n02180875": 2587, "n02181235": 2588, "n02181477": 2589, "n02181724": 2590, "n02182045": 2591, "n02182355": 2592, "n02182642": 2593, "n02182930": 2594, "n02183096": 2595, "n02183507": 2596, "n02183857": 2597, "n02184473": 2598, "n02184589": 2599, "n02184720": 2600, "n02185167": 2601, "n02185481": 2602, "n02186153": 2603, "n02186717": 2604, "n02187150": 2605, "n02187279": 2606, "n02187554": 2607, "n02187900": 2608, "n02188699": 2609, "n02189363": 2610, "n02189670": 2611, "n02190166": 2612, "n02190790": 2613, "n02191273": 2614, "n02191773": 2615, "n02191979": 2616, "n02192252": 2617, "n02192513": 2618, "n02192814": 2619, "n02193009": 2620, "n02193163": 2621, "n02194249": 2622, "n02194750": 2623, "n02195091": 2624, "n02195526": 2625, "n02195819": 2626, "n02196119": 2627, "n02196344": 2628, "n02196896": 2629, "n02197185": 2630, "n02197689": 2631, "n02197877": 2632, "n02198129": 2633, "n02198532": 2634, "n02198859": 2635, "n02199170": 2636, "n02199502": 2637, "n02200198": 2638, "n02200509": 2639, "n02200630": 2640, "n02200850": 2641, "n02201000": 2642, "n02201497": 2643, "n02201626": 2644, "n02202006": 2645, "n02202124": 2646, "n02202287": 2647, "n02202678": 2648, "n02203152": 2649, "n02203592": 2650, "n02203978": 2651, "n02204249": 2652, "n02204722": 2653, "n02204907": 2654, "n02205219": 2655, "n02205673": 2656, "n02206270": 2657, "n02206856": 2658, "n02207179": 2659, "n02207345": 2660, "n02207449": 2661, "n02207647": 2662, "n02207805": 2663, "n02208280": 2664, "n02208498": 2665, "n02208848": 2666, "n02208979": 2667, "n02209111": 2668, "n02209354": 2669, "n02209624": 2670, "n02209964": 2671, "n02210427": 2672, "n02210921": 2673, "n02211444": 2674, "n02211627": 2675, "n02211896": 2676, "n02212062": 2677, "n02212602": 2678, "n02212958": 2679, "n02213107": 2680, "n02213239": 2681, "n02213543": 2682, "n02213663": 2683, "n02213788": 2684, "n02214096": 2685, "n02214341": 2686, "n02214499": 2687, "n02214660": 2688, "n02214773": 2689, "n02215161": 2690, "n02215621": 2691, "n02215770": 2692, "n02216211": 2693, "n02216365": 2694, "n02216740": 2695, "n02217563": 2696, "n02217839": 2697, "n02218134": 2698, "n02218371": 2699, "n02218713": 2700, "n02219015": 2701, "n02219486": 2702, "n02220055": 2703, "n02220225": 2704, "n02220518": 2705, "n02220804": 2706, "n02221083": 2707, "n02221414": 2708, "n02221571": 2709, "n02221715": 2710, "n02221820": 2711, "n02222035": 2712, "n02222321": 2713, "n02222582": 2714, "n02223266": 2715, "n02223520": 2716, "n02224023": 2717, "n02224713": 2718, "n02225081": 2719, "n02225798": 2720, "n02226183": 2721, "n02226429": 2722, "n02226821": 2723, "n02226970": 2724, "n02227247": 2725, "n02227604": 2726, "n02227966": 2727, "n02228341": 2728, "n02228697": 2729, "n02229156": 2730, "n02229544": 2731, "n02229765": 2732, "n02230023": 2733, "n02230187": 2734, "n02230480": 2735, "n02230634": 2736, "n02231052": 2737, "n02231487": 2738, "n02231803": 2739, "n02232223": 2740, "n02233338": 2741, "n02233943": 2742, "n02234355": 2743, "n02234570": 2744, "n02234848": 2745, "n02235205": 2746, "n02236044": 2747, "n02236241": 2748, "n02236355": 2749, "n02236896": 2750, "n02237424": 2751, "n02237581": 2752, "n02237868": 2753, "n02238235": 2754, "n02238358": 2755, "n02238594": 2756, "n02238887": 2757, "n02239192": 2758, "n02239528": 2759, "n02239774": 2760, "n02240068": 2761, "n02240517": 2762, "n02241008": 2763, "n02241426": 2764, "n02241569": 2765, "n02241799": 2766, "n02242137": 2767, "n02242455": 2768, "n02243209": 2769, "n02243562": 2770, "n02243878": 2771, "n02244173": 2772, "n02244515": 2773, "n02244797": 2774, "n02245111": 2775, "n02245443": 2776, "n02246011": 2777, "n02246628": 2778, "n02246941": 2779, "n02247216": 2780, "n02247511": 2781, "n02247655": 2782, "n02248062": 2783, "n02248368": 2784, "n02248510": 2785, "n02248887": 2786, "n02249134": 2787, "n02249515": 2788, "n02249809": 2789, "n02250280": 2790, "n02250822": 2791, "n02251067": 2792, "n02251233": 2793, "n02251593": 2794, "n02251775": 2795, "n02252226": 2796, "n02252799": 2797, "n02252972": 2798, "n02253127": 2799, "n02253264": 2800, "n02253494": 2801, "n02253715": 2802, "n02253913": 2803, "n02254246": 2804, "n02254697": 2805, "n02254901": 2806, "n02255023": 2807, "n02255391": 2808, "n02256172": 2809, "n02256656": 2810, "n02257003": 2811, "n02257284": 2812, "n02257715": 2813, "n02257985": 2814, "n02258198": 2815, "n02258508": 2816, "n02258629": 2817, "n02259212": 2818, "n02259377": 2819, "n02259708": 2820, "n02259987": 2821, "n02260421": 2822, "n02260863": 2823, "n02261063": 2824, "n02261419": 2825, "n02261757": 2826, "n02262178": 2827, "n02262449": 2828, "n02262803": 2829, "n02263378": 2830, "n02264021": 2831, "n02264232": 2832, "n02264363": 2833, "n02264591": 2834, "n02264885": 2835, "n02265330": 2836, "n02266050": 2837, "n02266269": 2838, "n02266421": 2839, "n02266864": 2840, "n02267208": 2841, "n02267483": 2842, "n02268148": 2843, "n02268443": 2844, "n02268853": 2845, "n02269196": 2846, "n02269340": 2847, "n02269522": 2848, "n02269657": 2849, "n02270011": 2850, "n02270200": 2851, "n02270623": 2852, "n02270945": 2853, "n02271222": 2854, "n02271570": 2855, "n02271897": 2856, "n02272286": 2857, "n02272552": 2858, "n02272871": 2859, "n02273392": 2860, "n02274024": 2861, "n02274259": 2862, "n02274822": 2863, "n02275560": 2864, "n02275773": 2865, "n02276078": 2866, "n02276258": 2867, "n02276355": 2868, "n02276749": 2869, "n02276902": 2870, "n02277094": 2871, "n02277268": 2872, "n02277422": 2873, "n02277742": 2874, "n02278024": 2875, "n02278210": 2876, "n02278463": 2877, "n02278839": 2878, "n02278980": 2879, "n02279257": 2880, "n02279637": 2881, "n02279972": 2882, "n02280458": 2883, "n02280649": 2884, "n02281015": 2885, "n02281136": 2886, "n02281267": 2887, "n02281406": 2888, "n02281787": 2889, "n02282257": 2890, "n02282385": 2891, "n02282553": 2892, "n02282903": 2893, "n02283077": 2894, "n02283201": 2895, "n02283617": 2896, "n02283951": 2897, "n02284224": 2898, "n02284611": 2899, "n02284884": 2900, "n02285179": 2901, "n02285548": 2902, "n02285801": 2903, "n02286089": 2904, "n02286425": 2905, "n02286654": 2906, "n02287004": 2907, "n02287352": 2908, "n02287622": 2909, "n02287799": 2910, "n02287987": 2911, "n02288122": 2912, "n02288268": 2913, "n02288789": 2914, "n02289307": 2915, "n02289610": 2916, "n02289988": 2917, "n02290340": 2918, "n02290664": 2919, "n02290870": 2920, "n02291220": 2921, "n02291572": 2922, "n02291748": 2923, "n02292085": 2924, "n02292401": 2925, "n02292692": 2926, "n02293352": 2927, "n02293868": 2928, "n02294097": 2929, "n02294407": 2930, "n02294577": 2931, "n02295064": 2932, "n02295390": 2933, "n02295870": 2934, "n02296021": 2935, "n02296276": 2936, "n02296612": 2937, "n02296912": 2938, "n02297294": 2939, "n02297442": 2940, "n02297819": 2941, "n02297938": 2942, "n02298095": 2943, "n02298218": 2944, "n02298541": 2945, "n02299039": 2946, "n02299157": 2947, "n02299378": 2948, "n02299505": 2949, "n02299846": 2950, "n02300173": 2951, "n02300554": 2952, "n02300797": 2953, "n02301452": 2954, "n02301935": 2955, "n02302244": 2956, "n02302459": 2957, "n02302620": 2958, "n02302969": 2959, "n02303284": 2960, "n02303585": 2961, "n02303777": 2962, "n02304036": 2963, "n02304432": 2964, "n02304657": 2965, "n02304797": 2966, "n02305085": 2967, "n02305407": 2968, "n02305636": 2969, "n02305929": 2970, "n02306433": 2971, "n02306825": 2972, "n02307176": 2973, "n02307325": 2974, "n02307515": 2975, "n02307681": 2976, "n02307910": 2977, "n02308033": 2978, "n02308139": 2979, "n02308471": 2980, "n02308618": 2981, "n02308735": 2982, "n02309120": 2983, "n02309242": 2984, "n02309337": 2985, "n02309841": 2986, "n02310000": 2987, "n02310149": 2988, "n02310334": 2989, "n02310585": 2990, "n02310717": 2991, "n02310941": 2992, "n02311060": 2993, "n02311617": 2994, "n02311748": 2995, "n02312006": 2996, "n02312175": 2997, "n02312325": 2998, "n02312427": 2999, "n02312640": 3000, "n02312912": 3001, "n02313008": 3002, "n02313360": 3003, "n02313709": 3004, "n02315487": 3005, "n02315821": 3006, "n02316707": 3007, "n02317335": 3008, "n02317781": 3009, "n02318167": 3010, "n02318687": 3011, "n02319095": 3012, "n02319308": 3013, "n02319555": 3014, "n02319829": 3015, "n02320127": 3016, "n02320465": 3017, "n02321170": 3018, "n02321529": 3019, "n02322047": 3020, "n02322992": 3021, "n02323449": 3022, "n02323902": 3023, "n02324045": 3024, "n02324431": 3025, "n02324514": 3026, "n02324587": 3027, "n02324850": 3028, "n02325366": 3029, "n02325722": 3030, "n02325884": 3031, "n02326074": 3032, "n02326432": 3033, "n02326763": 3034, "n02326862": 3035, "n02327028": 3036, "n02327175": 3037, "n02327435": 3038, "n02327656": 3039, "n02327842": 3040, "n02328009": 3041, "n02328150": 3042, "n02328429": 3043, "n02328820": 3044, "n02328942": 3045, "n02329401": 3046, "n02330245": 3047, "n02331046": 3048, "n02331309": 3049, "n02331842": 3050, "n02332156": 3051, "n02332447": 3052, "n02332755": 3053, "n02332954": 3054, "n02333190": 3055, "n02333546": 3056, "n02333733": 3057, "n02333819": 3058, "n02333909": 3059, "n02334201": 3060, "n02334460": 3061, "n02334728": 3062, "n02335127": 3063, "n02335231": 3064, "n02336011": 3065, "n02336275": 3066, "n02336641": 3067, "n02336826": 3068, "n02337001": 3069, "n02337171": 3070, "n02337332": 3071, "n02337598": 3072, "n02337902": 3073, "n02338145": 3074, "n02338449": 3075, "n02338722": 3076, "n02338901": 3077, "n02339282": 3078, "n02339376": 3079, "n02339922": 3080, "n02340186": 3081, "n02340358": 3082, "n02340640": 3083, "n02340930": 3084, "n02341288": 3085, "n02341475": 3086, "n02341616": 3087, "n02341974": 3088, "n02342250": 3089, "n02342534": 3090, "n02342885": 3091, "n02343058": 3092, "n02343320": 3093, "n02343772": 3094, "n02344175": 3095, "n02344270": 3096, "n02344408": 3097, "n02344528": 3098, "n02344918": 3099, "n02345078": 3100, "n02345340": 3101, "n02345600": 3102, "n02345774": 3103, "n02345997": 3104, "n02346170": 3105, "n02346627": 3106, "n02346998": 3107, "n02347274": 3108, "n02347573": 3109, "n02347744": 3110, "n02348173": 3111, "n02348788": 3112, "n02349205": 3113, "n02349390": 3114, "n02349557": 3115, "n02349847": 3116, "n02350105": 3117, "n02350357": 3118, "n02350670": 3119, "n02350989": 3120, "n02351343": 3121, "n02351870": 3122, "n02352002": 3123, "n02352290": 3124, "n02352591": 3125, "n02352932": 3126, "n02353172": 3127, "n02353411": 3128, "n02353861": 3129, "n02354162": 3130, "n02354320": 3131, "n02354621": 3132, "n02354781": 3133, "n02355227": 3134, "n02355477": 3135, "n02356381": 3136, "n02356612": 3137, "n02356798": 3138, "n02356977": 3139, "n02357111": 3140, "n02357401": 3141, "n02357585": 3142, "n02357911": 3143, "n02358091": 3144, "n02358390": 3145, "n02358584": 3146, "n02358712": 3147, "n02358890": 3148, "n02359047": 3149, "n02359324": 3150, "n02359556": 3151, "n02359667": 3152, "n02359915": 3153, "n02360282": 3154, "n02360480": 3155, "n02360781": 3156, "n02360933": 3157, "n02361090": 3158, "n02361337": 3159, "n02361587": 3160, "n02361706": 3161, "n02361850": 3162, "n02362194": 3163, "n02363005": 3164, "n02363245": 3165, "n02363351": 3166, "n02363996": 3167, "n02364520": 3168, "n02364673": 3169, "n02364840": 3170, "n02365108": 3171, "n02365480": 3172, "n02366002": 3173, "n02366301": 3174, "n02366579": 3175, "n02366959": 3176, "n02367492": 3177, "n02367812": 3178, "n02368116": 3179, "n02368399": 3180, "n02368821": 3181, "n02369293": 3182, "n02369555": 3183, "n02369680": 3184, "n02369935": 3185, "n02370137": 3186, "n02370525": 3187, "n02370806": 3188, "n02371344": 3189, "n02372140": 3190, "n02372584": 3191, "n02372952": 3192, "n02373336": 3193, "n02374149": 3194, "n02374451": 3195, "n02375302": 3196, "n02375438": 3197, "n02375757": 3198, "n02375862": 3199, "n02376542": 3200, "n02376679": 3201, "n02376791": 3202, "n02376918": 3203, "n02377063": 3204, "n02377181": 3205, "n02377291": 3206, "n02377388": 3207, "n02377480": 3208, "n02377603": 3209, "n02377703": 3210, "n02378149": 3211, "n02378299": 3212, "n02378415": 3213, "n02378541": 3214, "n02378625": 3215, "n02378755": 3216, "n02378870": 3217, "n02378969": 3218, "n02379081": 3219, "n02379183": 3220, "n02379329": 3221, "n02379430": 3222, "n02379630": 3223, "n02379743": 3224, "n02379908": 3225, "n02380052": 3226, "n02380335": 3227, "n02380464": 3228, "n02380583": 3229, "n02380745": 3230, "n02380875": 3231, "n02381004": 3232, "n02381119": 3233, "n02381261": 3234, "n02381364": 3235, "n02381460": 3236, "n02381609": 3237, "n02381831": 3238, "n02382039": 3239, "n02382132": 3240, "n02382204": 3241, "n02382338": 3242, "n02382437": 3243, "n02382635": 3244, "n02382750": 3245, "n02382850": 3246, "n02382948": 3247, "n02383231": 3248, "n02384741": 3249, "n02384858": 3250, "n02385002": 3251, "n02385098": 3252, "n02385214": 3253, "n02385580": 3254, "n02385676": 3255, "n02385776": 3256, "n02385898": 3257, "n02386014": 3258, "n02386141": 3259, "n02386224": 3260, "n02386310": 3261, "n02386496": 3262, "n02386746": 3263, "n02386853": 3264, "n02386968": 3265, "n02387093": 3266, "n02387254": 3267, "n02387346": 3268, "n02387452": 3269, "n02387722": 3270, "n02387887": 3271, "n02387983": 3272, "n02388143": 3273, "n02388276": 3274, "n02388453": 3275, "n02388588": 3276, "n02388735": 3277, "n02388832": 3278, "n02388917": 3279, "n02389026": 3280, "n02389128": 3281, "n02389261": 3282, "n02389346": 3283, "n02389559": 3284, "n02389779": 3285, "n02389865": 3286, "n02389943": 3287, "n02390015": 3288, "n02390101": 3289, "n02390258": 3290, "n02390454": 3291, "n02390640": 3292, "n02390738": 3293, "n02390834": 3294, "n02390938": 3295, "n02391049": 3296, "n02391234": 3297, "n02391373": 3298, "n02391508": 3299, "n02391617": 3300, "n02391994": 3301, "n02392434": 3302, "n02392555": 3303, "n02392824": 3304, "n02393161": 3305, "n02393580": 3306, "n02393807": 3307, "n02393940": 3308, "n02394477": 3309, "n02395003": 3310, "n02395406": 3311, "n02395694": 3312, "n02395855": 3313, "n02395931": 3314, "n02396014": 3315, "n02396088": 3316, "n02396157": 3317, "n02396427": 3318, "n02396796": 3319, "n02397096": 3320, "n02397529": 3321, "n02397744": 3322, "n02397987": 3323, "n02398521": 3324, "n02399000": 3325, "n02401031": 3326, "n02402010": 3327, "n02402175": 3328, "n02402425": 3329, "n02403003": 3330, "n02403153": 3331, "n02403231": 3332, "n02403325": 3333, "n02403454": 3334, "n02403740": 3335, "n02403820": 3336, "n02403920": 3337, "n02404028": 3338, "n02404186": 3339, "n02404432": 3340, "n02404573": 3341, "n02404906": 3342, "n02405101": 3343, "n02405302": 3344, "n02405440": 3345, "n02405577": 3346, "n02405692": 3347, "n02405799": 3348, "n02405929": 3349, "n02406046": 3350, "n02406174": 3351, "n02406432": 3352, "n02406533": 3353, "n02406647": 3354, "n02406749": 3355, "n02406859": 3356, "n02406952": 3357, "n02407071": 3358, "n02407172": 3359, "n02407276": 3360, "n02407390": 3361, "n02407521": 3362, "n02407625": 3363, "n02407763": 3364, "n02407959": 3365, "n02408429": 3366, "n02408660": 3367, "n02408817": 3368, "n02409038": 3369, "n02409202": 3370, "n02409508": 3371, "n02409870": 3372, "n02410011": 3373, "n02410141": 3374, "n02410509": 3375, "n02410702": 3376, "n02410900": 3377, "n02411206": 3378, "n02411705": 3379, "n02411999": 3380, "n02412080": 3381, "n02412210": 3382, "n02412440": 3383, "n02412629": 3384, "n02412700": 3385, "n02412787": 3386, "n02412909": 3387, "n02412977": 3388, "n02413050": 3389, "n02413131": 3390, "n02413484": 3391, "n02413593": 3392, "n02413717": 3393, "n02413824": 3394, "n02413917": 3395, "n02414043": 3396, "n02414209": 3397, "n02414290": 3398, "n02414442": 3399, "n02414578": 3400, "n02414763": 3401, "n02414904": 3402, "n02415130": 3403, "n02415253": 3404, "n02415435": 3405, "n02415577": 3406, "n02415829": 3407, "n02416104": 3408, "n02416519": 3409, "n02416820": 3410, "n02416880": 3411, "n02416964": 3412, "n02417070": 3413, "n02417242": 3414, "n02417387": 3415, "n02417534": 3416, "n02417663": 3417, "n02417785": 3418, "n02417914": 3419, "n02418064": 3420, "n02418465": 3421, "n02418770": 3422, "n02419056": 3423, "n02419336": 3424, "n02419634": 3425, "n02419796": 3426, "n02420509": 3427, "n02420828": 3428, "n02421136": 3429, "n02421449": 3430, "n02421792": 3431, "n02422106": 3432, "n02422391": 3433, "n02422699": 3434, "n02423022": 3435, "n02423218": 3436, "n02423362": 3437, "n02423589": 3438, "n02424085": 3439, "n02424305": 3440, "n02424486": 3441, "n02424589": 3442, "n02424695": 3443, "n02424909": 3444, "n02425086": 3445, "n02425228": 3446, "n02425532": 3447, "n02425887": 3448, "n02426176": 3449, "n02426481": 3450, "n02426813": 3451, "n02427032": 3452, "n02427183": 3453, "n02427470": 3454, "n02427576": 3455, "n02427724": 3456, "n02428089": 3457, "n02428349": 3458, "n02428508": 3459, "n02428842": 3460, "n02429456": 3461, "n02430045": 3462, "n02430559": 3463, "n02430643": 3464, "n02430748": 3465, "n02430830": 3466, "n02431122": 3467, "n02431337": 3468, "n02431441": 3469, "n02431542": 3470, "n02431628": 3471, "n02431785": 3472, "n02431976": 3473, "n02432291": 3474, "n02432511": 3475, "n02432704": 3476, "n02432983": 3477, "n02433318": 3478, "n02433546": 3479, "n02433729": 3480, "n02433925": 3481, "n02434190": 3482, "n02434415": 3483, "n02434712": 3484, "n02434954": 3485, "n02435216": 3486, "n02435517": 3487, "n02435853": 3488, "n02436224": 3489, "n02436353": 3490, "n02436645": 3491, "n02437136": 3492, "n02437312": 3493, "n02437482": 3494, "n02437616": 3495, "n02437971": 3496, "n02438173": 3497, "n02438272": 3498, "n02438580": 3499, "n02439033": 3500, "n02439398": 3501, "n02441326": 3502, "n02441942": 3503, "n02442172": 3504, "n02442336": 3505, "n02442446": 3506, "n02442572": 3507, "n02442668": 3508, "n02442845": 3509, "n02443015": 3510, "n02443114": 3511, "n02443346": 3512, "n02443484": 3513, "n02443808": 3514, "n02443959": 3515, "n02444251": 3516, "n02444819": 3517, "n02445004": 3518, "n02445171": 3519, "n02445394": 3520, "n02445715": 3521, "n02446206": 3522, "n02446352": 3523, "n02446645": 3524, "n02447021": 3525, "n02447366": 3526, "n02447762": 3527, "n02448060": 3528, "n02448318": 3529, "n02448633": 3530, "n02448885": 3531, "n02449183": 3532, "n02449350": 3533, "n02449699": 3534, "n02450034": 3535, "n02450295": 3536, "n02450426": 3537, "n02450561": 3538, "n02450677": 3539, "n02450829": 3540, "n02451125": 3541, "n02451415": 3542, "n02451575": 3543, "n02453108": 3544, "n02453611": 3545, "n02454379": 3546, "n02454794": 3547, "n02455135": 3548, "n02455428": 3549, "n02455720": 3550, "n02456008": 3551, "n02456275": 3552, "n02456962": 3553, "n02457408": 3554, "n02457945": 3555, "n02458135": 3556, "n02458517": 3557, "n02459190": 3558, "n02460009": 3559, "n02460451": 3560, "n02460817": 3561, "n02461128": 3562, "n02461830": 3563, "n02462213": 3564, "n02469248": 3565, "n02469472": 3566, "n02469914": 3567, "n02470238": 3568, "n02470325": 3569, "n02470709": 3570, "n02470899": 3571, "n02471300": 3572, "n02471762": 3573, "n02472293": 3574, "n02472987": 3575, "n02473307": 3576, "n02473554": 3577, "n02473720": 3578, "n02473857": 3579, "n02473983": 3580, "n02474110": 3581, "n02474282": 3582, "n02474605": 3583, "n02474777": 3584, "n02475078": 3585, "n02475358": 3586, "n02475669": 3587, "n02476219": 3588, "n02476567": 3589, "n02476870": 3590, "n02477028": 3591, "n02477187": 3592, "n02477329": 3593, "n02477516": 3594, "n02477782": 3595, "n02478239": 3596, "n02478875": 3597, "n02479332": 3598, "n02480153": 3599, "n02480495": 3600, "n02480855": 3601, "n02481103": 3602, "n02481235": 3603, "n02481366": 3604, "n02481500": 3605, "n02481823": 3606, "n02482060": 3607, "n02482286": 3608, "n02482474": 3609, "n02482650": 3610, "n02483092": 3611, "n02483362": 3612, "n02483708": 3613, "n02484322": 3614, "n02484473": 3615, "n02484975": 3616, "n02485225": 3617, "n02485371": 3618, "n02485536": 3619, "n02485688": 3620, "n02485988": 3621, "n02486261": 3622, "n02486410": 3623, "n02486657": 3624, "n02486908": 3625, "n02487079": 3626, "n02487347": 3627, "n02487547": 3628, "n02487675": 3629, "n02487847": 3630, "n02488003": 3631, "n02488291": 3632, "n02488415": 3633, "n02488702": 3634, "n02488894": 3635, "n02489166": 3636, "n02489589": 3637, "n02490219": 3638, "n02490597": 3639, "n02490811": 3640, "n02491107": 3641, "n02491329": 3642, "n02491474": 3643, "n02492035": 3644, "n02492356": 3645, "n02492660": 3646, "n02492948": 3647, "n02493224": 3648, "n02493509": 3649, "n02493793": 3650, "n02494079": 3651, "n02494383": 3652, "n02495242": 3653, "n02496052": 3654, "n02496913": 3655, "n02497673": 3656, "n02498153": 3657, "n02498743": 3658, "n02499022": 3659, "n02499316": 3660, "n02499568": 3661, "n02499808": 3662, "n02500267": 3663, "n02500596": 3664, "n02501583": 3665, "n02501923": 3666, "n02502006": 3667, "n02502514": 3668, "n02502807": 3669, "n02503127": 3670, "n02503517": 3671, "n02503756": 3672, "n02504013": 3673, "n02504458": 3674, "n02504770": 3675, "n02505063": 3676, "n02505238": 3677, "n02505485": 3678, "n02505998": 3679, "n02506947": 3680, "n02507148": 3681, "n02507649": 3682, "n02508021": 3683, "n02508213": 3684, "n02508346": 3685, "n02508742": 3686, "n02509197": 3687, "n02509515": 3688, "n02509815": 3689, "n02510455": 3690, "n02511730": 3691, "n02512053": 3692, "n02512752": 3693, "n02512830": 3694, "n02512938": 3695, "n02513248": 3696, "n02513355": 3697, "n02513560": 3698, "n02513727": 3699, "n02513805": 3700, "n02513939": 3701, "n02514041": 3702, "n02515214": 3703, "n02515713": 3704, "n02516188": 3705, "n02516776": 3706, "n02517442": 3707, "n02517938": 3708, "n02518324": 3709, "n02518622": 3710, "n02519148": 3711, "n02519340": 3712, "n02519472": 3713, "n02519686": 3714, "n02519862": 3715, "n02520147": 3716, "n02520525": 3717, "n02520810": 3718, "n02521646": 3719, "n02522399": 3720, "n02522637": 3721, "n02522722": 3722, "n02522866": 3723, "n02523110": 3724, "n02523427": 3725, "n02523877": 3726, "n02524202": 3727, "n02524524": 3728, "n02524659": 3729, "n02524928": 3730, "n02525382": 3731, "n02525703": 3732, "n02526121": 3733, "n02526425": 3734, "n02526818": 3735, "n02527057": 3736, "n02527271": 3737, "n02527622": 3738, "n02528163": 3739, "n02529293": 3740, "n02529772": 3741, "n02530052": 3742, "n02530188": 3743, "n02530421": 3744, "n02530637": 3745, "n02530831": 3746, "n02530999": 3747, "n02531114": 3748, "n02531625": 3749, "n02532028": 3750, "n02532272": 3751, "n02532451": 3752, "n02532602": 3753, "n02532786": 3754, "n02532918": 3755, "n02533209": 3756, "n02533545": 3757, "n02533834": 3758, "n02534165": 3759, "n02534559": 3760, "n02534734": 3761, "n02535080": 3762, "n02535163": 3763, "n02535258": 3764, "n02535537": 3765, "n02535759": 3766, "n02536165": 3767, "n02536456": 3768, "n02536864": 3769, "n02537085": 3770, "n02537319": 3771, "n02537525": 3772, "n02537716": 3773, "n02538010": 3774, "n02538216": 3775, "n02538406": 3776, "n02538562": 3777, "n02538985": 3778, "n02539424": 3779, "n02539573": 3780, "n02539894": 3781, "n02540412": 3782, "n02540983": 3783, "n02541257": 3784, "n02541687": 3785, "n02542017": 3786, "n02542432": 3787, "n02542958": 3788, "n02543255": 3789, "n02543565": 3790, "n02544274": 3791, "n02545841": 3792, "n02546028": 3793, "n02546331": 3794, "n02546627": 3795, "n02547014": 3796, "n02547733": 3797, "n02548247": 3798, "n02548689": 3799, "n02548884": 3800, "n02549248": 3801, "n02549376": 3802, "n02549989": 3803, "n02550203": 3804, "n02550460": 3805, "n02550655": 3806, "n02551134": 3807, "n02551668": 3808, "n02552171": 3809, "n02553028": 3810, "n02554730": 3811, "n02555863": 3812, "n02556373": 3813, "n02556846": 3814, "n02557182": 3815, "n02557318": 3816, "n02557591": 3817, "n02557749": 3818, "n02557909": 3819, "n02558206": 3820, "n02558860": 3821, "n02559144": 3822, "n02559383": 3823, "n02559862": 3824, "n02560110": 3825, "n02561108": 3826, "n02561381": 3827, "n02561514": 3828, "n02561661": 3829, "n02561803": 3830, "n02561937": 3831, "n02562315": 3832, "n02562796": 3833, "n02562971": 3834, "n02563079": 3835, "n02563182": 3836, "n02563648": 3837, "n02563792": 3838, "n02563949": 3839, "n02564270": 3840, "n02564403": 3841, "n02564720": 3842, "n02564935": 3843, "n02565072": 3844, "n02565324": 3845, "n02565573": 3846, "n02566109": 3847, "n02566489": 3848, "n02566665": 3849, "n02567334": 3850, "n02567633": 3851, "n02568087": 3852, "n02568447": 3853, "n02568959": 3854, "n02569484": 3855, "n02569631": 3856, "n02569905": 3857, "n02570164": 3858, "n02570484": 3859, "n02570838": 3860, "n02571167": 3861, "n02571652": 3862, "n02571810": 3863, "n02572196": 3864, "n02572484": 3865, "n02573249": 3866, "n02573704": 3867, "n02574271": 3868, "n02574910": 3869, "n02575325": 3870, "n02575590": 3871, "n02576223": 3872, "n02576575": 3873, "n02576906": 3874, "n02577041": 3875, "n02577164": 3876, "n02577403": 3877, "n02577662": 3878, "n02577952": 3879, "n02578233": 3880, "n02578454": 3881, "n02578771": 3882, "n02578928": 3883, "n02579303": 3884, "n02579557": 3885, "n02579762": 3886, "n02579928": 3887, "n02580336": 3888, "n02580679": 3889, "n02580830": 3890, "n02581108": 3891, "n02581482": 3892, "n02581642": 3893, "n02581957": 3894, "n02582220": 3895, "n02582349": 3896, "n02582721": 3897, "n02583567": 3898, "n02583890": 3899, "n02584145": 3900, "n02584449": 3901, "n02585872": 3902, "n02586238": 3903, "n02586543": 3904, "n02587051": 3905, "n02587300": 3906, "n02587479": 3907, "n02587618": 3908, "n02587877": 3909, "n02588286": 3910, "n02588794": 3911, "n02588945": 3912, "n02589062": 3913, "n02589196": 3914, "n02589316": 3915, "n02589623": 3916, "n02589796": 3917, "n02590094": 3918, "n02590495": 3919, "n02590702": 3920, "n02590987": 3921, "n02591330": 3922, "n02591613": 3923, "n02591911": 3924, "n02592055": 3925, "n02592371": 3926, "n02592734": 3927, "n02593019": 3928, "n02593191": 3929, "n02593453": 3930, "n02593679": 3931, "n02594250": 3932, "n02594942": 3933, "n02595056": 3934, "n02595339": 3935, "n02595702": 3936, "n02596067": 3937, "n02596252": 3938, "n02596381": 3939, "n02596720": 3940, "n02597004": 3941, "n02597367": 3942, "n02597608": 3943, "n02597818": 3944, "n02597972": 3945, "n02598134": 3946, "n02598573": 3947, "n02598878": 3948, "n02599052": 3949, "n02599347": 3950, "n02599557": 3951, "n02599958": 3952, "n02600298": 3953, "n02600503": 3954, "n02600798": 3955, "n02601344": 3956, "n02601767": 3957, "n02601921": 3958, "n02602059": 3959, "n02602405": 3960, "n02602760": 3961, "n02603317": 3962, "n02603540": 3963, "n02603862": 3964, "n02604157": 3965, "n02604480": 3966, "n02604954": 3967, "n02605316": 3968, "n02605703": 3969, "n02605936": 3970, "n02606052": 3971, "n02606384": 3972, "n02606751": 3973, "n02607072": 3974, "n02607201": 3975, "n02607470": 3976, "n02607862": 3977, "n02608284": 3978, "n02608547": 3979, "n02608860": 3980, "n02608996": 3981, "n02609302": 3982, "n02609823": 3983, "n02610066": 3984, "n02610373": 3985, "n02610664": 3986, "n02610980": 3987, "n02611561": 3988, "n02611898": 3989, "n02612167": 3990, "n02613181": 3991, "n02613572": 3992, "n02613820": 3993, "n02614140": 3994, "n02614482": 3995, "n02614653": 3996, "n02614978": 3997, "n02615298": 3998, "n02616128": 3999, "n02616397": 4000, "n02616851": 4001, "n02617537": 4002, "n02618094": 4003, "n02618513": 4004, "n02618827": 4005, "n02619165": 4006, "n02619550": 4007, "n02619861": 4008, "n02620167": 4009, "n02620578": 4010, "n02621258": 4011, "n02621908": 4012, "n02622249": 4013, "n02622547": 4014, "n02622712": 4015, "n02622955": 4016, "n02623445": 4017, "n02624167": 4018, "n02624551": 4019, "n02624807": 4020, "n02624987": 4021, "n02625258": 4022, "n02625612": 4023, "n02625851": 4024, "n02626089": 4025, "n02626265": 4026, "n02626471": 4027, "n02626762": 4028, "n02627037": 4029, "n02627292": 4030, "n02627532": 4031, "n02627835": 4032, "n02628062": 4033, "n02628259": 4034, "n02628600": 4035, "n02629230": 4036, "n02629716": 4037, "n02630281": 4038, "n02630615": 4039, "n02630739": 4040, "n02631041": 4041, "n02631330": 4042, "n02631475": 4043, "n02631628": 4044, "n02631775": 4045, "n02632039": 4046, "n02632494": 4047, "n02633422": 4048, "n02633677": 4049, "n02633977": 4050, "n02634545": 4051, "n02635154": 4052, "n02635580": 4053, "n02636170": 4054, "n02636405": 4055, "n02636550": 4056, "n02636854": 4057, "n02637179": 4058, "n02637475": 4059, "n02637977": 4060, "n02638596": 4061, "n02639087": 4062, "n02639605": 4063, "n02639922": 4064, "n02640242": 4065, "n02640626": 4066, "n02640857": 4067, "n02641379": 4068, "n02642107": 4069, "n02642644": 4070, "n02643112": 4071, "n02643316": 4072, "n02643566": 4073, "n02643836": 4074, "n02644113": 4075, "n02644360": 4076, "n02644501": 4077, "n02644665": 4078, "n02644817": 4079, "n02645538": 4080, "n02645691": 4081, "n02645953": 4082, "n02646667": 4083, "n02646892": 4084, "n02648035": 4085, "n02648625": 4086, "n02648916": 4087, "n02649218": 4088, "n02649546": 4089, "n02650050": 4090, "n02650413": 4091, "n02650541": 4092, "n02651060": 4093, "n02652132": 4094, "n02652668": 4095, "n02653145": 4096, "n02653497": 4097, "n02653786": 4098, "n02654112": 4099, "n02654425": 4100, "n02654745": 4101, "n02655020": 4102, "n02655523": 4103, "n02655848": 4104, "n02656032": 4105, "n02656301": 4106, "n02656670": 4107, "n02656969": 4108, "n02657368": 4109, "n02657694": 4110, "n02658079": 4111, "n02658531": 4112, "n02658811": 4113, "n02659176": 4114, "n02659478": 4115, "n02659808": 4116, "n02660091": 4117, "n02660208": 4118, "n02660519": 4119, "n02660640": 4120, "n02661017": 4121, "n02661473": 4122, "n02661618": 4123, "n02662239": 4124, "n02662397": 4125, "n02662559": 4126, "n02662825": 4127, "n02662993": 4128, "n02663211": 4129, "n02663485": 4130, "n02663849": 4131, "n02664285": 4132, "n02664642": 4133, "n02665250": 4134, "n02665985": 4135, "n02666196": 4136, "n02666501": 4137, "n02666624": 4138, "n02666943": 4139, "n02667093": 4140, "n02667244": 4141, "n02667379": 4142, "n02667478": 4143, "n02667576": 4144, "n02667693": 4145, "n02668393": 4146, "n02668613": 4147, "n02669295": 4148, "n02669442": 4149, "n02669534": 4150, "n02669723": 4151, "n02670186": 4152, "n02670382": 4153, "n02670683": 4154, "n02670935": 4155, "n02671780": 4156, "n02672152": 4157, "n02672371": 4158, "n02672831": 4159, "n02675077": 4160, "n02675219": 4161, "n02675522": 4162, "n02676097": 4163, "n02676261": 4164, "n02676566": 4165, "n02676670": 4166, "n02676938": 4167, "n02677028": 4168, "n02677136": 4169, "n02677436": 4170, "n02677718": 4171, "n02678010": 4172, "n02678384": 4173, "n02678897": 4174, "n02679142": 4175, "n02679257": 4176, "n02679961": 4177, "n02680110": 4178, "n02680512": 4179, "n02680638": 4180, "n02680754": 4181, "n02681392": 4182, "n02682311": 4183, "n02682407": 4184, "n02682569": 4185, "n02682811": 4186, "n02682922": 4187, "n02683183": 4188, "n02683323": 4189, "n02683454": 4190, "n02683558": 4191, "n02683791": 4192, "n02684248": 4193, "n02684356": 4194, "n02684515": 4195, "n02684649": 4196, "n02684962": 4197, "n02685082": 4198, "n02685253": 4199, "n02685365": 4200, "n02685701": 4201, "n02685995": 4202, "n02686121": 4203, "n02686227": 4204, "n02686379": 4205, "n02686568": 4206, "n02687172": 4207, "n02687423": 4208, "n02687682": 4209, "n02687821": 4210, "n02687992": 4211, "n02688273": 4212, "n02688443": 4213, "n02689144": 4214, "n02689274": 4215, "n02689434": 4216, "n02689748": 4217, "n02689819": 4218, "n02690373": 4219, "n02690715": 4220, "n02691156": 4221, "n02692086": 4222, "n02692232": 4223, "n02692513": 4224, "n02692680": 4225, "n02692877": 4226, "n02693246": 4227, "n02693413": 4228, "n02693540": 4229, "n02694045": 4230, "n02694279": 4231, "n02694426": 4232, "n02694662": 4233, "n02694966": 4234, "n02695627": 4235, "n02695762": 4236, "n02696165": 4237, "n02696246": 4238, "n02696569": 4239, "n02696843": 4240, "n02697022": 4241, "n02697221": 4242, "n02697576": 4243, "n02697675": 4244, "n02697876": 4245, "n02698244": 4246, "n02698473": 4247, "n02698634": 4248, "n02699494": 4249, "n02699629": 4250, "n02699770": 4251, "n02699915": 4252, "n02700064": 4253, "n02700258": 4254, "n02700895": 4255, "n02701002": 4256, "n02701260": 4257, "n02701730": 4258, "n02702989": 4259, "n02703124": 4260, "n02703275": 4261, "n02704645": 4262, "n02704792": 4263, "n02704949": 4264, "n02705201": 4265, "n02705429": 4266, "n02705944": 4267, "n02706221": 4268, "n02706806": 4269, "n02708093": 4270, "n02708224": 4271, "n02708433": 4272, "n02708555": 4273, "n02708711": 4274, "n02708885": 4275, "n02709101": 4276, "n02709367": 4277, "n02709637": 4278, "n02709763": 4279, "n02709908": 4280, "n02710044": 4281, "n02710201": 4282, "n02710324": 4283, "n02710429": 4284, "n02710600": 4285, "n02711237": 4286, "n02711780": 4287, "n02712545": 4288, "n02712643": 4289, "n02713003": 4290, "n02713218": 4291, "n02713364": 4292, "n02713496": 4293, "n02714315": 4294, "n02714535": 4295, "n02714751": 4296, "n02715229": 4297, "n02715513": 4298, "n02715712": 4299, "n02716626": 4300, "n02720048": 4301, "n02720576": 4302, "n02721813": 4303, "n02723165": 4304, "n02724722": 4305, "n02725872": 4306, "n02726017": 4307, "n02726210": 4308, "n02726305": 4309, "n02726681": 4310, "n02727016": 4311, "n02727141": 4312, "n02727426": 4313, "n02727825": 4314, "n02728440": 4315, "n02729222": 4316, "n02729837": 4317, "n02729965": 4318, "n02730265": 4319, "n02730568": 4320, "n02730930": 4321, "n02731251": 4322, "n02731398": 4323, "n02731629": 4324, "n02731900": 4325, "n02732072": 4326, "n02732572": 4327, "n02732827": 4328, "n02733213": 4329, "n02733524": 4330, "n02734725": 4331, "n02734835": 4332, "n02735268": 4333, "n02735361": 4334, "n02735538": 4335, "n02735688": 4336, "n02736396": 4337, "n02736798": 4338, "n02737351": 4339, "n02737660": 4340, "n02738031": 4341, "n02738271": 4342, "n02738449": 4343, "n02738535": 4344, "n02738741": 4345, "n02738859": 4346, "n02738978": 4347, "n02739123": 4348, "n02739427": 4349, "n02739550": 4350, "n02739668": 4351, "n02739889": 4352, "n02740061": 4353, "n02740300": 4354, "n02740533": 4355, "n02740764": 4356, "n02741367": 4357, "n02741475": 4358, "n02742070": 4359, "n02742194": 4360, "n02742322": 4361, "n02742468": 4362, "n02742753": 4363, "n02743426": 4364, "n02744323": 4365, "n02744844": 4366, "n02744961": 4367, "n02745492": 4368, "n02745611": 4369, "n02745816": 4370, "n02746008": 4371, "n02746225": 4372, "n02746365": 4373, "n02746595": 4374, "n02746683": 4375, "n02746978": 4376, "n02747063": 4377, "n02747177": 4378, "n02747672": 4379, "n02747802": 4380, "n02748183": 4381, "n02748359": 4382, "n02748491": 4383, "n02749169": 4384, "n02749292": 4385, "n02749479": 4386, "n02749670": 4387, "n02749790": 4388, "n02749953": 4389, "n02750070": 4390, "n02750169": 4391, "n02750320": 4392, "n02750652": 4393, "n02751067": 4394, "n02751215": 4395, "n02751295": 4396, "n02751490": 4397, "n02752199": 4398, "n02752496": 4399, "n02752615": 4400, "n02752810": 4401, "n02752917": 4402, "n02753044": 4403, "n02753394": 4404, "n02753710": 4405, "n02754103": 4406, "n02754656": 4407, "n02755140": 4408, "n02755352": 4409, "n02755529": 4410, "n02755675": 4411, "n02755823": 4412, "n02755984": 4413, "n02756098": 4414, "n02756854": 4415, "n02756977": 4416, "n02757061": 4417, "n02757337": 4418, "n02757462": 4419, "n02757714": 4420, "n02757810": 4421, "n02757927": 4422, "n02758134": 4423, "n02758490": 4424, "n02758863": 4425, "n02758960": 4426, "n02759257": 4427, "n02759387": 4428, "n02759700": 4429, "n02759963": 4430, "n02760099": 4431, "n02760199": 4432, "n02760298": 4433, "n02760429": 4434, "n02760658": 4435, "n02760855": 4436, "n02761034": 4437, "n02761206": 4438, "n02761392": 4439, "n02761557": 4440, "n02761696": 4441, "n02761834": 4442, "n02762169": 4443, "n02762371": 4444, "n02762508": 4445, "n02762725": 4446, "n02762909": 4447, "n02763083": 4448, "n02763198": 4449, "n02763306": 4450, "n02763604": 4451, "n02763714": 4452, "n02763901": 4453, "n02764044": 4454, "n02764398": 4455, "n02764505": 4456, "n02764614": 4457, "n02764779": 4458, "n02764935": 4459, "n02765028": 4460, "n02766168": 4461, "n02766320": 4462, "n02766534": 4463, "n02766792": 4464, "n02767038": 4465, "n02767147": 4466, "n02767433": 4467, "n02767665": 4468, "n02767956": 4469, "n02768114": 4470, "n02768226": 4471, "n02768433": 4472, "n02768655": 4473, "n02768973": 4474, "n02769075": 4475, "n02769290": 4476, "n02769669": 4477, "n02769748": 4478, "n02769963": 4479, "n02770078": 4480, "n02770211": 4481, "n02770585": 4482, "n02770721": 4483, "n02770830": 4484, "n02771004": 4485, "n02771166": 4486, "n02771286": 4487, "n02771547": 4488, "n02771750": 4489, "n02772101": 4490, "n02772435": 4491, "n02772554": 4492, "n02772700": 4493, "n02773037": 4494, "n02773838": 4495, "n02774152": 4496, "n02774630": 4497, "n02774921": 4498, "n02775039": 4499, "n02775178": 4500, "n02775483": 4501, "n02775689": 4502, "n02775813": 4503, "n02775897": 4504, "n02776007": 4505, "n02776205": 4506, "n02776505": 4507, "n02776631": 4508, "n02776825": 4509, "n02776978": 4510, "n02777100": 4511, "n02777292": 4512, "n02777402": 4513, "n02777638": 4514, "n02777734": 4515, "n02777927": 4516, "n02778131": 4517, "n02778294": 4518, "n02778456": 4519, "n02778588": 4520, "n02778669": 4521, "n02779435": 4522, "n02779609": 4523, "n02779719": 4524, "n02779971": 4525, "n02780315": 4526, "n02780445": 4527, "n02780588": 4528, "n02780704": 4529, "n02780815": 4530, "n02781121": 4531, "n02781213": 4532, "n02781338": 4533, "n02781517": 4534, "n02781764": 4535, "n02782093": 4536, "n02782432": 4537, "n02782602": 4538, "n02782681": 4539, "n02782778": 4540, "n02783035": 4541, "n02783161": 4542, "n02783324": 4543, "n02783459": 4544, "n02783900": 4545, "n02783994": 4546, "n02784124": 4547, "n02784998": 4548, "n02785648": 4549, "n02786058": 4550, "n02786198": 4551, "n02786331": 4552, "n02786463": 4553, "n02786611": 4554, "n02786736": 4555, "n02786837": 4556, "n02787120": 4557, "n02787269": 4558, "n02787435": 4559, "n02787622": 4560, "n02788021": 4561, "n02788148": 4562, "n02788386": 4563, "n02788462": 4564, "n02788572": 4565, "n02788689": 4566, "n02789487": 4567, "n02790669": 4568, "n02790823": 4569, "n02790996": 4570, "n02791124": 4571, "n02791270": 4572, "n02791532": 4573, "n02791665": 4574, "n02791795": 4575, "n02792409": 4576, "n02792552": 4577, "n02792948": 4578, "n02793089": 4579, "n02793199": 4580, "n02793296": 4581, "n02793414": 4582, "n02793495": 4583, "n02793684": 4584, "n02793842": 4585, "n02793930": 4586, "n02794008": 4587, "n02794156": 4588, "n02794368": 4589, "n02794474": 4590, "n02794664": 4591, "n02794779": 4592, "n02794972": 4593, "n02795169": 4594, "n02795528": 4595, "n02795670": 4596, "n02795783": 4597, "n02795978": 4598, "n02796207": 4599, "n02796318": 4600, "n02796412": 4601, "n02796623": 4602, "n02796995": 4603, "n02797295": 4604, "n02797535": 4605, "n02797692": 4606, "n02797881": 4607, "n02799071": 4608, "n02799175": 4609, "n02799323": 4610, "n02799897": 4611, "n02800213": 4612, "n02800497": 4613, "n02800675": 4614, "n02800940": 4615, "n02801047": 4616, "n02801184": 4617, "n02801450": 4618, "n02801525": 4619, "n02801823": 4620, "n02801938": 4621, "n02802215": 4622, "n02802426": 4623, "n02802544": 4624, "n02802721": 4625, "n02802990": 4626, "n02803349": 4627, "n02803539": 4628, "n02803666": 4629, "n02803809": 4630, "n02803934": 4631, "n02804123": 4632, "n02804252": 4633, "n02804414": 4634, "n02804515": 4635, "n02804610": 4636, "n02805283": 4637, "n02805845": 4638, "n02805983": 4639, "n02806088": 4640, "n02806379": 4641, "n02806530": 4642, "n02806762": 4643, "n02806875": 4644, "n02806992": 4645, "n02807133": 4646, "n02807523": 4647, "n02807616": 4648, "n02807731": 4649, "n02808185": 4650, "n02808304": 4651, "n02808440": 4652, "n02808829": 4653, "n02808968": 4654, "n02809105": 4655, "n02809241": 4656, "n02809364": 4657, "n02809491": 4658, "n02809605": 4659, "n02809736": 4660, "n02810139": 4661, "n02810270": 4662, "n02810471": 4663, "n02810782": 4664, "n02811059": 4665, "n02811204": 4666, "n02811350": 4667, "n02811468": 4668, "n02811618": 4669, "n02811719": 4670, "n02811936": 4671, "n02812201": 4672, "n02812342": 4673, "n02812631": 4674, "n02812785": 4675, "n02812949": 4676, "n02813252": 4677, "n02813399": 4678, "n02813544": 4679, "n02813645": 4680, "n02813752": 4681, "n02813981": 4682, "n02814116": 4683, "n02814338": 4684, "n02814428": 4685, "n02814533": 4686, "n02814774": 4687, "n02814860": 4688, "n02815478": 4689, "n02815749": 4690, "n02815834": 4691, "n02815950": 4692, "n02816494": 4693, "n02816656": 4694, "n02816768": 4695, "n02817031": 4696, "n02817251": 4697, "n02817386": 4698, "n02817516": 4699, "n02817650": 4700, "n02817799": 4701, "n02818135": 4702, "n02818254": 4703, "n02818687": 4704, "n02818832": 4705, "n02819697": 4706, "n02820085": 4707, "n02820210": 4708, "n02820556": 4709, "n02820675": 4710, "n02821202": 4711, "n02821415": 4712, "n02821543": 4713, "n02821627": 4714, "n02821943": 4715, "n02822064": 4716, "n02822220": 4717, "n02822399": 4718, "n02822579": 4719, "n02822762": 4720, "n02822865": 4721, "n02823124": 4722, "n02823335": 4723, "n02823428": 4724, "n02823510": 4725, "n02823586": 4726, "n02823750": 4727, "n02823848": 4728, "n02823964": 4729, "n02824058": 4730, "n02824152": 4731, "n02824319": 4732, "n02824448": 4733, "n02825153": 4734, "n02825240": 4735, "n02825442": 4736, "n02825657": 4737, "n02825872": 4738, "n02825961": 4739, "n02826068": 4740, "n02826259": 4741, "n02826459": 4742, "n02826589": 4743, "n02826683": 4744, "n02826812": 4745, "n02826886": 4746, "n02827148": 4747, "n02827606": 4748, "n02828115": 4749, "n02828299": 4750, "n02828427": 4751, "n02828884": 4752, "n02829246": 4753, "n02829353": 4754, "n02829510": 4755, "n02829596": 4756, "n02830157": 4757, "n02831237": 4758, "n02831335": 4759, "n02831595": 4760, "n02831724": 4761, "n02831894": 4762, "n02831998": 4763, "n02833040": 4764, "n02833140": 4765, "n02833275": 4766, "n02833403": 4767, "n02833793": 4768, "n02834027": 4769, "n02834397": 4770, "n02834506": 4771, "n02834642": 4772, "n02834778": 4773, "n02835271": 4774, "n02835412": 4775, "n02835551": 4776, "n02835724": 4777, "n02835829": 4778, "n02835915": 4779, "n02836035": 4780, "n02836174": 4781, "n02836268": 4782, "n02836392": 4783, "n02836513": 4784, "n02836607": 4785, "n02836900": 4786, "n02837134": 4787, "n02837567": 4788, "n02837789": 4789, "n02837887": 4790, "n02838014": 4791, "n02838178": 4792, "n02838345": 4793, "n02838577": 4794, "n02838728": 4795, "n02838958": 4796, "n02839110": 4797, "n02839351": 4798, "n02839592": 4799, "n02839910": 4800, "n02840134": 4801, "n02840245": 4802, "n02840515": 4803, "n02840619": 4804, "n02841063": 4805, "n02841187": 4806, "n02841315": 4807, "n02841506": 4808, "n02841641": 4809, "n02841847": 4810, "n02842133": 4811, "n02842573": 4812, "n02842809": 4813, "n02843029": 4814, "n02843158": 4815, "n02843276": 4816, "n02843465": 4817, "n02843553": 4818, "n02843684": 4819, "n02843777": 4820, "n02843909": 4821, "n02844056": 4822, "n02844214": 4823, "n02844307": 4824, "n02844714": 4825, "n02845130": 4826, "n02845293": 4827, "n02845985": 4828, "n02846141": 4829, "n02846260": 4830, "n02846511": 4831, "n02846619": 4832, "n02846733": 4833, "n02846874": 4834, "n02847461": 4835, "n02847631": 4836, "n02847852": 4837, "n02848118": 4838, "n02848216": 4839, "n02848523": 4840, "n02848806": 4841, "n02848921": 4842, "n02849154": 4843, "n02849885": 4844, "n02850060": 4845, "n02850358": 4846, "n02850732": 4847, "n02850950": 4848, "n02851099": 4849, "n02851795": 4850, "n02851939": 4851, "n02852043": 4852, "n02852173": 4853, "n02852360": 4854, "n02853016": 4855, "n02853218": 4856, "n02853336": 4857, "n02853745": 4858, "n02853870": 4859, "n02854378": 4860, "n02854532": 4861, "n02854630": 4862, "n02854739": 4863, "n02854926": 4864, "n02855089": 4865, "n02855390": 4866, "n02855701": 4867, "n02855793": 4868, "n02855925": 4869, "n02856013": 4870, "n02856237": 4871, "n02856362": 4872, "n02857365": 4873, "n02857477": 4874, "n02857644": 4875, "n02857907": 4876, "n02858304": 4877, "n02859184": 4878, "n02859343": 4879, "n02859443": 4880, "n02859557": 4881, "n02859729": 4882, "n02859955": 4883, "n02860415": 4884, "n02860640": 4885, "n02860847": 4886, "n02861022": 4887, "n02861147": 4888, "n02861286": 4889, "n02861387": 4890, "n02861509": 4891, "n02861658": 4892, "n02861777": 4893, "n02861886": 4894, "n02862048": 4895, "n02862916": 4896, "n02863014": 4897, "n02863176": 4898, "n02863340": 4899, "n02863426": 4900, "n02863536": 4901, "n02863638": 4902, "n02863750": 4903, "n02864122": 4904, "n02864504": 4905, "n02864593": 4906, "n02864987": 4907, "n02865351": 4908, "n02865665": 4909, "n02865931": 4910, "n02866106": 4911, "n02866386": 4912, "n02866578": 4913, "n02867401": 4914, "n02867592": 4915, "n02867715": 4916, "n02867966": 4917, "n02868240": 4918, "n02868429": 4919, "n02868546": 4920, "n02868638": 4921, "n02868975": 4922, "n02869155": 4923, "n02869249": 4924, "n02869563": 4925, "n02869737": 4926, "n02869837": 4927, "n02870526": 4928, "n02870676": 4929, "n02870772": 4930, "n02870880": 4931, "n02871005": 4932, "n02871147": 4933, "n02871314": 4934, "n02871439": 4935, "n02871525": 4936, "n02871631": 4937, "n02871824": 4938, "n02871963": 4939, "n02872333": 4940, "n02872529": 4941, "n02872752": 4942, "n02873520": 4943, "n02873623": 4944, "n02873733": 4945, "n02873839": 4946, "n02874086": 4947, "n02874214": 4948, "n02874336": 4949, "n02874442": 4950, "n02874537": 4951, "n02874642": 4952, "n02874750": 4953, "n02875436": 4954, "n02875626": 4955, "n02875948": 4956, "n02876084": 4957, "n02876326": 4958, "n02876457": 4959, "n02876657": 4960, "n02877266": 4961, "n02877513": 4962, "n02877642": 4963, "n02877765": 4964, "n02877962": 4965, "n02878107": 4966, "n02878222": 4967, "n02878425": 4968, "n02878534": 4969, "n02878628": 4970, "n02878796": 4971, "n02879087": 4972, "n02879309": 4973, "n02879422": 4974, "n02879517": 4975, "n02879718": 4976, "n02880189": 4977, "n02880393": 4978, "n02880546": 4979, "n02880842": 4980, "n02880940": 4981, "n02881193": 4982, "n02881546": 4983, "n02881757": 4984, "n02881906": 4985, "n02882190": 4986, "n02882301": 4987, "n02882483": 4988, "n02882647": 4989, "n02882894": 4990, "n02883004": 4991, "n02883101": 4992, "n02883205": 4993, "n02883344": 4994, "n02884225": 4995, "n02884450": 4996, "n02884859": 4997, "n02884994": 4998, "n02885108": 4999, "n02885233": 5000, "n02885338": 5001, "n02885462": 5002, "n02885882": 5003, "n02886321": 5004, "n02886434": 5005, "n02886599": 5006, "n02887079": 5007, "n02887209": 5008, "n02887489": 5009, "n02887832": 5010, "n02887970": 5011, "n02888270": 5012, "n02888429": 5013, "n02888569": 5014, "n02888898": 5015, "n02889425": 5016, "n02889646": 5017, "n02889856": 5018, "n02889996": 5019, "n02890188": 5020, "n02890351": 5021, "n02890513": 5022, "n02890662": 5023, "n02890804": 5024, "n02890940": 5025, "n02891188": 5026, "n02891788": 5027, "n02892201": 5028, "n02892304": 5029, "n02892392": 5030, "n02892499": 5031, "n02892626": 5032, "n02892767": 5033, "n02892948": 5034, "n02893269": 5035, "n02893418": 5036, "n02893608": 5037, "n02893692": 5038, "n02893941": 5039, "n02894024": 5040, "n02894158": 5041, "n02894337": 5042, "n02894605": 5043, "n02894847": 5044, "n02895008": 5045, "n02895154": 5046, "n02895328": 5047, "n02895438": 5048, "n02896074": 5049, "n02896294": 5050, "n02896442": 5051, "n02896694": 5052, "n02896856": 5053, "n02896949": 5054, "n02897097": 5055, "n02897389": 5056, "n02897820": 5057, "n02898093": 5058, "n02898173": 5059, "n02898269": 5060, "n02898369": 5061, "n02898585": 5062, "n02898711": 5063, "n02899439": 5064, "n02900160": 5065, "n02900459": 5066, "n02900594": 5067, "n02900705": 5068, "n02900857": 5069, "n02900987": 5070, "n02901114": 5071, "n02901259": 5072, "n02901377": 5073, "n02901481": 5074, "n02901620": 5075, "n02901793": 5076, "n02901901": 5077, "n02902079": 5078, "n02902687": 5079, "n02902816": 5080, "n02902916": 5081, "n02903006": 5082, "n02903126": 5083, "n02903204": 5084, "n02903727": 5085, "n02903852": 5086, "n02904109": 5087, "n02904233": 5088, "n02904505": 5089, "n02904640": 5090, "n02904803": 5091, "n02904927": 5092, "n02905036": 5093, "n02905152": 5094, "n02905886": 5095, "n02906734": 5096, "n02906963": 5097, "n02907082": 5098, "n02907296": 5099, "n02907391": 5100, "n02907656": 5101, "n02907873": 5102, "n02908123": 5103, "n02908217": 5104, "n02908773": 5105, "n02908951": 5106, "n02909053": 5107, "n02909165": 5108, "n02909285": 5109, "n02909706": 5110, "n02909870": 5111, "n02910145": 5112, "n02910241": 5113, "n02910353": 5114, "n02910542": 5115, "n02910701": 5116, "n02910864": 5117, "n02910964": 5118, "n02911332": 5119, "n02911485": 5120, "n02912065": 5121, "n02912319": 5122, "n02912557": 5123, "n02912894": 5124, "n02913152": 5125, "n02914991": 5126, "n02915904": 5127, "n02916065": 5128, "n02916179": 5129, "n02916350": 5130, "n02916936": 5131, "n02917067": 5132, "n02917377": 5133, "n02917521": 5134, "n02917607": 5135, "n02917742": 5136, "n02917964": 5137, "n02918112": 5138, "n02918330": 5139, "n02918455": 5140, "n02918595": 5141, "n02918831": 5142, "n02918964": 5143, "n02919148": 5144, "n02919308": 5145, "n02919414": 5146, "n02919648": 5147, "n02919792": 5148, "n02919890": 5149, "n02919976": 5150, "n02920083": 5151, "n02920164": 5152, "n02920259": 5153, "n02920369": 5154, "n02920503": 5155, "n02920658": 5156, "n02921029": 5157, "n02921195": 5158, "n02921292": 5159, "n02921406": 5160, "n02921592": 5161, "n02921756": 5162, "n02921884": 5163, "n02922159": 5164, "n02922292": 5165, "n02922461": 5166, "n02922578": 5167, "n02922798": 5168, "n02922877": 5169, "n02923129": 5170, "n02923535": 5171, "n02923682": 5172, "n02923915": 5173, "n02924116": 5174, "n02925009": 5175, "n02925107": 5176, "n02925385": 5177, "n02925519": 5178, "n02925666": 5179, "n02926426": 5180, "n02926591": 5181, "n02927053": 5182, "n02927161": 5183, "n02927764": 5184, "n02927887": 5185, "n02928049": 5186, "n02928299": 5187, "n02928413": 5188, "n02928608": 5189, "n02929184": 5190, "n02929289": 5191, "n02929462": 5192, "n02929582": 5193, "n02929923": 5194, "n02930080": 5195, "n02930214": 5196, "n02930339": 5197, "n02930645": 5198, "n02930766": 5199, "n02931013": 5200, "n02931148": 5201, "n02931294": 5202, "n02931417": 5203, "n02931836": 5204, "n02932019": 5205, "n02932400": 5206, "n02932523": 5207, "n02932693": 5208, "n02932891": 5209, "n02933112": 5210, "n02933340": 5211, "n02933462": 5212, "n02933649": 5213, "n02933750": 5214, "n02933990": 5215, "n02934168": 5216, "n02934451": 5217, "n02935017": 5218, "n02935387": 5219, "n02935490": 5220, "n02935658": 5221, "n02935891": 5222, "n02936176": 5223, "n02936281": 5224, "n02936402": 5225, "n02936570": 5226, "n02936714": 5227, "n02936921": 5228, "n02937010": 5229, "n02937336": 5230, "n02937958": 5231, "n02938218": 5232, "n02938321": 5233, "n02938886": 5234, "n02939185": 5235, "n02939763": 5236, "n02939866": 5237, "n02940289": 5238, "n02940385": 5239, "n02940570": 5240, "n02940706": 5241, "n02941095": 5242, "n02941228": 5243, "n02941845": 5244, "n02942015": 5245, "n02942147": 5246, "n02942349": 5247, "n02942460": 5248, "n02942699": 5249, "n02943241": 5250, "n02943465": 5251, "n02943686": 5252, "n02943871": 5253, "n02943964": 5254, "n02944075": 5255, "n02944146": 5256, "n02944256": 5257, "n02944459": 5258, "n02944579": 5259, "n02944826": 5260, "n02945161": 5261, "n02945813": 5262, "n02945964": 5263, "n02946127": 5264, "n02946270": 5265, "n02946348": 5266, "n02946509": 5267, "n02946753": 5268, "n02946824": 5269, "n02946921": 5270, "n02947212": 5271, "n02947660": 5272, "n02947818": 5273, "n02947977": 5274, "n02948072": 5275, "n02948293": 5276, "n02948403": 5277, "n02948557": 5278, "n02948834": 5279, "n02948942": 5280, "n02949084": 5281, "n02949202": 5282, "n02949356": 5283, "n02949542": 5284, "n02950018": 5285, "n02950120": 5286, "n02950186": 5287, "n02950256": 5288, "n02950482": 5289, "n02950632": 5290, "n02950826": 5291, "n02950943": 5292, "n02951358": 5293, "n02951585": 5294, "n02951703": 5295, "n02951843": 5296, "n02952109": 5297, "n02952237": 5298, "n02952374": 5299, "n02952485": 5300, "n02952585": 5301, "n02952674": 5302, "n02952798": 5303, "n02952935": 5304, "n02953056": 5305, "n02953197": 5306, "n02953455": 5307, "n02953552": 5308, "n02953673": 5309, "n02953850": 5310, "n02954163": 5311, "n02954340": 5312, "n02954938": 5313, "n02955065": 5314, "n02955247": 5315, "n02955540": 5316, "n02955767": 5317, "n02956393": 5318, "n02956699": 5319, "n02956795": 5320, "n02956883": 5321, "n02957008": 5322, "n02957135": 5323, "n02957252": 5324, "n02957427": 5325, "n02957755": 5326, "n02957862": 5327, "n02958343": 5328, "n02959942": 5329, "n02960352": 5330, "n02960690": 5331, "n02960903": 5332, "n02961035": 5333, "n02961225": 5334, "n02961451": 5335, "n02961544": 5336, "n02961947": 5337, "n02962061": 5338, "n02962200": 5339, "n02962414": 5340, "n02962843": 5341, "n02962938": 5342, "n02963159": 5343, "n02963302": 5344, "n02963503": 5345, "n02963692": 5346, "n02963821": 5347, "n02963987": 5348, "n02964075": 5349, "n02964196": 5350, "n02964295": 5351, "n02964634": 5352, "n02964843": 5353, "n02964934": 5354, "n02965024": 5355, "n02965122": 5356, "n02965216": 5357, "n02965300": 5358, "n02965529": 5359, "n02965783": 5360, "n02966068": 5361, "n02966193": 5362, "n02966545": 5363, "n02966687": 5364, "n02966786": 5365, "n02966942": 5366, "n02967081": 5367, "n02967170": 5368, "n02967294": 5369, "n02967407": 5370, "n02967540": 5371, "n02967626": 5372, "n02967782": 5373, "n02967991": 5374, "n02968074": 5375, "n02968210": 5376, "n02968333": 5377, "n02968473": 5378, "n02969010": 5379, "n02969163": 5380, "n02969323": 5381, "n02969527": 5382, "n02969634": 5383, "n02969886": 5384, "n02970408": 5385, "n02970534": 5386, "n02970685": 5387, "n02970849": 5388, "n02971167": 5389, "n02971356": 5390, "n02971473": 5391, "n02971579": 5392, "n02971691": 5393, "n02971940": 5394, "n02972397": 5395, "n02972714": 5396, "n02972934": 5397, "n02973017": 5398, "n02973236": 5399, "n02973805": 5400, "n02973904": 5401, "n02974003": 5402, "n02974348": 5403, "n02974454": 5404, "n02974565": 5405, "n02974697": 5406, "n02975212": 5407, "n02975589": 5408, "n02975994": 5409, "n02976123": 5410, "n02976249": 5411, "n02976350": 5412, "n02976455": 5413, "n02976552": 5414, "n02976641": 5415, "n02976815": 5416, "n02976939": 5417, "n02977058": 5418, "n02977330": 5419, "n02977438": 5420, "n02977619": 5421, "n02977936": 5422, "n02978055": 5423, "n02978205": 5424, "n02978367": 5425, "n02978478": 5426, "n02978753": 5427, "n02978881": 5428, "n02979074": 5429, "n02979186": 5430, "n02979290": 5431, "n02979399": 5432, "n02979516": 5433, "n02979836": 5434, "n02980036": 5435, "n02980203": 5436, "n02980441": 5437, "n02980625": 5438, "n02981024": 5439, "n02981198": 5440, "n02981321": 5441, "n02981565": 5442, "n02981792": 5443, "n02981911": 5444, "n02982232": 5445, "n02982416": 5446, "n02982515": 5447, "n02982599": 5448, "n02983072": 5449, "n02983189": 5450, "n02983357": 5451, "n02983507": 5452, "n02983904": 5453, "n02984061": 5454, "n02984203": 5455, "n02984469": 5456, "n02984699": 5457, "n02985137": 5458, "n02985606": 5459, "n02985828": 5460, "n02985963": 5461, "n02986066": 5462, "n02986160": 5463, "n02986348": 5464, "n02987047": 5465, "n02987379": 5466, "n02987492": 5467, "n02987706": 5468, "n02987823": 5469, "n02987950": 5470, "n02988066": 5471, "n02988156": 5472, "n02988304": 5473, "n02988486": 5474, "n02988679": 5475, "n02988963": 5476, "n02989099": 5477, "n02990373": 5478, "n02990758": 5479, "n02991048": 5480, "n02991302": 5481, "n02991847": 5482, "n02992032": 5483, "n02992211": 5484, "n02992368": 5485, "n02992529": 5486, "n02992795": 5487, "n02993194": 5488, "n02993368": 5489, "n02993546": 5490, "n02994573": 5491, "n02994743": 5492, "n02995345": 5493, "n02995871": 5494, "n02995998": 5495, "n02997391": 5496, "n02997607": 5497, "n02997910": 5498, "n02998003": 5499, "n02998107": 5500, "n02998563": 5501, "n02998696": 5502, "n02998841": 5503, "n02999138": 5504, "n02999410": 5505, "n02999936": 5506, "n03000134": 5507, "n03000247": 5508, "n03000530": 5509, "n03000684": 5510, "n03001115": 5511, "n03001282": 5512, "n03001540": 5513, "n03001627": 5514, "n03002096": 5515, "n03002210": 5516, "n03002341": 5517, "n03002555": 5518, "n03002711": 5519, "n03002816": 5520, "n03002948": 5521, "n03003091": 5522, "n03003633": 5523, "n03004275": 5524, "n03004409": 5525, "n03004531": 5526, "n03004620": 5527, "n03004713": 5528, "n03004824": 5529, "n03005033": 5530, "n03005147": 5531, "n03005285": 5532, "n03005515": 5533, "n03005619": 5534, "n03006626": 5535, "n03006788": 5536, "n03006903": 5537, "n03007130": 5538, "n03007297": 5539, "n03007444": 5540, "n03007591": 5541, "n03008177": 5542, "n03008817": 5543, "n03008976": 5544, "n03009111": 5545, "n03009269": 5546, "n03009794": 5547, "n03010473": 5548, "n03010656": 5549, "n03010795": 5550, "n03010915": 5551, "n03011018": 5552, "n03011355": 5553, "n03011741": 5554, "n03012013": 5555, "n03012159": 5556, "n03012373": 5557, "n03012499": 5558, "n03012644": 5559, "n03012734": 5560, "n03012897": 5561, "n03013006": 5562, "n03013438": 5563, "n03013580": 5564, "n03013850": 5565, "n03014440": 5566, "n03014705": 5567, "n03015149": 5568, "n03015254": 5569, "n03015478": 5570, "n03015631": 5571, "n03015851": 5572, "n03016209": 5573, "n03016389": 5574, "n03016609": 5575, "n03016737": 5576, "n03016868": 5577, "n03016953": 5578, "n03017070": 5579, "n03017168": 5580, "n03017698": 5581, "n03017835": 5582, "n03018209": 5583, "n03018349": 5584, "n03018614": 5585, "n03018712": 5586, "n03018848": 5587, "n03019198": 5588, "n03019304": 5589, "n03019434": 5590, "n03019685": 5591, "n03019806": 5592, "n03019938": 5593, "n03020034": 5594, "n03020416": 5595, "n03020692": 5596, "n03021228": 5597, "n03024064": 5598, "n03024233": 5599, "n03024333": 5600, "n03024518": 5601, "n03025070": 5602, "n03025165": 5603, "n03025250": 5604, "n03025886": 5605, "n03026506": 5606, "n03026907": 5607, "n03027001": 5608, "n03027108": 5609, "n03027250": 5610, "n03027505": 5611, "n03027625": 5612, "n03028079": 5613, "n03028596": 5614, "n03028785": 5615, "n03029066": 5616, "n03029197": 5617, "n03029296": 5618, "n03029445": 5619, "n03029925": 5620, "n03030262": 5621, "n03030353": 5622, "n03030557": 5623, "n03030880": 5624, "n03031012": 5625, "n03031152": 5626, "n03031422": 5627, "n03031756": 5628, "n03032252": 5629, "n03032453": 5630, "n03032811": 5631, "n03033267": 5632, "n03033362": 5633, "n03033986": 5634, "n03034244": 5635, "n03034405": 5636, "n03034516": 5637, "n03034663": 5638, "n03035252": 5639, "n03035510": 5640, "n03035715": 5641, "n03035832": 5642, "n03036022": 5643, "n03036149": 5644, "n03036244": 5645, "n03036341": 5646, "n03036469": 5647, "n03036701": 5648, "n03036866": 5649, "n03037108": 5650, "n03037228": 5651, "n03037404": 5652, "n03037590": 5653, "n03037709": 5654, "n03038041": 5655, "n03038281": 5656, "n03038480": 5657, "n03038685": 5658, "n03038870": 5659, "n03039015": 5660, "n03039259": 5661, "n03039353": 5662, "n03039493": 5663, "n03039827": 5664, "n03039947": 5665, "n03040229": 5666, "n03040376": 5667, "n03040836": 5668, "n03041114": 5669, "n03041265": 5670, "n03041449": 5671, "n03041632": 5672, "n03041810": 5673, "n03042139": 5674, "n03042384": 5675, "n03042490": 5676, "n03042697": 5677, "n03042829": 5678, "n03042984": 5679, "n03043173": 5680, "n03043274": 5681, "n03043423": 5682, "n03043693": 5683, "n03043798": 5684, "n03043958": 5685, "n03044671": 5686, "n03044801": 5687, "n03044934": 5688, "n03045074": 5689, "n03045228": 5690, "n03045337": 5691, "n03045698": 5692, "n03045800": 5693, "n03046029": 5694, "n03046133": 5695, "n03046257": 5696, "n03046802": 5697, "n03046921": 5698, "n03047052": 5699, "n03047171": 5700, "n03047690": 5701, "n03047799": 5702, "n03047941": 5703, "n03048883": 5704, "n03049066": 5705, "n03049326": 5706, "n03049457": 5707, "n03049782": 5708, "n03049924": 5709, "n03050026": 5710, "n03050453": 5711, "n03050546": 5712, "n03050655": 5713, "n03050864": 5714, "n03051041": 5715, "n03051249": 5716, "n03051396": 5717, "n03051540": 5718, "n03052464": 5719, "n03052917": 5720, "n03053047": 5721, "n03053976": 5722, "n03054491": 5723, "n03054605": 5724, "n03054901": 5725, "n03055159": 5726, "n03055418": 5727, "n03055670": 5728, "n03055857": 5729, "n03056097": 5730, "n03056215": 5731, "n03056288": 5732, "n03056493": 5733, "n03056583": 5734, "n03056873": 5735, "n03057021": 5736, "n03057541": 5737, "n03057636": 5738, "n03057724": 5739, "n03057841": 5740, "n03057920": 5741, "n03058107": 5742, "n03058603": 5743, "n03058949": 5744, "n03059103": 5745, "n03059236": 5746, "n03059366": 5747, "n03059685": 5748, "n03059934": 5749, "n03060728": 5750, "n03061050": 5751, "n03061211": 5752, "n03061345": 5753, "n03061505": 5754, "n03061674": 5755, "n03061819": 5756, "n03061893": 5757, "n03062015": 5758, "n03062122": 5759, "n03062245": 5760, "n03062336": 5761, "n03062651": 5762, "n03062798": 5763, "n03062985": 5764, "n03063073": 5765, "n03063199": 5766, "n03063338": 5767, "n03063485": 5768, "n03063599": 5769, "n03063689": 5770, "n03063834": 5771, "n03063968": 5772, "n03064250": 5773, "n03064350": 5774, "n03064562": 5775, "n03064758": 5776, "n03064935": 5777, "n03065243": 5778, "n03065424": 5779, "n03065708": 5780, "n03066232": 5781, "n03066359": 5782, "n03066464": 5783, "n03066849": 5784, "n03067093": 5785, "n03067212": 5786, "n03067339": 5787, "n03067518": 5788, "n03068181": 5789, "n03068998": 5790, "n03069752": 5791, "n03070059": 5792, "n03070193": 5793, "n03070396": 5794, "n03070587": 5795, "n03070854": 5796, "n03071021": 5797, "n03071160": 5798, "n03071288": 5799, "n03071552": 5800, "n03072056": 5801, "n03072201": 5802, "n03072440": 5803, "n03072682": 5804, "n03073296": 5805, "n03073384": 5806, "n03073545": 5807, "n03073694": 5808, "n03073977": 5809, "n03074380": 5810, "n03074855": 5811, "n03075097": 5812, "n03075248": 5813, "n03075370": 5814, "n03075500": 5815, "n03075634": 5816, "n03075768": 5817, "n03075946": 5818, "n03076411": 5819, "n03076623": 5820, "n03076708": 5821, "n03077442": 5822, "n03077616": 5823, "n03077741": 5824, "n03078287": 5825, "n03078506": 5826, "n03078670": 5827, "n03078802": 5828, "n03078995": 5829, "n03079136": 5830, "n03079230": 5831, "n03079494": 5832, "n03079616": 5833, "n03079741": 5834, "n03080309": 5835, "n03080497": 5836, "n03080633": 5837, "n03080731": 5838, "n03080904": 5839, "n03081859": 5840, "n03081986": 5841, "n03082127": 5842, "n03082280": 5843, "n03082450": 5844, "n03082656": 5845, "n03082807": 5846, "n03082979": 5847, "n03084420": 5848, "n03084834": 5849, "n03085013": 5850, "n03085219": 5851, "n03085333": 5852, "n03085602": 5853, "n03085781": 5854, "n03085915": 5855, "n03086183": 5856, "n03086457": 5857, "n03086580": 5858, "n03086670": 5859, "n03086868": 5860, "n03087069": 5861, "n03087245": 5862, "n03087366": 5863, "n03087521": 5864, "n03087643": 5865, "n03087816": 5866, "n03088389": 5867, "n03088580": 5868, "n03088707": 5869, "n03089477": 5870, "n03089624": 5871, "n03089753": 5872, "n03089879": 5873, "n03090000": 5874, "n03090172": 5875, "n03090437": 5876, "n03090710": 5877, "n03090856": 5878, "n03091044": 5879, "n03091223": 5880, "n03091374": 5881, "n03091907": 5882, "n03092053": 5883, "n03092166": 5884, "n03092314": 5885, "n03092476": 5886, "n03092656": 5887, "n03092883": 5888, "n03093427": 5889, "n03093792": 5890, "n03094159": 5891, "n03094503": 5892, "n03095699": 5893, "n03095965": 5894, "n03096439": 5895, "n03096960": 5896, "n03097362": 5897, "n03097535": 5898, "n03097673": 5899, "n03098140": 5900, "n03098515": 5901, "n03098688": 5902, "n03098806": 5903, "n03098959": 5904, "n03099147": 5905, "n03099274": 5906, "n03099454": 5907, "n03099622": 5908, "n03099771": 5909, "n03099945": 5910, "n03100240": 5911, "n03100346": 5912, "n03100490": 5913, "n03100897": 5914, "n03101156": 5915, "n03101302": 5916, "n03101375": 5917, "n03101517": 5918, "n03101664": 5919, "n03101796": 5920, "n03101986": 5921, "n03102371": 5922, "n03102516": 5923, "n03102654": 5924, "n03102859": 5925, "n03103128": 5926, "n03103396": 5927, "n03103563": 5928, "n03103904": 5929, "n03104019": 5930, "n03104512": 5931, "n03105088": 5932, "n03105214": 5933, "n03105306": 5934, "n03105467": 5935, "n03105645": 5936, "n03105810": 5937, "n03105974": 5938, "n03106722": 5939, "n03106898": 5940, "n03107046": 5941, "n03107488": 5942, "n03107716": 5943, "n03108455": 5944, "n03108624": 5945, "n03108759": 5946, "n03108853": 5947, "n03109033": 5948, "n03109150": 5949, "n03109253": 5950, "n03109693": 5951, "n03109881": 5952, "n03110202": 5953, "n03110669": 5954, "n03111041": 5955, "n03111177": 5956, "n03111296": 5957, "n03111690": 5958, "n03112240": 5959, "n03112719": 5960, "n03112869": 5961, "n03113152": 5962, "n03113505": 5963, "n03113657": 5964, "n03113835": 5965, "n03114041": 5966, "n03114236": 5967, "n03114379": 5968, "n03114504": 5969, "n03114743": 5970, "n03114839": 5971, "n03115014": 5972, "n03115180": 5973, "n03115400": 5974, "n03115663": 5975, "n03115762": 5976, "n03115897": 5977, "n03116008": 5978, "n03116163": 5979, "n03116530": 5980, "n03116767": 5981, "n03117199": 5982, "n03117642": 5983, "n03118346": 5984, "n03118969": 5985, "n03119203": 5986, "n03119396": 5987, "n03119510": 5988, "n03120198": 5989, "n03120491": 5990, "n03120778": 5991, "n03121040": 5992, "n03121190": 5993, "n03121298": 5994, "n03121431": 5995, "n03121897": 5996, "n03122073": 5997, "n03122202": 5998, "n03122295": 5999, "n03122748": 6000, "n03123553": 6001, "n03123666": 6002, "n03123809": 6003, "n03123917": 6004, "n03124043": 6005, "n03124170": 6006, "n03124313": 6007, "n03124474": 6008, "n03124590": 6009, "n03125057": 6010, "n03125588": 6011, "n03125729": 6012, "n03125870": 6013, "n03126090": 6014, "n03126385": 6015, "n03126580": 6016, "n03126707": 6017, "n03126927": 6018, "n03127024": 6019, "n03127203": 6020, "n03127408": 6021, "n03127531": 6022, "n03127747": 6023, "n03127925": 6024, "n03128085": 6025, "n03128248": 6026, "n03128427": 6027, "n03128519": 6028, "n03129001": 6029, "n03129471": 6030, "n03129636": 6031, "n03129753": 6032, "n03129848": 6033, "n03130066": 6034, "n03130233": 6035, "n03130563": 6036, "n03130761": 6037, "n03130866": 6038, "n03131193": 6039, "n03131574": 6040, "n03131669": 6041, "n03131967": 6042, "n03132076": 6043, "n03132261": 6044, "n03132438": 6045, "n03132666": 6046, "n03132776": 6047, "n03133050": 6048, "n03133415": 6049, "n03133878": 6050, "n03134118": 6051, "n03134232": 6052, "n03134394": 6053, "n03134739": 6054, "n03134853": 6055, "n03135030": 6056, "n03135532": 6057, "n03135656": 6058, "n03135788": 6059, "n03135917": 6060, "n03136051": 6061, "n03136254": 6062, "n03136369": 6063, "n03136504": 6064, "n03137473": 6065, "n03137579": 6066, "n03138128": 6067, "n03138217": 6068, "n03138344": 6069, "n03138669": 6070, "n03139089": 6071, "n03139464": 6072, "n03139640": 6073, "n03139998": 6074, "n03140126": 6075, "n03140292": 6076, "n03140431": 6077, "n03140546": 6078, "n03140652": 6079, "n03140771": 6080, "n03140900": 6081, "n03141065": 6082, "n03141327": 6083, "n03141455": 6084, "n03141612": 6085, "n03141702": 6086, "n03141823": 6087, "n03142099": 6088, "n03142205": 6089, "n03142325": 6090, "n03142431": 6091, "n03142679": 6092, "n03143400": 6093, "n03143572": 6094, "n03143754": 6095, "n03144156": 6096, "n03144873": 6097, "n03144982": 6098, "n03145147": 6099, "n03145277": 6100, "n03145384": 6101, "n03145522": 6102, "n03145719": 6103, "n03145843": 6104, "n03146219": 6105, "n03146342": 6106, "n03146449": 6107, "n03146560": 6108, "n03146687": 6109, "n03146777": 6110, "n03146846": 6111, "n03147084": 6112, "n03147156": 6113, "n03147280": 6114, "n03147509": 6115, "n03148324": 6116, "n03148518": 6117, "n03148727": 6118, "n03148808": 6119, "n03149135": 6120, "n03149401": 6121, "n03149686": 6122, "n03149810": 6123, "n03150232": 6124, "n03150511": 6125, "n03150661": 6126, "n03150795": 6127, "n03151077": 6128, "n03152303": 6129, "n03152951": 6130, "n03153246": 6131, "n03153585": 6132, "n03153948": 6133, "n03154073": 6134, "n03154316": 6135, "n03154446": 6136, "n03154616": 6137, "n03154745": 6138, "n03154895": 6139, "n03155178": 6140, "n03155502": 6141, "n03155915": 6142, "n03156071": 6143, "n03156279": 6144, "n03156405": 6145, "n03156767": 6146, "n03157348": 6147, "n03158186": 6148, "n03158414": 6149, "n03158668": 6150, "n03158796": 6151, "n03158885": 6152, "n03159535": 6153, "n03159640": 6154, "n03160001": 6155, "n03160186": 6156, "n03160309": 6157, "n03160740": 6158, "n03161016": 6159, "n03161450": 6160, "n03161893": 6161, "n03162297": 6162, "n03162460": 6163, "n03162556": 6164, "n03162714": 6165, "n03162818": 6166, "n03163222": 6167, "n03163381": 6168, "n03163488": 6169, "n03163798": 6170, "n03163973": 6171, "n03164192": 6172, "n03164344": 6173, "n03164605": 6174, "n03164722": 6175, "n03164929": 6176, "n03165096": 6177, "n03165211": 6178, "n03165466": 6179, "n03165616": 6180, "n03165823": 6181, "n03165955": 6182, "n03166120": 6183, "n03166514": 6184, "n03166600": 6185, "n03166685": 6186, "n03166809": 6187, "n03166951": 6188, "n03167153": 6189, "n03167978": 6190, "n03168107": 6191, "n03168217": 6192, "n03168543": 6193, "n03168663": 6194, "n03168774": 6195, "n03168933": 6196, "n03169063": 6197, "n03169176": 6198, "n03170292": 6199, "n03170459": 6200, "n03170635": 6201, "n03170872": 6202, "n03171228": 6203, "n03171356": 6204, "n03171635": 6205, "n03171910": 6206, "n03172038": 6207, "n03172738": 6208, "n03172965": 6209, "n03173270": 6210, "n03173387": 6211, "n03173929": 6212, "n03174079": 6213, "n03174450": 6214, "n03174731": 6215, "n03175081": 6216, "n03175189": 6217, "n03175301": 6218, "n03175457": 6219, "n03175604": 6220, "n03175843": 6221, "n03175983": 6222, "n03176238": 6223, "n03176386": 6224, "n03176594": 6225, "n03176763": 6226, "n03177059": 6227, "n03177165": 6228, "n03177708": 6229, "n03178000": 6230, "n03178173": 6231, "n03178430": 6232, "n03178538": 6233, "n03178674": 6234, "n03179701": 6235, "n03179910": 6236, "n03180011": 6237, "n03180384": 6238, "n03180504": 6239, "n03180732": 6240, "n03180865": 6241, "n03180969": 6242, "n03181293": 6243, "n03181667": 6244, "n03182140": 6245, "n03182232": 6246, "n03182912": 6247, "n03183080": 6248, "n03185868": 6249, "n03186199": 6250, "n03186285": 6251, "n03186818": 6252, "n03187037": 6253, "n03187153": 6254, "n03187268": 6255, "n03187595": 6256, "n03187751": 6257, "n03188290": 6258, "n03188531": 6259, "n03188725": 6260, "n03188871": 6261, "n03189083": 6262, "n03189311": 6263, "n03189818": 6264, "n03190458": 6265, "n03191286": 6266, "n03191451": 6267, "n03191561": 6268, "n03191776": 6269, "n03192543": 6270, "n03192907": 6271, "n03193107": 6272, "n03193260": 6273, "n03193423": 6274, "n03193597": 6275, "n03193754": 6276, "n03194170": 6277, "n03194297": 6278, "n03194812": 6279, "n03194992": 6280, "n03195332": 6281, "n03195485": 6282, "n03195799": 6283, "n03195959": 6284, "n03196062": 6285, "n03196217": 6286, "n03196324": 6287, "n03196598": 6288, "n03196990": 6289, "n03197201": 6290, "n03197337": 6291, "n03197446": 6292, "n03198223": 6293, "n03198500": 6294, "n03199358": 6295, "n03199488": 6296, "n03199647": 6297, "n03199775": 6298, "n03199901": 6299, "n03200231": 6300, "n03200357": 6301, "n03200539": 6302, "n03200701": 6303, "n03200906": 6304, "n03201035": 6305, "n03201208": 6306, "n03201529": 6307, "n03201638": 6308, "n03201776": 6309, "n03201895": 6310, "n03201996": 6311, "n03202354": 6312, "n03202481": 6313, "n03202760": 6314, "n03202940": 6315, "n03203089": 6316, "n03203806": 6317, "n03204134": 6318, "n03204306": 6319, "n03204436": 6320, "n03204558": 6321, "n03204955": 6322, "n03205143": 6323, "n03205304": 6324, "n03205458": 6325, "n03205574": 6326, "n03205669": 6327, "n03205903": 6328, "n03206023": 6329, "n03206158": 6330, "n03206282": 6331, "n03206405": 6332, "n03206602": 6333, "n03206718": 6334, "n03206908": 6335, "n03207305": 6336, "n03207548": 6337, "n03207630": 6338, "n03207743": 6339, "n03207835": 6340, "n03207941": 6341, "n03208556": 6342, "n03208938": 6343, "n03209359": 6344, "n03209477": 6345, "n03209666": 6346, "n03209910": 6347, "n03210245": 6348, "n03210372": 6349, "n03210552": 6350, "n03210683": 6351, "n03211117": 6352, "n03211413": 6353, "n03211616": 6354, "n03211789": 6355, "n03212114": 6356, "n03212247": 6357, "n03212406": 6358, "n03212811": 6359, "n03213014": 6360, "n03213361": 6361, "n03213538": 6362, "n03213715": 6363, "n03213826": 6364, "n03214253": 6365, "n03214450": 6366, "n03214582": 6367, "n03214966": 6368, "n03215076": 6369, "n03215191": 6370, "n03215337": 6371, "n03215508": 6372, "n03215749": 6373, "n03215930": 6374, "n03216199": 6375, "n03216402": 6376, "n03216562": 6377, "n03216710": 6378, "n03216828": 6379, "n03217653": 6380, "n03217739": 6381, "n03217889": 6382, "n03218198": 6383, "n03218446": 6384, "n03219010": 6385, "n03219135": 6386, "n03219483": 6387, "n03219612": 6388, "n03219859": 6389, "n03219966": 6390, "n03220095": 6391, "n03220237": 6392, "n03220513": 6393, "n03220692": 6394, "n03221059": 6395, "n03221351": 6396, "n03221540": 6397, "n03221720": 6398, "n03222176": 6399, "n03222318": 6400, "n03222516": 6401, "n03222722": 6402, "n03222857": 6403, "n03223162": 6404, "n03223299": 6405, "n03223441": 6406, "n03223553": 6407, "n03223686": 6408, "n03223923": 6409, "n03224490": 6410, "n03224603": 6411, "n03224753": 6412, "n03224893": 6413, "n03225108": 6414, "n03225458": 6415, "n03225616": 6416, "n03225777": 6417, "n03225988": 6418, "n03226090": 6419, "n03226254": 6420, "n03226375": 6421, "n03226538": 6422, "n03226880": 6423, "n03227010": 6424, "n03227184": 6425, "n03227317": 6426, "n03227721": 6427, "n03227856": 6428, "n03228016": 6429, "n03228254": 6430, "n03228365": 6431, "n03228533": 6432, "n03228692": 6433, "n03228796": 6434, "n03228967": 6435, "n03229115": 6436, "n03229244": 6437, "n03229526": 6438, "n03231160": 6439, "n03231368": 6440, "n03231819": 6441, "n03232309": 6442, "n03232417": 6443, "n03232543": 6444, "n03232815": 6445, "n03232923": 6446, "n03233123": 6447, "n03233624": 6448, "n03233744": 6449, "n03233905": 6450, "n03234164": 6451, "n03234952": 6452, "n03235042": 6453, "n03235180": 6454, "n03235327": 6455, "n03235796": 6456, "n03235979": 6457, "n03236093": 6458, "n03236217": 6459, "n03236423": 6460, "n03236580": 6461, "n03236735": 6462, "n03237212": 6463, "n03237340": 6464, "n03237416": 6465, "n03237639": 6466, "n03237839": 6467, "n03237992": 6468, "n03238131": 6469, "n03238286": 6470, "n03238586": 6471, "n03238762": 6472, "n03238879": 6473, "n03239054": 6474, "n03239259": 6475, "n03239607": 6476, "n03239726": 6477, "n03240140": 6478, "n03240683": 6479, "n03240892": 6480, "n03241093": 6481, "n03241335": 6482, "n03241496": 6483, "n03241903": 6484, "n03242120": 6485, "n03242264": 6486, "n03242390": 6487, "n03242506": 6488, "n03242995": 6489, "n03243218": 6490, "n03243625": 6491, "n03244047": 6492, "n03244231": 6493, "n03244388": 6494, "n03244775": 6495, "n03244919": 6496, "n03245271": 6497, "n03245421": 6498, "n03245724": 6499, "n03245889": 6500, "n03246197": 6501, "n03246312": 6502, "n03246454": 6503, "n03246653": 6504, "n03246933": 6505, "n03247083": 6506, "n03247351": 6507, "n03247495": 6508, "n03248835": 6509, "n03249342": 6510, "n03249569": 6511, "n03249956": 6512, "n03250089": 6513, "n03250279": 6514, "n03250405": 6515, "n03250588": 6516, "n03250847": 6517, "n03250952": 6518, "n03251100": 6519, "n03251280": 6520, "n03251533": 6521, "n03251766": 6522, "n03251932": 6523, "n03252231": 6524, "n03252324": 6525, "n03252422": 6526, "n03252637": 6527, "n03252787": 6528, "n03253071": 6529, "n03253187": 6530, "n03253279": 6531, "n03253714": 6532, "n03253796": 6533, "n03253886": 6534, "n03254046": 6535, "n03254189": 6536, "n03254374": 6537, "n03254625": 6538, "n03254737": 6539, "n03254862": 6540, "n03255030": 6541, "n03255167": 6542, "n03255322": 6543, "n03255488": 6544, "n03255899": 6545, "n03256032": 6546, "n03256166": 6547, "n03256472": 6548, "n03256631": 6549, "n03256788": 6550, "n03256928": 6551, "n03257065": 6552, "n03257210": 6553, "n03257586": 6554, "n03258192": 6555, "n03258330": 6556, "n03258456": 6557, "n03258577": 6558, "n03258905": 6559, "n03259009": 6560, "n03259280": 6561, "n03259401": 6562, "n03259505": 6563, "n03260206": 6564, "n03260504": 6565, "n03260733": 6566, "n03260849": 6567, "n03261019": 6568, "n03261263": 6569, "n03261395": 6570, "n03261603": 6571, "n03261776": 6572, "n03262072": 6573, "n03262248": 6574, "n03262519": 6575, "n03262717": 6576, "n03262809": 6577, "n03262932": 6578, "n03263076": 6579, "n03263338": 6580, "n03263640": 6581, "n03263758": 6582, "n03264906": 6583, "n03265032": 6584, "n03265754": 6585, "n03266195": 6586, "n03266371": 6587, "n03266620": 6588, "n03266749": 6589, "n03267113": 6590, "n03267468": 6591, "n03267696": 6592, "n03267821": 6593, "n03268142": 6594, "n03268311": 6595, "n03268645": 6596, "n03268790": 6597, "n03268918": 6598, "n03269073": 6599, "n03269203": 6600, "n03269401": 6601, "n03270165": 6602, "n03270695": 6603, "n03270854": 6604, "n03271030": 6605, "n03271260": 6606, "n03271376": 6607, "n03271574": 6608, "n03271765": 6609, "n03271865": 6610, "n03272010": 6611, "n03272125": 6612, "n03272239": 6613, "n03272383": 6614, "n03272562": 6615, "n03272810": 6616, "n03272940": 6617, "n03273061": 6618, "n03273551": 6619, "n03273740": 6620, "n03273913": 6621, "n03274265": 6622, "n03274435": 6623, "n03274561": 6624, "n03274796": 6625, "n03275125": 6626, "n03275311": 6627, "n03275566": 6628, "n03275681": 6629, "n03275864": 6630, "n03276179": 6631, "n03276696": 6632, "n03276839": 6633, "n03277004": 6634, "n03277149": 6635, "n03277459": 6636, "n03277602": 6637, "n03277771": 6638, "n03278248": 6639, "n03278914": 6640, "n03279153": 6641, "n03279364": 6642, "n03279508": 6643, "n03279804": 6644, "n03279918": 6645, "n03280216": 6646, "n03280394": 6647, "n03280644": 6648, "n03281145": 6649, "n03281524": 6650, "n03281673": 6651, "n03282060": 6652, "n03282295": 6653, "n03282401": 6654, "n03283221": 6655, "n03283413": 6656, "n03283827": 6657, "n03284308": 6658, "n03284482": 6659, "n03284743": 6660, "n03284886": 6661, "n03284981": 6662, "n03285578": 6663, "n03285730": 6664, "n03285912": 6665, "n03286572": 6666, "n03287351": 6667, "n03287733": 6668, "n03288003": 6669, "n03288500": 6670, "n03288643": 6671, "n03288742": 6672, "n03288886": 6673, "n03289660": 6674, "n03289985": 6675, "n03290096": 6676, "n03290195": 6677, "n03290653": 6678, "n03291413": 6679, "n03291551": 6680, "n03291741": 6681, "n03291819": 6682, "n03291963": 6683, "n03292085": 6684, "n03292362": 6685, "n03292475": 6686, "n03292603": 6687, "n03292736": 6688, "n03292960": 6689, "n03293095": 6690, "n03293741": 6691, "n03293863": 6692, "n03294048": 6693, "n03294604": 6694, "n03294833": 6695, "n03295012": 6696, "n03295140": 6697, "n03295246": 6698, "n03295928": 6699, "n03296081": 6700, "n03296217": 6701, "n03296328": 6702, "n03296478": 6703, "n03296963": 6704, "n03297103": 6705, "n03297226": 6706, "n03297495": 6707, "n03297644": 6708, "n03297735": 6709, "n03298089": 6710, "n03298352": 6711, "n03298716": 6712, "n03298858": 6713, "n03299406": 6714, "n03300216": 6715, "n03300443": 6716, "n03301175": 6717, "n03301291": 6718, "n03301389": 6719, "n03301568": 6720, "n03301833": 6721, "n03301940": 6722, "n03302671": 6723, "n03302790": 6724, "n03302938": 6725, "n03303217": 6726, "n03303669": 6727, "n03303831": 6728, "n03304197": 6729, "n03304323": 6730, "n03304465": 6731, "n03305300": 6732, "n03305522": 6733, "n03305953": 6734, "n03306385": 6735, "n03306869": 6736, "n03307037": 6737, "n03307573": 6738, "n03307792": 6739, "n03308152": 6740, "n03308481": 6741, "n03308614": 6742, "n03309110": 6743, "n03309356": 6744, "n03309465": 6745, "n03309687": 6746, "n03309808": 6747, "n03313333": 6748, "n03314227": 6749, "n03314378": 6750, "n03314608": 6751, "n03314780": 6752, "n03314884": 6753, "n03315644": 6754, "n03315805": 6755, "n03315990": 6756, "n03316105": 6757, "n03316406": 6758, "n03316873": 6759, "n03317233": 6760, "n03317510": 6761, "n03317673": 6762, "n03317788": 6763, "n03317889": 6764, "n03318136": 6765, "n03318294": 6766, "n03318865": 6767, "n03318983": 6768, "n03319167": 6769, "n03319457": 6770, "n03319576": 6771, "n03319745": 6772, "n03320046": 6773, "n03320262": 6774, "n03320421": 6775, "n03320519": 6776, "n03320845": 6777, "n03320959": 6778, "n03321103": 6779, "n03321419": 6780, "n03321563": 6781, "n03321843": 6782, "n03321954": 6783, "n03322570": 6784, "n03322704": 6785, "n03322836": 6786, "n03322940": 6787, "n03323096": 6788, "n03323211": 6789, "n03323319": 6790, "n03323703": 6791, "n03324629": 6792, "n03324814": 6793, "n03324928": 6794, "n03325088": 6795, "n03325288": 6796, "n03325403": 6797, "n03325584": 6798, "n03325691": 6799, "n03325941": 6800, "n03326073": 6801, "n03326371": 6802, "n03326475": 6803, "n03326660": 6804, "n03326795": 6805, "n03326948": 6806, "n03327133": 6807, "n03327234": 6808, "n03327553": 6809, "n03327691": 6810, "n03327841": 6811, "n03328201": 6812, "n03329302": 6813, "n03329536": 6814, "n03329663": 6815, "n03330002": 6816, "n03330665": 6817, "n03330792": 6818, "n03330947": 6819, "n03331077": 6820, "n03331244": 6821, "n03331599": 6822, "n03332005": 6823, "n03332173": 6824, "n03332271": 6825, "n03332393": 6826, "n03332591": 6827, "n03332784": 6828, "n03332989": 6829, "n03333129": 6830, "n03333252": 6831, "n03333349": 6832, "n03333610": 6833, "n03333711": 6834, "n03333851": 6835, "n03334017": 6836, "n03334291": 6837, "n03334382": 6838, "n03334492": 6839, "n03334912": 6840, "n03335030": 6841, "n03335333": 6842, "n03335461": 6843, "n03335846": 6844, "n03336168": 6845, "n03336282": 6846, "n03336575": 6847, "n03336742": 6848, "n03336839": 6849, "n03337140": 6850, "n03337383": 6851, "n03337494": 6852, "n03337822": 6853, "n03338287": 6854, "n03338821": 6855, "n03339296": 6856, "n03339529": 6857, "n03339643": 6858, "n03340009": 6859, "n03340723": 6860, "n03340923": 6861, "n03341035": 6862, "n03341153": 6863, "n03341297": 6864, "n03341606": 6865, "n03342015": 6866, "n03342127": 6867, "n03342262": 6868, "n03342432": 6869, "n03342657": 6870, "n03342863": 6871, "n03342961": 6872, "n03343047": 6873, "n03343234": 6874, "n03343354": 6875, "n03343560": 6876, "n03343737": 6877, "n03343853": 6878, "n03344305": 6879, "n03344393": 6880, "n03344509": 6881, "n03344642": 6882, "n03344784": 6883, "n03344935": 6884, "n03345487": 6885, "n03345837": 6886, "n03346135": 6887, "n03346289": 6888, "n03346455": 6889, "n03347037": 6890, "n03347472": 6891, "n03347617": 6892, "n03348142": 6893, "n03348868": 6894, "n03349020": 6895, "n03349296": 6896, "n03349367": 6897, "n03349469": 6898, "n03349599": 6899, "n03349771": 6900, "n03349892": 6901, "n03350204": 6902, "n03350352": 6903, "n03350456": 6904, "n03350602": 6905, "n03351151": 6906, "n03351262": 6907, "n03351434": 6908, "n03351979": 6909, "n03352232": 6910, "n03352366": 6911, "n03352628": 6912, "n03352961": 6913, "n03353281": 6914, "n03353951": 6915, "n03354207": 6916, "n03354903": 6917, "n03355468": 6918, "n03355768": 6919, "n03355925": 6920, "n03356038": 6921, "n03356279": 6922, "n03356446": 6923, "n03356559": 6924, "n03356858": 6925, "n03356982": 6926, "n03357081": 6927, "n03357267": 6928, "n03357716": 6929, "n03358172": 6930, "n03358380": 6931, "n03358726": 6932, "n03358841": 6933, "n03359137": 6934, "n03359285": 6935, "n03359436": 6936, "n03359566": 6937, "n03360133": 6938, "n03360300": 6939, "n03360431": 6940, "n03360622": 6941, "n03360731": 6942, "n03361109": 6943, "n03361297": 6944, "n03361380": 6945, "n03361550": 6946, "n03361683": 6947, "n03362639": 6948, "n03362771": 6949, "n03362890": 6950, "n03363363": 6951, "n03363549": 6952, "n03363749": 6953, "n03364008": 6954, "n03364156": 6955, "n03364599": 6956, "n03364937": 6957, "n03365231": 6958, "n03365374": 6959, "n03365592": 6960, "n03365991": 6961, "n03366464": 6962, "n03366721": 6963, "n03366823": 6964, "n03366974": 6965, "n03367059": 6966, "n03367321": 6967, "n03367410": 6968, "n03367545": 6969, "n03367875": 6970, "n03367969": 6971, "n03368048": 6972, "n03368352": 6973, "n03369276": 6974, "n03369407": 6975, "n03369512": 6976, "n03369866": 6977, "n03370387": 6978, "n03370646": 6979, "n03371875": 6980, "n03372029": 6981, "n03372549": 6982, "n03372822": 6983, "n03372933": 6984, "n03373237": 6985, "n03373611": 6986, "n03373943": 6987, "n03374102": 6988, "n03374282": 6989, "n03374372": 6990, "n03374473": 6991, "n03374570": 6992, "n03374649": 6993, "n03374838": 6994, "n03375171": 6995, "n03375329": 6996, "n03375575": 6997, "n03376159": 6998, "n03376279": 6999, "n03376595": 7000, "n03376771": 7001, "n03376938": 7002, "n03378005": 7003, "n03378174": 7004, "n03378342": 7005, "n03378442": 7006, "n03378593": 7007, "n03378765": 7008, "n03379051": 7009, "n03379204": 7010, "n03379343": 7011, "n03379719": 7012, "n03379828": 7013, "n03379989": 7014, "n03380301": 7015, "n03380647": 7016, "n03380724": 7017, "n03380867": 7018, "n03381126": 7019, "n03381231": 7020, "n03381450": 7021, "n03381565": 7022, "n03381776": 7023, "n03382104": 7024, "n03382292": 7025, "n03382413": 7026, "n03382533": 7027, "n03382708": 7028, "n03382856": 7029, "n03382969": 7030, "n03383099": 7031, "n03383211": 7032, "n03383378": 7033, "n03383468": 7034, "n03383562": 7035, "n03383821": 7036, "n03384167": 7037, "n03384352": 7038, "n03384891": 7039, "n03385295": 7040, "n03385557": 7041, "n03386011": 7042, "n03386343": 7043, "n03386544": 7044, "n03386726": 7045, "n03386870": 7046, "n03387323": 7047, "n03387653": 7048, "n03388043": 7049, "n03388183": 7050, "n03388323": 7051, "n03388549": 7052, "n03388711": 7053, "n03388990": 7054, "n03389611": 7055, "n03389761": 7056, "n03389889": 7057, "n03389983": 7058, "n03390075": 7059, "n03390327": 7060, "n03390673": 7061, "n03390786": 7062, "n03390983": 7063, "n03391301": 7064, "n03391613": 7065, "n03391770": 7066, "n03392648": 7067, "n03392741": 7068, "n03393017": 7069, "n03393199": 7070, "n03393324": 7071, "n03393761": 7072, "n03393912": 7073, "n03394149": 7074, "n03394272": 7075, "n03394480": 7076, "n03394649": 7077, "n03394916": 7078, "n03395256": 7079, "n03395401": 7080, "n03395514": 7081, "n03395859": 7082, "n03396074": 7083, "n03396580": 7084, "n03396654": 7085, "n03396997": 7086, "n03397087": 7087, "n03397266": 7088, "n03397412": 7089, "n03397532": 7090, "n03397947": 7091, "n03398153": 7092, "n03398228": 7093, "n03399579": 7094, "n03399677": 7095, "n03399761": 7096, "n03399971": 7097, "n03400231": 7098, "n03400972": 7099, "n03401129": 7100, "n03401279": 7101, "n03401721": 7102, "n03402188": 7103, "n03402369": 7104, "n03402511": 7105, "n03402785": 7106, "n03402941": 7107, "n03403643": 7108, "n03404012": 7109, "n03404149": 7110, "n03404251": 7111, "n03404360": 7112, "n03404449": 7113, "n03404900": 7114, "n03405111": 7115, "n03405265": 7116, "n03405595": 7117, "n03405725": 7118, "n03406759": 7119, "n03406966": 7120, "n03407369": 7121, "n03407865": 7122, "n03408054": 7123, "n03408264": 7124, "n03408340": 7125, "n03408444": 7126, "n03409297": 7127, "n03409393": 7128, "n03409591": 7129, "n03409920": 7130, "n03410022": 7131, "n03410147": 7132, "n03410303": 7133, "n03410423": 7134, "n03410571": 7135, "n03410740": 7136, "n03410938": 7137, "n03411079": 7138, "n03411208": 7139, "n03411339": 7140, "n03411927": 7141, "n03412058": 7142, "n03412220": 7143, "n03412387": 7144, "n03412511": 7145, "n03412906": 7146, "n03413124": 7147, "n03413264": 7148, "n03413428": 7149, "n03413684": 7150, "n03413828": 7151, "n03414029": 7152, "n03414162": 7153, "n03414676": 7154, "n03415252": 7155, "n03415486": 7156, "n03415626": 7157, "n03415749": 7158, "n03415868": 7159, "n03416094": 7160, "n03416489": 7161, "n03416640": 7162, "n03416775": 7163, "n03416900": 7164, "n03417042": 7165, "n03417202": 7166, "n03417345": 7167, "n03417749": 7168, "n03417970": 7169, "n03418158": 7170, "n03418242": 7171, "n03418402": 7172, "n03418618": 7173, "n03418749": 7174, "n03418915": 7175, "n03419014": 7176, "n03420345": 7177, "n03420801": 7178, "n03420935": 7179, "n03421117": 7180, "n03421324": 7181, "n03421485": 7182, "n03421669": 7183, "n03421768": 7184, "n03421960": 7185, "n03422072": 7186, "n03422484": 7187, "n03422589": 7188, "n03422771": 7189, "n03423099": 7190, "n03423224": 7191, "n03423306": 7192, "n03423479": 7193, "n03423568": 7194, "n03423719": 7195, "n03423877": 7196, "n03424204": 7197, "n03424325": 7198, "n03424489": 7199, "n03424630": 7200, "n03424862": 7201, "n03425241": 7202, "n03425325": 7203, "n03425413": 7204, "n03425595": 7205, "n03425769": 7206, "n03426134": 7207, "n03426285": 7208, "n03426462": 7209, "n03426574": 7210, "n03426871": 7211, "n03427202": 7212, "n03427296": 7213, "n03428090": 7214, "n03428226": 7215, "n03428349": 7216, "n03429003": 7217, "n03429137": 7218, "n03429288": 7219, "n03429682": 7220, "n03429771": 7221, "n03429914": 7222, "n03430091": 7223, "n03430313": 7224, "n03430418": 7225, "n03430551": 7226, "n03430959": 7227, "n03431243": 7228, "n03431570": 7229, "n03431745": 7230, "n03432061": 7231, "n03432129": 7232, "n03432360": 7233, "n03432509": 7234, "n03433247": 7235, "n03433637": 7236, "n03433877": 7237, "n03434188": 7238, "n03434285": 7239, "n03434830": 7240, "n03435593": 7241, "n03435743": 7242, "n03435991": 7243, "n03436075": 7244, "n03436182": 7245, "n03436417": 7246, "n03436549": 7247, "n03436656": 7248, "n03436772": 7249, "n03436891": 7250, "n03436990": 7251, "n03437184": 7252, "n03437295": 7253, "n03437430": 7254, "n03437581": 7255, "n03437741": 7256, "n03437829": 7257, "n03437941": 7258, "n03438071": 7259, "n03438257": 7260, "n03438661": 7261, "n03438780": 7262, "n03438863": 7263, "n03439348": 7264, "n03439631": 7265, "n03439814": 7266, "n03440216": 7267, "n03440682": 7268, "n03440876": 7269, "n03441112": 7270, "n03441345": 7271, "n03441465": 7272, "n03441582": 7273, "n03442288": 7274, "n03442487": 7275, "n03442597": 7276, "n03442756": 7277, "n03443005": 7278, "n03443149": 7279, "n03443371": 7280, "n03443543": 7281, "n03443912": 7282, "n03444034": 7283, "n03445326": 7284, "n03445617": 7285, "n03445777": 7286, "n03445924": 7287, "n03446070": 7288, "n03446268": 7289, "n03446832": 7290, "n03447075": 7291, "n03447358": 7292, "n03447447": 7293, "n03447721": 7294, "n03447894": 7295, "n03448031": 7296, "n03448590": 7297, "n03448696": 7298, "n03448956": 7299, "n03449217": 7300, "n03449309": 7301, "n03449451": 7302, "n03449564": 7303, "n03449858": 7304, "n03450230": 7305, "n03450516": 7306, "n03450734": 7307, "n03450881": 7308, "n03450974": 7309, "n03451120": 7310, "n03451253": 7311, "n03451365": 7312, "n03451711": 7313, "n03451798": 7314, "n03452267": 7315, "n03452449": 7316, "n03452594": 7317, "n03452741": 7318, "n03453231": 7319, "n03453320": 7320, "n03453443": 7321, "n03454110": 7322, "n03454211": 7323, "n03454442": 7324, "n03454536": 7325, "n03454707": 7326, "n03454885": 7327, "n03455355": 7328, "n03455488": 7329, "n03455642": 7330, "n03455802": 7331, "n03456024": 7332, "n03456186": 7333, "n03456299": 7334, "n03456447": 7335, "n03456548": 7336, "n03456665": 7337, "n03457008": 7338, "n03457451": 7339, "n03457686": 7340, "n03457902": 7341, "n03458271": 7342, "n03458422": 7343, "n03459328": 7344, "n03459591": 7345, "n03459775": 7346, "n03459914": 7347, "n03460040": 7348, "n03460147": 7349, "n03460297": 7350, "n03460455": 7351, "n03460899": 7352, "n03461288": 7353, "n03461385": 7354, "n03461651": 7355, "n03461882": 7356, "n03461988": 7357, "n03462110": 7358, "n03462315": 7359, "n03462747": 7360, "n03462972": 7361, "n03463185": 7362, "n03463381": 7363, "n03463666": 7364, "n03464053": 7365, "n03464467": 7366, "n03464628": 7367, "n03464952": 7368, "n03465040": 7369, "n03465151": 7370, "n03465320": 7371, "n03465426": 7372, "n03465500": 7373, "n03465605": 7374, "n03465718": 7375, "n03465818": 7376, "n03466162": 7377, "n03466493": 7378, "n03466600": 7379, "n03466839": 7380, "n03466947": 7381, "n03467068": 7382, "n03467254": 7383, "n03467380": 7384, "n03467517": 7385, "n03467796": 7386, "n03467887": 7387, "n03467984": 7388, "n03468570": 7389, "n03468696": 7390, "n03468821": 7391, "n03469031": 7392, "n03469175": 7393, "n03469493": 7394, "n03469832": 7395, "n03469903": 7396, "n03470005": 7397, "n03470222": 7398, "n03470387": 7399, "n03470629": 7400, "n03470948": 7401, "n03471030": 7402, "n03471190": 7403, "n03471347": 7404, "n03471779": 7405, "n03472232": 7406, "n03472535": 7407, "n03472672": 7408, "n03472796": 7409, "n03472937": 7410, "n03473078": 7411, "n03473227": 7412, "n03473465": 7413, "n03473817": 7414, "n03473966": 7415, "n03474167": 7416, "n03474352": 7417, "n03474779": 7418, "n03474896": 7419, "n03475581": 7420, "n03475674": 7421, "n03475823": 7422, "n03475961": 7423, "n03476083": 7424, "n03476313": 7425, "n03476542": 7426, "n03476684": 7427, "n03476991": 7428, "n03477143": 7429, "n03477303": 7430, "n03477410": 7431, "n03477512": 7432, "n03477773": 7433, "n03477902": 7434, "n03478589": 7435, "n03478756": 7436, "n03478907": 7437, "n03479121": 7438, "n03479266": 7439, "n03479397": 7440, "n03479502": 7441, "n03480579": 7442, "n03480719": 7443, "n03480973": 7444, "n03481172": 7445, "n03481521": 7446, "n03482001": 7447, "n03482128": 7448, "n03482252": 7449, "n03482405": 7450, "n03482523": 7451, "n03482877": 7452, "n03483086": 7453, "n03483230": 7454, "n03483316": 7455, "n03483531": 7456, "n03483637": 7457, "n03483823": 7458, "n03483971": 7459, "n03484083": 7460, "n03484487": 7461, "n03484576": 7462, "n03484809": 7463, "n03484931": 7464, "n03485198": 7465, "n03485309": 7466, "n03485407": 7467, "n03485575": 7468, "n03485794": 7469, "n03487090": 7470, "n03487331": 7471, "n03487444": 7472, "n03487533": 7473, "n03487642": 7474, "n03487774": 7475, "n03487886": 7476, "n03488111": 7477, "n03488188": 7478, "n03488438": 7479, "n03488603": 7480, "n03488784": 7481, "n03488887": 7482, "n03489048": 7483, "n03489162": 7484, "n03490006": 7485, "n03490119": 7486, "n03490324": 7487, "n03490449": 7488, "n03490649": 7489, "n03490784": 7490, "n03490884": 7491, "n03491032": 7492, "n03491724": 7493, "n03491988": 7494, "n03492087": 7495, "n03492250": 7496, "n03492542": 7497, "n03492922": 7498, "n03493219": 7499, "n03493792": 7500, "n03493911": 7501, "n03494278": 7502, "n03494537": 7503, "n03494706": 7504, "n03495039": 7505, "n03495258": 7506, "n03495570": 7507, "n03495671": 7508, "n03495941": 7509, "n03496183": 7510, "n03496296": 7511, "n03496486": 7512, "n03496612": 7513, "n03496892": 7514, "n03497100": 7515, "n03497352": 7516, "n03497657": 7517, "n03498441": 7518, "n03498536": 7519, "n03498662": 7520, "n03498781": 7521, "n03498866": 7522, "n03498962": 7523, "n03499354": 7524, "n03499468": 7525, "n03499907": 7526, "n03500090": 7527, "n03500209": 7528, "n03500295": 7529, "n03500389": 7530, "n03500457": 7531, "n03500557": 7532, "n03500699": 7533, "n03500838": 7534, "n03500971": 7535, "n03501152": 7536, "n03501288": 7537, "n03501520": 7538, "n03501614": 7539, "n03502200": 7540, "n03502331": 7541, "n03502509": 7542, "n03502777": 7543, "n03502897": 7544, "n03503097": 7545, "n03503233": 7546, "n03503358": 7547, "n03503477": 7548, "n03503567": 7549, "n03503718": 7550, "n03503997": 7551, "n03504205": 7552, "n03504293": 7553, "n03504723": 7554, "n03505015": 7555, "n03505133": 7556, "n03505383": 7557, "n03505504": 7558, "n03505667": 7559, "n03505764": 7560, "n03506028": 7561, "n03506184": 7562, "n03506370": 7563, "n03506560": 7564, "n03506727": 7565, "n03506880": 7566, "n03507241": 7567, "n03507458": 7568, "n03507658": 7569, "n03507963": 7570, "n03508101": 7571, "n03508485": 7572, "n03508881": 7573, "n03509394": 7574, "n03509608": 7575, "n03509843": 7576, "n03510072": 7577, "n03510244": 7578, "n03510384": 7579, "n03510487": 7580, "n03510583": 7581, "n03510866": 7582, "n03510987": 7583, "n03511175": 7584, "n03511333": 7585, "n03512030": 7586, "n03512147": 7587, "n03512452": 7588, "n03512624": 7589, "n03512911": 7590, "n03513137": 7591, "n03513376": 7592, "n03514129": 7593, "n03514340": 7594, "n03514451": 7595, "n03514693": 7596, "n03514894": 7597, "n03515338": 7598, "n03515934": 7599, "n03516266": 7600, "n03516367": 7601, "n03516647": 7602, "n03516844": 7603, "n03516996": 7604, "n03517509": 7605, "n03517647": 7606, "n03517760": 7607, "n03517899": 7608, "n03517982": 7609, "n03518135": 7610, "n03518230": 7611, "n03518305": 7612, "n03518445": 7613, "n03518631": 7614, "n03518829": 7615, "n03518943": 7616, "n03519081": 7617, "n03519226": 7618, "n03519387": 7619, "n03519674": 7620, "n03519848": 7621, "n03520493": 7622, "n03521076": 7623, "n03521431": 7624, "n03521544": 7625, "n03521675": 7626, "n03521771": 7627, "n03521899": 7628, "n03522003": 7629, "n03522100": 7630, "n03522634": 7631, "n03522863": 7632, "n03522990": 7633, "n03523134": 7634, "n03523398": 7635, "n03523506": 7636, "n03523987": 7637, "n03524150": 7638, "n03524287": 7639, "n03524425": 7640, "n03524574": 7641, "n03524745": 7642, "n03524976": 7643, "n03525074": 7644, "n03525252": 7645, "n03525454": 7646, "n03525693": 7647, "n03525827": 7648, "n03526062": 7649, "n03527149": 7650, "n03527444": 7651, "n03527565": 7652, "n03527675": 7653, "n03528100": 7654, "n03528263": 7655, "n03528523": 7656, "n03528901": 7657, "n03529175": 7658, "n03529444": 7659, "n03529629": 7660, "n03529860": 7661, "n03530189": 7662, "n03530511": 7663, "n03530642": 7664, "n03530910": 7665, "n03531281": 7666, "n03531447": 7667, "n03531546": 7668, "n03531691": 7669, "n03531982": 7670, "n03532342": 7671, "n03532672": 7672, "n03532919": 7673, "n03533014": 7674, "n03533392": 7675, "n03533486": 7676, "n03533654": 7677, "n03533845": 7678, "n03534580": 7679, "n03534695": 7680, "n03534776": 7681, "n03535024": 7682, "n03535284": 7683, "n03535647": 7684, "n03535780": 7685, "n03536122": 7686, "n03536568": 7687, "n03536761": 7688, "n03537085": 7689, "n03537241": 7690, "n03537412": 7691, "n03537550": 7692, "n03538037": 7693, "n03538179": 7694, "n03538300": 7695, "n03538406": 7696, "n03538542": 7697, "n03538634": 7698, "n03538817": 7699, "n03538957": 7700, "n03539103": 7701, "n03539293": 7702, "n03539433": 7703, "n03539546": 7704, "n03539678": 7705, "n03539754": 7706, "n03540090": 7707, "n03540267": 7708, "n03540476": 7709, "n03540595": 7710, "n03540914": 7711, "n03541091": 7712, "n03541269": 7713, "n03541393": 7714, "n03541537": 7715, "n03541696": 7716, "n03541923": 7717, "n03542333": 7718, "n03542605": 7719, "n03542727": 7720, "n03542860": 7721, "n03543012": 7722, "n03543112": 7723, "n03543254": 7724, "n03543394": 7725, "n03543511": 7726, "n03543603": 7727, "n03543735": 7728, "n03543945": 7729, "n03544143": 7730, "n03544238": 7731, "n03544360": 7732, "n03545150": 7733, "n03545470": 7734, "n03545585": 7735, "n03545756": 7736, "n03545961": 7737, "n03546112": 7738, "n03546235": 7739, "n03546340": 7740, "n03547054": 7741, "n03547229": 7742, "n03547397": 7743, "n03547530": 7744, "n03547861": 7745, "n03548086": 7746, "n03548195": 7747, "n03548320": 7748, "n03548402": 7749, "n03548533": 7750, "n03548626": 7751, "n03548930": 7752, "n03549199": 7753, "n03549350": 7754, "n03549473": 7755, "n03549589": 7756, "n03549732": 7757, "n03549897": 7758, "n03550153": 7759, "n03550289": 7760, "n03550420": 7761, "n03551084": 7762, "n03551395": 7763, "n03551582": 7764, "n03551790": 7765, "n03552001": 7766, "n03552449": 7767, "n03552749": 7768, "n03553019": 7769, "n03553248": 7770, "n03553486": 7771, "n03554375": 7772, "n03554460": 7773, "n03554645": 7774, "n03555006": 7775, "n03555217": 7776, "n03555426": 7777, "n03555564": 7778, "n03555662": 7779, "n03555862": 7780, "n03555996": 7781, "n03556173": 7782, "n03556679": 7783, "n03556811": 7784, "n03556992": 7785, "n03557270": 7786, "n03557360": 7787, "n03557590": 7788, "n03557692": 7789, "n03557840": 7790, "n03558007": 7791, "n03558176": 7792, "n03558404": 7793, "n03558633": 7794, "n03558739": 7795, "n03559373": 7796, "n03559531": 7797, "n03559999": 7798, "n03560430": 7799, "n03560860": 7800, "n03561047": 7801, "n03561169": 7802, "n03561573": 7803, "n03562565": 7804, "n03563200": 7805, "n03563460": 7806, "n03563710": 7807, "n03563967": 7808, "n03564849": 7809, "n03565288": 7810, "n03565565": 7811, "n03565710": 7812, "n03565830": 7813, "n03565991": 7814, "n03566193": 7815, "n03566329": 7816, "n03566555": 7817, "n03566730": 7818, "n03566860": 7819, "n03567066": 7820, "n03567635": 7821, "n03567788": 7822, "n03567912": 7823, "n03568117": 7824, "n03568818": 7825, "n03569014": 7826, "n03569174": 7827, "n03569293": 7828, "n03569494": 7829, "n03571280": 7830, "n03571439": 7831, "n03571625": 7832, "n03571853": 7833, "n03571942": 7834, "n03572107": 7835, "n03572205": 7836, "n03572321": 7837, "n03572631": 7838, "n03573574": 7839, "n03573848": 7840, "n03574243": 7841, "n03574416": 7842, "n03574555": 7843, "n03574816": 7844, "n03575958": 7845, "n03576215": 7846, "n03576443": 7847, "n03576955": 7848, "n03577090": 7849, "n03577312": 7850, "n03577474": 7851, "n03577672": 7852, "n03577818": 7853, "n03578055": 7854, "n03578251": 7855, "n03578656": 7856, "n03578981": 7857, "n03579538": 7858, "n03579982": 7859, "n03580518": 7860, "n03580615": 7861, "n03580845": 7862, "n03580990": 7863, "n03581125": 7864, "n03581531": 7865, "n03581897": 7866, "n03582508": 7867, "n03582959": 7868, "n03583419": 7869, "n03583621": 7870, "n03584254": 7871, "n03584400": 7872, "n03584829": 7873, "n03585073": 7874, "n03585337": 7875, "n03585438": 7876, "n03585551": 7877, "n03585682": 7878, "n03585778": 7879, "n03585875": 7880, "n03586219": 7881, "n03586631": 7882, "n03586911": 7883, "n03587205": 7884, "n03588216": 7885, "n03588841": 7886, "n03588951": 7887, "n03589313": 7888, "n03589513": 7889, "n03589672": 7890, "n03589791": 7891, "n03590306": 7892, "n03590475": 7893, "n03590588": 7894, "n03590841": 7895, "n03590932": 7896, "n03591116": 7897, "n03591313": 7898, "n03591592": 7899, "n03591798": 7900, "n03591901": 7901, "n03592245": 7902, "n03592669": 7903, "n03592773": 7904, "n03592931": 7905, "n03593122": 7906, "n03593222": 7907, "n03593526": 7908, "n03593862": 7909, "n03594010": 7910, "n03594148": 7911, "n03594277": 7912, "n03594523": 7913, "n03594734": 7914, "n03594945": 7915, "n03595055": 7916, "n03595264": 7917, "n03595409": 7918, "n03595523": 7919, "n03595614": 7920, "n03595860": 7921, "n03596099": 7922, "n03596285": 7923, "n03596543": 7924, "n03597147": 7925, "n03597317": 7926, "n03597916": 7927, "n03598151": 7928, "n03598299": 7929, "n03598385": 7930, "n03598515": 7931, "n03598646": 7932, "n03598783": 7933, "n03598930": 7934, "n03599486": 7935, "n03599964": 7936, "n03600285": 7937, "n03600475": 7938, "n03600722": 7939, "n03600977": 7940, "n03601442": 7941, "n03601638": 7942, "n03601840": 7943, "n03602081": 7944, "n03602194": 7945, "n03602365": 7946, "n03602686": 7947, "n03602790": 7948, "n03602883": 7949, "n03603442": 7950, "n03603594": 7951, "n03603722": 7952, "n03604156": 7953, "n03604311": 7954, "n03604400": 7955, "n03604536": 7956, "n03604629": 7957, "n03604763": 7958, "n03604843": 7959, "n03605417": 7960, "n03605504": 7961, "n03605598": 7962, "n03605722": 7963, "n03605915": 7964, "n03606106": 7965, "n03606251": 7966, "n03606347": 7967, "n03606465": 7968, "n03607029": 7969, "n03607186": 7970, "n03607527": 7971, "n03607659": 7972, "n03607923": 7973, "n03608504": 7974, "n03609147": 7975, "n03609235": 7976, "n03609397": 7977, "n03609542": 7978, "n03609786": 7979, "n03609959": 7980, "n03610098": 7981, "n03610418": 7982, "n03610524": 7983, "n03610682": 7984, "n03610836": 7985, "n03610992": 7986, "n03612010": 7987, "n03612814": 7988, "n03612965": 7989, "n03613294": 7990, "n03613592": 7991, "n03614007": 7992, "n03614383": 7993, "n03614532": 7994, "n03614782": 7995, "n03614887": 7996, "n03615300": 7997, "n03615406": 7998, "n03615563": 7999, "n03615655": 8000, "n03615790": 8001, "n03616091": 8002, "n03616225": 8003, "n03616428": 8004, "n03616763": 8005, "n03616979": 8006, "n03617095": 8007, "n03617312": 8008, "n03617480": 8009, "n03617594": 8010, "n03617834": 8011, "n03618101": 8012, "n03618339": 8013, "n03618546": 8014, "n03618678": 8015, "n03618797": 8016, "n03618982": 8017, "n03619050": 8018, "n03619196": 8019, "n03619275": 8020, "n03619396": 8021, "n03619650": 8022, "n03619793": 8023, "n03619890": 8024, "n03620052": 8025, "n03620353": 8026, "n03620967": 8027, "n03621049": 8028, "n03621377": 8029, "n03621694": 8030, "n03622058": 8031, "n03622401": 8032, "n03622526": 8033, "n03622839": 8034, "n03622931": 8035, "n03623198": 8036, "n03623338": 8037, "n03623556": 8038, "n03624134": 8039, "n03624400": 8040, "n03624767": 8041, "n03625355": 8042, "n03625539": 8043, "n03625646": 8044, "n03625943": 8045, "n03626115": 8046, "n03626272": 8047, "n03626418": 8048, "n03626502": 8049, "n03626760": 8050, "n03627232": 8051, "n03627954": 8052, "n03628071": 8053, "n03628215": 8054, "n03628421": 8055, "n03628511": 8056, "n03628728": 8057, "n03628831": 8058, "n03628984": 8059, "n03629100": 8060, "n03629231": 8061, "n03629520": 8062, "n03629643": 8063, "n03630262": 8064, "n03630383": 8065, "n03631177": 8066, "n03631811": 8067, "n03631922": 8068, "n03632100": 8069, "n03632577": 8070, "n03632729": 8071, "n03632852": 8072, "n03632963": 8073, "n03633091": 8074, "n03633341": 8075, "n03633632": 8076, "n03633886": 8077, "n03634034": 8078, "n03634899": 8079, "n03635032": 8080, "n03635108": 8081, "n03635330": 8082, "n03635516": 8083, "n03635668": 8084, "n03635932": 8085, "n03636248": 8086, "n03636649": 8087, "n03637027": 8088, "n03637181": 8089, "n03637318": 8090, "n03637480": 8091, "n03637787": 8092, "n03637898": 8093, "n03638014": 8094, "n03638180": 8095, "n03638623": 8096, "n03638743": 8097, "n03638883": 8098, "n03639077": 8099, "n03639230": 8100, "n03639497": 8101, "n03639675": 8102, "n03639880": 8103, "n03640850": 8104, "n03640988": 8105, "n03641569": 8106, "n03641947": 8107, "n03642144": 8108, "n03642341": 8109, "n03642444": 8110, "n03642573": 8111, "n03642806": 8112, "n03643149": 8113, "n03643253": 8114, "n03643491": 8115, "n03643737": 8116, "n03643907": 8117, "n03644073": 8118, "n03644378": 8119, "n03644858": 8120, "n03645011": 8121, "n03645168": 8122, "n03645290": 8123, "n03645577": 8124, "n03646020": 8125, "n03646148": 8126, "n03646296": 8127, "n03646809": 8128, "n03646916": 8129, "n03647423": 8130, "n03647520": 8131, "n03648219": 8132, "n03648431": 8133, "n03648667": 8134, "n03649003": 8135, "n03649161": 8136, "n03649288": 8137, "n03649674": 8138, "n03649797": 8139, "n03649909": 8140, "n03650551": 8141, "n03651388": 8142, "n03651605": 8143, "n03651843": 8144, "n03652100": 8145, "n03652389": 8146, "n03652729": 8147, "n03652826": 8148, "n03652932": 8149, "n03653110": 8150, "n03653220": 8151, "n03653454": 8152, "n03653583": 8153, "n03653740": 8154, "n03653833": 8155, "n03653975": 8156, "n03654576": 8157, "n03654826": 8158, "n03655072": 8159, "n03655470": 8160, "n03655720": 8161, "n03656484": 8162, "n03656957": 8163, "n03657121": 8164, "n03657239": 8165, "n03657511": 8166, "n03658102": 8167, "n03658185": 8168, "n03658635": 8169, "n03658858": 8170, "n03659292": 8171, "n03659686": 8172, "n03659809": 8173, "n03659950": 8174, "n03660124": 8175, "n03660562": 8176, "n03660909": 8177, "n03661043": 8178, "n03661340": 8179, "n03662301": 8180, "n03662452": 8181, "n03662601": 8182, "n03662719": 8183, "n03662887": 8184, "n03663433": 8185, "n03663531": 8186, "n03663910": 8187, "n03664159": 8188, "n03664675": 8189, "n03664840": 8190, "n03664943": 8191, "n03665232": 8192, "n03665366": 8193, "n03665851": 8194, "n03665924": 8195, "n03666238": 8196, "n03666362": 8197, "n03666591": 8198, "n03666917": 8199, "n03667060": 8200, "n03667235": 8201, "n03667552": 8202, "n03667664": 8203, "n03667829": 8204, "n03668067": 8205, "n03668279": 8206, "n03668488": 8207, "n03668803": 8208, "n03669245": 8209, "n03669534": 8210, "n03669886": 8211, "n03670208": 8212, "n03671914": 8213, "n03672521": 8214, "n03672827": 8215, "n03673027": 8216, "n03673270": 8217, "n03673450": 8218, "n03673767": 8219, "n03674270": 8220, "n03674440": 8221, "n03674731": 8222, "n03674842": 8223, "n03675076": 8224, "n03675235": 8225, "n03675445": 8226, "n03675558": 8227, "n03675907": 8228, "n03676087": 8229, "n03676483": 8230, "n03676623": 8231, "n03676759": 8232, "n03677115": 8233, "n03677682": 8234, "n03677766": 8235, "n03678558": 8236, "n03678729": 8237, "n03678879": 8238, "n03679384": 8239, "n03679712": 8240, "n03680248": 8241, "n03680355": 8242, "n03680512": 8243, "n03680734": 8244, "n03680858": 8245, "n03680942": 8246, "n03681477": 8247, "n03681813": 8248, "n03682380": 8249, "n03682487": 8250, "n03682877": 8251, "n03683079": 8252, "n03683341": 8253, "n03683457": 8254, "n03683606": 8255, "n03683708": 8256, "n03683995": 8257, "n03684143": 8258, "n03684224": 8259, "n03684489": 8260, "n03684611": 8261, "n03684740": 8262, "n03684823": 8263, "n03685307": 8264, "n03685486": 8265, "n03685640": 8266, "n03685820": 8267, "n03686130": 8268, "n03686363": 8269, "n03686470": 8270, "n03686924": 8271, "n03687137": 8272, "n03687928": 8273, "n03688066": 8274, "n03688192": 8275, "n03688405": 8276, "n03688504": 8277, "n03688605": 8278, "n03688707": 8279, "n03688832": 8280, "n03688943": 8281, "n03689157": 8282, "n03689570": 8283, "n03690168": 8284, "n03690279": 8285, "n03690473": 8286, "n03690851": 8287, "n03690938": 8288, "n03691459": 8289, "n03691817": 8290, "n03692004": 8291, "n03692136": 8292, "n03692272": 8293, "n03692379": 8294, "n03692522": 8295, "n03692842": 8296, "n03693293": 8297, "n03693474": 8298, "n03693707": 8299, "n03693860": 8300, "n03694196": 8301, "n03694356": 8302, "n03694639": 8303, "n03694761": 8304, "n03694949": 8305, "n03695122": 8306, "n03695452": 8307, "n03695616": 8308, "n03695753": 8309, "n03695857": 8310, "n03695957": 8311, "n03696065": 8312, "n03696301": 8313, "n03696445": 8314, "n03696568": 8315, "n03696746": 8316, "n03696909": 8317, "n03697007": 8318, "n03697366": 8319, "n03697552": 8320, "n03697812": 8321, "n03697913": 8322, "n03698123": 8323, "n03698226": 8324, "n03698360": 8325, "n03698604": 8326, "n03698723": 8327, "n03698815": 8328, "n03699280": 8329, "n03699591": 8330, "n03699754": 8331, "n03699975": 8332, "n03700963": 8333, "n03701191": 8334, "n03701391": 8335, "n03701640": 8336, "n03701790": 8337, "n03702248": 8338, "n03702440": 8339, "n03702582": 8340, "n03703075": 8341, "n03703203": 8342, "n03703463": 8343, "n03703590": 8344, "n03703730": 8345, "n03703862": 8346, "n03703945": 8347, "n03704549": 8348, "n03704834": 8349, "n03705379": 8350, "n03705808": 8351, "n03706229": 8352, "n03706415": 8353, "n03706653": 8354, "n03706939": 8355, "n03707171": 8356, "n03707372": 8357, "n03707597": 8358, "n03707766": 8359, "n03708036": 8360, "n03708425": 8361, "n03708843": 8362, "n03708962": 8363, "n03709206": 8364, "n03709363": 8365, "n03709545": 8366, "n03709644": 8367, "n03709823": 8368, "n03709960": 8369, "n03710079": 8370, "n03710193": 8371, "n03710294": 8372, "n03710421": 8373, "n03710528": 8374, "n03710637": 8375, "n03710721": 8376, "n03710937": 8377, "n03711044": 8378, "n03711711": 8379, "n03711999": 8380, "n03712111": 8381, "n03712337": 8382, "n03712444": 8383, "n03712887": 8384, "n03712981": 8385, "n03713069": 8386, "n03713151": 8387, "n03713436": 8388, "n03714235": 8389, "n03715114": 8390, "n03715275": 8391, "n03715386": 8392, "n03715669": 8393, "n03715892": 8394, "n03716228": 8395, "n03716887": 8396, "n03716966": 8397, "n03717131": 8398, "n03717285": 8399, "n03717447": 8400, "n03717622": 8401, "n03718212": 8402, "n03718335": 8403, "n03718458": 8404, "n03718581": 8405, "n03718699": 8406, "n03718789": 8407, "n03718935": 8408, "n03719053": 8409, "n03719343": 8410, "n03719560": 8411, "n03719743": 8412, "n03720005": 8413, "n03720163": 8414, "n03720665": 8415, "n03720891": 8416, "n03721047": 8417, "n03721252": 8418, "n03721384": 8419, "n03721590": 8420, "n03722007": 8421, "n03722288": 8422, "n03722646": 8423, "n03722944": 8424, "n03723153": 8425, "n03723267": 8426, "n03723439": 8427, "n03723781": 8428, "n03723885": 8429, "n03724066": 8430, "n03724176": 8431, "n03724417": 8432, "n03724538": 8433, "n03724623": 8434, "n03724756": 8435, "n03724870": 8436, "n03725035": 8437, "n03725506": 8438, "n03725600": 8439, "n03725717": 8440, "n03725869": 8441, "n03726116": 8442, "n03726233": 8443, "n03726371": 8444, "n03726516": 8445, "n03726760": 8446, "n03726993": 8447, "n03727067": 8448, "n03727465": 8449, "n03727605": 8450, "n03727837": 8451, "n03727946": 8452, "n03728437": 8453, "n03728982": 8454, "n03729131": 8455, "n03729308": 8456, "n03729402": 8457, "n03729482": 8458, "n03729647": 8459, "n03729826": 8460, "n03729951": 8461, "n03730153": 8462, "n03730334": 8463, "n03730494": 8464, "n03730655": 8465, "n03730788": 8466, "n03730893": 8467, "n03731019": 8468, "n03731483": 8469, "n03731695": 8470, "n03731882": 8471, "n03732020": 8472, "n03732114": 8473, "n03732458": 8474, "n03732543": 8475, "n03732658": 8476, "n03733131": 8477, "n03733281": 8478, "n03733465": 8479, "n03733547": 8480, "n03733644": 8481, "n03733805": 8482, "n03733925": 8483, "n03735637": 8484, "n03735963": 8485, "n03736064": 8486, "n03736147": 8487, "n03736269": 8488, "n03736372": 8489, "n03736470": 8490, "n03736970": 8491, "n03738066": 8492, "n03738241": 8493, "n03738472": 8494, "n03739518": 8495, "n03739693": 8496, "n03742019": 8497, "n03742115": 8498, "n03742238": 8499, "n03743016": 8500, "n03743279": 8501, "n03743902": 8502, "n03744276": 8503, "n03744684": 8504, "n03744840": 8505, "n03745146": 8506, "n03745487": 8507, "n03745571": 8508, "n03746005": 8509, "n03746155": 8510, "n03746330": 8511, "n03746486": 8512, "n03748162": 8513, "n03749504": 8514, "n03749634": 8515, "n03749807": 8516, "n03750206": 8517, "n03750437": 8518, "n03750614": 8519, "n03751065": 8520, "n03751269": 8521, "n03751458": 8522, "n03751590": 8523, "n03751757": 8524, "n03752071": 8525, "n03752185": 8526, "n03752398": 8527, "n03752922": 8528, "n03753077": 8529, "n03753514": 8530, "n03757604": 8531, "n03758089": 8532, "n03758220": 8533, "n03758894": 8534, "n03758992": 8535, "n03759243": 8536, "n03759432": 8537, "n03759661": 8538, "n03759954": 8539, "n03760310": 8540, "n03760671": 8541, "n03760944": 8542, "n03761084": 8543, "n03761588": 8544, "n03761731": 8545, "n03762238": 8546, "n03762332": 8547, "n03762434": 8548, "n03762602": 8549, "n03762982": 8550, "n03763727": 8551, "n03763968": 8552, "n03764276": 8553, "n03764606": 8554, "n03764736": 8555, "n03764822": 8556, "n03764995": 8557, "n03765128": 8558, "n03765467": 8559, "n03765561": 8560, "n03765934": 8561, "n03766044": 8562, "n03766218": 8563, "n03766322": 8564, "n03766508": 8565, "n03766600": 8566, "n03766697": 8567, "n03766935": 8568, "n03767112": 8569, "n03767203": 8570, "n03767459": 8571, "n03767745": 8572, "n03767966": 8573, "n03768132": 8574, "n03768683": 8575, "n03768823": 8576, "n03768916": 8577, "n03769610": 8578, "n03769722": 8579, "n03769881": 8580, "n03770085": 8581, "n03770224": 8582, "n03770316": 8583, "n03770439": 8584, "n03770520": 8585, "n03770679": 8586, "n03770834": 8587, "n03770954": 8588, "n03772077": 8589, "n03772269": 8590, "n03772584": 8591, "n03772674": 8592, "n03773035": 8593, "n03773504": 8594, "n03773835": 8595, "n03774327": 8596, "n03774461": 8597, "n03775071": 8598, "n03775199": 8599, "n03775388": 8600, "n03775546": 8601, "n03775636": 8602, "n03775747": 8603, "n03775847": 8604, "n03776167": 8605, "n03776460": 8606, "n03776877": 8607, "n03776997": 8608, "n03777126": 8609, "n03777568": 8610, "n03777754": 8611, "n03778459": 8612, "n03778817": 8613, "n03779000": 8614, "n03779128": 8615, "n03779246": 8616, "n03779370": 8617, "n03779884": 8618, "n03780047": 8619, "n03780799": 8620, "n03781055": 8621, "n03781244": 8622, "n03781467": 8623, "n03781594": 8624, "n03781683": 8625, "n03781787": 8626, "n03782006": 8627, "n03782190": 8628, "n03782794": 8629, "n03782929": 8630, "n03783304": 8631, "n03783430": 8632, "n03783575": 8633, "n03783873": 8634, "n03784139": 8635, "n03784270": 8636, "n03784793": 8637, "n03784896": 8638, "n03785016": 8639, "n03785142": 8640, "n03785237": 8641, "n03785499": 8642, "n03785721": 8643, "n03786096": 8644, "n03786194": 8645, "n03786313": 8646, "n03786621": 8647, "n03786715": 8648, "n03786901": 8649, "n03787032": 8650, "n03787523": 8651, "n03788047": 8652, "n03788195": 8653, "n03788365": 8654, "n03788498": 8655, "n03788601": 8656, "n03788914": 8657, "n03789171": 8658, "n03789400": 8659, "n03789603": 8660, "n03789794": 8661, "n03789946": 8662, "n03790230": 8663, "n03790512": 8664, "n03790755": 8665, "n03790953": 8666, "n03791053": 8667, "n03791235": 8668, "n03792048": 8669, "n03792334": 8670, "n03792526": 8671, "n03792782": 8672, "n03792972": 8673, "n03793489": 8674, "n03793850": 8675, "n03794056": 8676, "n03794136": 8677, "n03794798": 8678, "n03795123": 8679, "n03795269": 8680, "n03795758": 8681, "n03795976": 8682, "n03796181": 8683, "n03796401": 8684, "n03796522": 8685, "n03796605": 8686, "n03796848": 8687, "n03796974": 8688, "n03797062": 8689, "n03797182": 8690, "n03797264": 8691, "n03797390": 8692, "n03797896": 8693, "n03798061": 8694, "n03798442": 8695, "n03798610": 8696, "n03798982": 8697, "n03799113": 8698, "n03799240": 8699, "n03799375": 8700, "n03799610": 8701, "n03799876": 8702, "n03800371": 8703, "n03800485": 8704, "n03800563": 8705, "n03800772": 8706, "n03800933": 8707, "n03801353": 8708, "n03801533": 8709, "n03801671": 8710, "n03801760": 8711, "n03801880": 8712, "n03802007": 8713, "n03802228": 8714, "n03802393": 8715, "n03802643": 8716, "n03802800": 8717, "n03802973": 8718, "n03803116": 8719, "n03803284": 8720, "n03803780": 8721, "n03804211": 8722, "n03804744": 8723, "n03805180": 8724, "n03805280": 8725, "n03805374": 8726, "n03805503": 8727, "n03805725": 8728, "n03805933": 8729, "n03807334": 8730, "n03809211": 8731, "n03809312": 8732, "n03809603": 8733, "n03809686": 8734, "n03809802": 8735, "n03810412": 8736, "n03810952": 8737, "n03811295": 8738, "n03811444": 8739, "n03811847": 8740, "n03811965": 8741, "n03812263": 8742, "n03812382": 8743, "n03812789": 8744, "n03812924": 8745, "n03813078": 8746, "n03813176": 8747, "n03813946": 8748, "n03814528": 8749, "n03814639": 8750, "n03814727": 8751, "n03814817": 8752, "n03814906": 8753, "n03815149": 8754, "n03815278": 8755, "n03815482": 8756, "n03815615": 8757, "n03816005": 8758, "n03816136": 8759, "n03816394": 8760, "n03816530": 8761, "n03816849": 8762, "n03817191": 8763, "n03817331": 8764, "n03817522": 8765, "n03817647": 8766, "n03818001": 8767, "n03818343": 8768, "n03819047": 8769, "n03819336": 8770, "n03819448": 8771, "n03819595": 8772, "n03819994": 8773, "n03820154": 8774, "n03820318": 8775, "n03820728": 8776, "n03820950": 8777, "n03821145": 8778, "n03821424": 8779, "n03821518": 8780, "n03822171": 8781, "n03822361": 8782, "n03822504": 8783, "n03822656": 8784, "n03822767": 8785, "n03823111": 8786, "n03823216": 8787, "n03823312": 8788, "n03823673": 8789, "n03823906": 8790, "n03824197": 8791, "n03824284": 8792, "n03824381": 8793, "n03824589": 8794, "n03824713": 8795, "n03824999": 8796, "n03825080": 8797, "n03825271": 8798, "n03825442": 8799, "n03825673": 8800, "n03825788": 8801, "n03825913": 8802, "n03826039": 8803, "n03826186": 8804, "n03827420": 8805, "n03827536": 8806, "n03828020": 8807, "n03829340": 8808, "n03829857": 8809, "n03829954": 8810, "n03831203": 8811, "n03831382": 8812, "n03831757": 8813, "n03832144": 8814, "n03832673": 8815, "n03833907": 8816, "n03834040": 8817, "n03834472": 8818, "n03834604": 8819, "n03835197": 8820, "n03835729": 8821, "n03835941": 8822, "n03836062": 8823, "n03836451": 8824, "n03836602": 8825, "n03836906": 8826, "n03836976": 8827, "n03837422": 8828, "n03837606": 8829, "n03837698": 8830, "n03837869": 8831, "n03838024": 8832, "n03838298": 8833, "n03838748": 8834, "n03838899": 8835, "n03839172": 8836, "n03839276": 8837, "n03839424": 8838, "n03839671": 8839, "n03839795": 8840, "n03840327": 8841, "n03840681": 8842, "n03840823": 8843, "n03841011": 8844, "n03841143": 8845, "n03841290": 8846, "n03841666": 8847, "n03842012": 8848, "n03842156": 8849, "n03842276": 8850, "n03842377": 8851, "n03842585": 8852, "n03842754": 8853, "n03842986": 8854, "n03843092": 8855, "n03843316": 8856, "n03843438": 8857, "n03843555": 8858, "n03843883": 8859, "n03844045": 8860, "n03844233": 8861, "n03844550": 8862, "n03844673": 8863, "n03844815": 8864, "n03844965": 8865, "n03845107": 8866, "n03845190": 8867, "n03845990": 8868, "n03846100": 8869, "n03846234": 8870, "n03846431": 8871, "n03846677": 8872, "n03846772": 8873, "n03846970": 8874, "n03847471": 8875, "n03847823": 8876, "n03848033": 8877, "n03848168": 8878, "n03848348": 8879, "n03848537": 8880, "n03849275": 8881, "n03849412": 8882, "n03849679": 8883, "n03849814": 8884, "n03849943": 8885, "n03850053": 8886, "n03850245": 8887, "n03850492": 8888, "n03850613": 8889, "n03851341": 8890, "n03851787": 8891, "n03852280": 8892, "n03852544": 8893, "n03852688": 8894, "n03853291": 8895, "n03853924": 8896, "n03854065": 8897, "n03854421": 8898, "n03854506": 8899, "n03854722": 8900, "n03854815": 8901, "n03855214": 8902, "n03855333": 8903, "n03855464": 8904, "n03855604": 8905, "n03855756": 8906, "n03855908": 8907, "n03856012": 8908, "n03856335": 8909, "n03856465": 8910, "n03856728": 8911, "n03857026": 8912, "n03857156": 8913, "n03857291": 8914, "n03857687": 8915, "n03857828": 8916, "n03858085": 8917, "n03858183": 8918, "n03858418": 8919, "n03858533": 8920, "n03858837": 8921, "n03859000": 8922, "n03859170": 8923, "n03859280": 8924, "n03859495": 8925, "n03859608": 8926, "n03859958": 8927, "n03860234": 8928, "n03860404": 8929, "n03861048": 8930, "n03861271": 8931, "n03861430": 8932, "n03861596": 8933, "n03861842": 8934, "n03862379": 8935, "n03862676": 8936, "n03862862": 8937, "n03863108": 8938, "n03863262": 8939, "n03863657": 8940, "n03863783": 8941, "n03863923": 8942, "n03864139": 8943, "n03864356": 8944, "n03864692": 8945, "n03865288": 8946, "n03865371": 8947, "n03865557": 8948, "n03865820": 8949, "n03865949": 8950, "n03866082": 8951, "n03867854": 8952, "n03868044": 8953, "n03868242": 8954, "n03868324": 8955, "n03868406": 8956, "n03868643": 8957, "n03868763": 8958, "n03868863": 8959, "n03869838": 8960, "n03869976": 8961, "n03870105": 8962, "n03870290": 8963, "n03870546": 8964, "n03870672": 8965, "n03870980": 8966, "n03871083": 8967, "n03871371": 8968, "n03871524": 8969, "n03871628": 8970, "n03871724": 8971, "n03871860": 8972, "n03872016": 8973, "n03872167": 8974, "n03872273": 8975, "n03873416": 8976, "n03873699": 8977, "n03873848": 8978, "n03873996": 8979, "n03874138": 8980, "n03874293": 8981, "n03874487": 8982, "n03874599": 8983, "n03874823": 8984, "n03875218": 8985, "n03875806": 8986, "n03875955": 8987, "n03876111": 8988, "n03876231": 8989, "n03877351": 8990, "n03877472": 8991, "n03877674": 8992, "n03877845": 8993, "n03878066": 8994, "n03878211": 8995, "n03878294": 8996, "n03878418": 8997, "n03878511": 8998, "n03878674": 8999, "n03878828": 9000, "n03878963": 9001, "n03879456": 9002, "n03879705": 9003, "n03880032": 9004, "n03880129": 9005, "n03880323": 9006, "n03880531": 9007, "n03881305": 9008, "n03881404": 9009, "n03881534": 9010, "n03882611": 9011, "n03882960": 9012, "n03883054": 9013, "n03883385": 9014, "n03883524": 9015, "n03883664": 9016, "n03883773": 9017, "n03883944": 9018, "n03884397": 9019, "n03884554": 9020, "n03884639": 9021, "n03884778": 9022, "n03884926": 9023, "n03885028": 9024, "n03885194": 9025, "n03885293": 9026, "n03885410": 9027, "n03885535": 9028, "n03885669": 9029, "n03885788": 9030, "n03885904": 9031, "n03886053": 9032, "n03886641": 9033, "n03886762": 9034, "n03886940": 9035, "n03887185": 9036, "n03887330": 9037, "n03887512": 9038, "n03887697": 9039, "n03887899": 9040, "n03888022": 9041, "n03888257": 9042, "n03888605": 9043, "n03888808": 9044, "n03888998": 9045, "n03889397": 9046, "n03889503": 9047, "n03889626": 9048, "n03889726": 9049, "n03889871": 9050, "n03890093": 9051, "n03890233": 9052, "n03890358": 9053, "n03890514": 9054, "n03891051": 9055, "n03891251": 9056, "n03891332": 9057, "n03891538": 9058, "n03892178": 9059, "n03892425": 9060, "n03892557": 9061, "n03892728": 9062, "n03893935": 9063, "n03894051": 9064, "n03894379": 9065, "n03894677": 9066, "n03894933": 9067, "n03895038": 9068, "n03895170": 9069, "n03895866": 9070, "n03896103": 9071, "n03896233": 9072, "n03896419": 9073, "n03896526": 9074, "n03896628": 9075, "n03896984": 9076, "n03897130": 9077, "n03897634": 9078, "n03897943": 9079, "n03898129": 9080, "n03898271": 9081, "n03898395": 9082, "n03898633": 9083, "n03898787": 9084, "n03899100": 9085, "n03899612": 9086, "n03899768": 9087, "n03899933": 9088, "n03900028": 9089, "n03900194": 9090, "n03900301": 9091, "n03900393": 9092, "n03900979": 9093, "n03901229": 9094, "n03901338": 9095, "n03901750": 9096, "n03901974": 9097, "n03902125": 9098, "n03902220": 9099, "n03902482": 9100, "n03902756": 9101, "n03903133": 9102, "n03903290": 9103, "n03903424": 9104, "n03903733": 9105, "n03903868": 9106, "n03904060": 9107, "n03904183": 9108, "n03904433": 9109, "n03904657": 9110, "n03904782": 9111, "n03904909": 9112, "n03905361": 9113, "n03905540": 9114, "n03905730": 9115, "n03905947": 9116, "n03906106": 9117, "n03906224": 9118, "n03906463": 9119, "n03906590": 9120, "n03906789": 9121, "n03906894": 9122, "n03906997": 9123, "n03907475": 9124, "n03907654": 9125, "n03907908": 9126, "n03908111": 9127, "n03908204": 9128, "n03908456": 9129, "n03908618": 9130, "n03908714": 9131, "n03909020": 9132, "n03909160": 9133, "n03909406": 9134, "n03909516": 9135, "n03909658": 9136, "n03911406": 9137, "n03911513": 9138, "n03911658": 9139, "n03911767": 9140, "n03911866": 9141, "n03912218": 9142, "n03912821": 9143, "n03913343": 9144, "n03913930": 9145, "n03914106": 9146, "n03914337": 9147, "n03914438": 9148, "n03914583": 9149, "n03914831": 9150, "n03915118": 9151, "n03915320": 9152, "n03915437": 9153, "n03915900": 9154, "n03916031": 9155, "n03916289": 9156, "n03916385": 9157, "n03916470": 9158, "n03916720": 9159, "n03917048": 9160, "n03917198": 9161, "n03917327": 9162, "n03917814": 9163, "n03918074": 9164, "n03918480": 9165, "n03918737": 9166, "n03919096": 9167, "n03919289": 9168, "n03919430": 9169, "n03919808": 9170, "n03920288": 9171, "n03920384": 9172, "n03920641": 9173, "n03920737": 9174, "n03920867": 9175, "n03923379": 9176, "n03923564": 9177, "n03923692": 9178, "n03923918": 9179, "n03924069": 9180, "n03924407": 9181, "n03924532": 9182, "n03924679": 9183, "n03926148": 9184, "n03926412": 9185, "n03926876": 9186, "n03927091": 9187, "n03927299": 9188, "n03927539": 9189, "n03927792": 9190, "n03928116": 9191, "n03928589": 9192, "n03928814": 9193, "n03928994": 9194, "n03929091": 9195, "n03929202": 9196, "n03929443": 9197, "n03929660": 9198, "n03929855": 9199, "n03930229": 9200, "n03930313": 9201, "n03930431": 9202, "n03930515": 9203, "n03930630": 9204, "n03931765": 9205, "n03931885": 9206, "n03931980": 9207, "n03932080": 9208, "n03932670": 9209, "n03933391": 9210, "n03933933": 9211, "n03934042": 9212, "n03934229": 9213, "n03934311": 9214, "n03934565": 9215, "n03934656": 9216, "n03934890": 9217, "n03935116": 9218, "n03935234": 9219, "n03935335": 9220, "n03935883": 9221, "n03936269": 9222, "n03936466": 9223, "n03937543": 9224, "n03937835": 9225, "n03937931": 9226, "n03938037": 9227, "n03938244": 9228, "n03938401": 9229, "n03938522": 9230, "n03938725": 9231, "n03939062": 9232, "n03939178": 9233, "n03939281": 9234, "n03939440": 9235, "n03939565": 9236, "n03939677": 9237, "n03939844": 9238, "n03940256": 9239, "n03940894": 9240, "n03941013": 9241, "n03941231": 9242, "n03941417": 9243, "n03941586": 9244, "n03941684": 9245, "n03941887": 9246, "n03942028": 9247, "n03942600": 9248, "n03942813": 9249, "n03942920": 9250, "n03943115": 9251, "n03943266": 9252, "n03943623": 9253, "n03943714": 9254, "n03943833": 9255, "n03943920": 9256, "n03944024": 9257, "n03944138": 9258, "n03944341": 9259, "n03945459": 9260, "n03945615": 9261, "n03945817": 9262, "n03945928": 9263, "n03946076": 9264, "n03946162": 9265, "n03947111": 9266, "n03947343": 9267, "n03947466": 9268, "n03947798": 9269, "n03947888": 9270, "n03948242": 9271, "n03948459": 9272, "n03948830": 9273, "n03948950": 9274, "n03949145": 9275, "n03949317": 9276, "n03949761": 9277, "n03950228": 9278, "n03950359": 9279, "n03950537": 9280, "n03950647": 9281, "n03950899": 9282, "n03951068": 9283, "n03951213": 9284, "n03951453": 9285, "n03951800": 9286, "n03951971": 9287, "n03952150": 9288, "n03952576": 9289, "n03953020": 9290, "n03953416": 9291, "n03953901": 9292, "n03954393": 9293, "n03954731": 9294, "n03955296": 9295, "n03955489": 9296, "n03955809": 9297, "n03955941": 9298, "n03956157": 9299, "n03956331": 9300, "n03956531": 9301, "n03956623": 9302, "n03956785": 9303, "n03956922": 9304, "n03957315": 9305, "n03957420": 9306, "n03957762": 9307, "n03957991": 9308, "n03958227": 9309, "n03958338": 9310, "n03958630": 9311, "n03958752": 9312, "n03959014": 9313, "n03959123": 9314, "n03959227": 9315, "n03959701": 9316, "n03960374": 9317, "n03960490": 9318, "n03961394": 9319, "n03961630": 9320, "n03961711": 9321, "n03961828": 9322, "n03961939": 9323, "n03962525": 9324, "n03962685": 9325, "n03962852": 9326, "n03962932": 9327, "n03963028": 9328, "n03963198": 9329, "n03963294": 9330, "n03963483": 9331, "n03963645": 9332, "n03964495": 9333, "n03964611": 9334, "n03965456": 9335, "n03965907": 9336, "n03966206": 9337, "n03966325": 9338, "n03966582": 9339, "n03966751": 9340, "n03966976": 9341, "n03967270": 9342, "n03967396": 9343, "n03967562": 9344, "n03967942": 9345, "n03968293": 9346, "n03968479": 9347, "n03968581": 9348, "n03968728": 9349, "n03969510": 9350, "n03970156": 9351, "n03970363": 9352, "n03970546": 9353, "n03971218": 9354, "n03971321": 9355, "n03971960": 9356, "n03972146": 9357, "n03972372": 9358, "n03972524": 9359, "n03973003": 9360, "n03973285": 9361, "n03973402": 9362, "n03973520": 9363, "n03973628": 9364, "n03973839": 9365, "n03973945": 9366, "n03974070": 9367, "n03974915": 9368, "n03975035": 9369, "n03975657": 9370, "n03975788": 9371, "n03975926": 9372, "n03976105": 9373, "n03976268": 9374, "n03976467": 9375, "n03976657": 9376, "n03977158": 9377, "n03977266": 9378, "n03977430": 9379, "n03977592": 9380, "n03977966": 9381, "n03978421": 9382, "n03978575": 9383, "n03978686": 9384, "n03978815": 9385, "n03978966": 9386, "n03979377": 9387, "n03979492": 9388, "n03980026": 9389, "n03980478": 9390, "n03980874": 9391, "n03980986": 9392, "n03981094": 9393, "n03981340": 9394, "n03981566": 9395, "n03981760": 9396, "n03981924": 9397, "n03982232": 9398, "n03982331": 9399, "n03982430": 9400, "n03982642": 9401, "n03982767": 9402, "n03982895": 9403, "n03983396": 9404, "n03983499": 9405, "n03983612": 9406, "n03983712": 9407, "n03983928": 9408, "n03984125": 9409, "n03984234": 9410, "n03984381": 9411, "n03984643": 9412, "n03984759": 9413, "n03985069": 9414, "n03985232": 9415, "n03985441": 9416, "n03985881": 9417, "n03986071": 9418, "n03986224": 9419, "n03986355": 9420, "n03986562": 9421, "n03986704": 9422, "n03986857": 9423, "n03986949": 9424, "n03987266": 9425, "n03987376": 9426, "n03987674": 9427, "n03987865": 9428, "n03987990": 9429, "n03988170": 9430, "n03988758": 9431, "n03988926": 9432, "n03989199": 9433, "n03989349": 9434, "n03989447": 9435, "n03989665": 9436, "n03989777": 9437, "n03989898": 9438, "n03990474": 9439, "n03991062": 9440, "n03991202": 9441, "n03991321": 9442, "n03991443": 9443, "n03991646": 9444, "n03991837": 9445, "n03992325": 9446, "n03992436": 9447, "n03992509": 9448, "n03992703": 9449, "n03992975": 9450, "n03993053": 9451, "n03993180": 9452, "n03993403": 9453, "n03993703": 9454, "n03993878": 9455, "n03994008": 9456, "n03994297": 9457, "n03994417": 9458, "n03994614": 9459, "n03994757": 9460, "n03995018": 9461, "n03995265": 9462, "n03995372": 9463, "n03995535": 9464, "n03995661": 9465, "n03995856": 9466, "n03996004": 9467, "n03996145": 9468, "n03996416": 9469, "n03996849": 9470, "n03997274": 9471, "n03997484": 9472, "n03997875": 9473, "n03998194": 9474, "n03998333": 9475, "n03998673": 9476, "n03999064": 9477, "n03999160": 9478, "n03999621": 9479, "n03999992": 9480, "n04000311": 9481, "n04000480": 9482, "n04000592": 9483, "n04000716": 9484, "n04000998": 9485, "n04001132": 9486, "n04001265": 9487, "n04001397": 9488, "n04001499": 9489, "n04001661": 9490, "n04001845": 9491, "n04002262": 9492, "n04002371": 9493, "n04002629": 9494, "n04003241": 9495, "n04003359": 9496, "n04003856": 9497, "n04004099": 9498, "n04004210": 9499, "n04004475": 9500, "n04004767": 9501, "n04004990": 9502, "n04005197": 9503, "n04005630": 9504, "n04005912": 9505, "n04006067": 9506, "n04006227": 9507, "n04006330": 9508, "n04006411": 9509, "n04007415": 9510, "n04007664": 9511, "n04008385": 9512, "n04008634": 9513, "n04009552": 9514, "n04009801": 9515, "n04009923": 9516, "n04010057": 9517, "n04010779": 9518, "n04010927": 9519, "n04011827": 9520, "n04012084": 9521, "n04012482": 9522, "n04012665": 9523, "n04013060": 9524, "n04013176": 9525, "n04013600": 9526, "n04013729": 9527, "n04014297": 9528, "n04015204": 9529, "n04015786": 9530, "n04015908": 9531, "n04016240": 9532, "n04016479": 9533, "n04016576": 9534, "n04016684": 9535, "n04016846": 9536, "n04017571": 9537, "n04017807": 9538, "n04018155": 9539, "n04018399": 9540, "n04018667": 9541, "n04019101": 9542, "n04019335": 9543, "n04019541": 9544, "n04019696": 9545, "n04019881": 9546, "n04020087": 9547, "n04020298": 9548, "n04020744": 9549, "n04020912": 9550, "n04021028": 9551, "n04021164": 9552, "n04021362": 9553, "n04021503": 9554, "n04021704": 9555, "n04021798": 9556, "n04022332": 9557, "n04022434": 9558, "n04022708": 9559, "n04022866": 9560, "n04023021": 9561, "n04023119": 9562, "n04023249": 9563, "n04023422": 9564, "n04023695": 9565, "n04023962": 9566, "n04024137": 9567, "n04024274": 9568, "n04024862": 9569, "n04024983": 9570, "n04025508": 9571, "n04025633": 9572, "n04026053": 9573, "n04026180": 9574, "n04026417": 9575, "n04026813": 9576, "n04026918": 9577, "n04027023": 9578, "n04027367": 9579, "n04027706": 9580, "n04027820": 9581, "n04027935": 9582, "n04028074": 9583, "n04028221": 9584, "n04028315": 9585, "n04028581": 9586, "n04028764": 9587, "n04029416": 9588, "n04029647": 9589, "n04029734": 9590, "n04029913": 9591, "n04030054": 9592, "n04030161": 9593, "n04030274": 9594, "n04030414": 9595, "n04030518": 9596, "n04030846": 9597, "n04030965": 9598, "n04031884": 9599, "n04032509": 9600, "n04032603": 9601, "n04032936": 9602, "n04033287": 9603, "n04033425": 9604, "n04033557": 9605, "n04033801": 9606, "n04033901": 9607, "n04033995": 9608, "n04034262": 9609, "n04034367": 9610, "n04035231": 9611, "n04035634": 9612, "n04035748": 9613, "n04035836": 9614, "n04035912": 9615, "n04036155": 9616, "n04036303": 9617, "n04036776": 9618, "n04036963": 9619, "n04037076": 9620, "n04037220": 9621, "n04037298": 9622, "n04037443": 9623, "n04037873": 9624, "n04037964": 9625, "n04038231": 9626, "n04038338": 9627, "n04038440": 9628, "n04038727": 9629, "n04039041": 9630, "n04039209": 9631, "n04039381": 9632, "n04039742": 9633, "n04039848": 9634, "n04040247": 9635, "n04040373": 9636, "n04040540": 9637, "n04040759": 9638, "n04041069": 9639, "n04041243": 9640, "n04041408": 9641, "n04041544": 9642, "n04041747": 9643, "n04042076": 9644, "n04042204": 9645, "n04042358": 9646, "n04042632": 9647, "n04042795": 9648, "n04042985": 9649, "n04043168": 9650, "n04043411": 9651, "n04043733": 9652, "n04044307": 9653, "n04044498": 9654, "n04044716": 9655, "n04044955": 9656, "n04045085": 9657, "n04045255": 9658, "n04045397": 9659, "n04045644": 9660, "n04045787": 9661, "n04045941": 9662, "n04046091": 9663, "n04046277": 9664, "n04046400": 9665, "n04046590": 9666, "n04046974": 9667, "n04047139": 9668, "n04047401": 9669, "n04047733": 9670, "n04047834": 9671, "n04048441": 9672, "n04049303": 9673, "n04049405": 9674, "n04049585": 9675, "n04049753": 9676, "n04050066": 9677, "n04050313": 9678, "n04050600": 9679, "n04050933": 9680, "n04051269": 9681, "n04051439": 9682, "n04051549": 9683, "n04051705": 9684, "n04051825": 9685, "n04052235": 9686, "n04052346": 9687, "n04052442": 9688, "n04052658": 9689, "n04052757": 9690, "n04053508": 9691, "n04053677": 9692, "n04053767": 9693, "n04054361": 9694, "n04054566": 9695, "n04054670": 9696, "n04055180": 9697, "n04055447": 9698, "n04055700": 9699, "n04055861": 9700, "n04056073": 9701, "n04056180": 9702, "n04056413": 9703, "n04056932": 9704, "n04057047": 9705, "n04057215": 9706, "n04057435": 9707, "n04057673": 9708, "n04057846": 9709, "n04057981": 9710, "n04058096": 9711, "n04058239": 9712, "n04058486": 9713, "n04058594": 9714, "n04058721": 9715, "n04059157": 9716, "n04059298": 9717, "n04059399": 9718, "n04059516": 9719, "n04059947": 9720, "n04060198": 9721, "n04060448": 9722, "n04060647": 9723, "n04060904": 9724, "n04061681": 9725, "n04061793": 9726, "n04061969": 9727, "n04062179": 9728, "n04062428": 9729, "n04062644": 9730, "n04062807": 9731, "n04063154": 9732, "n04063373": 9733, "n04063868": 9734, "n04064213": 9735, "n04064401": 9736, "n04064747": 9737, "n04064862": 9738, "n04065272": 9739, "n04065464": 9740, "n04065789": 9741, "n04065909": 9742, "n04066023": 9743, "n04066270": 9744, "n04066388": 9745, "n04066476": 9746, "n04066767": 9747, "n04067143": 9748, "n04067231": 9749, "n04067353": 9750, "n04067472": 9751, "n04067658": 9752, "n04067818": 9753, "n04067921": 9754, "n04068441": 9755, "n04068601": 9756, "n04069166": 9757, "n04069276": 9758, "n04069434": 9759, "n04069582": 9760, "n04069777": 9761, "n04070003": 9762, "n04070207": 9763, "n04070415": 9764, "n04070545": 9765, "n04070727": 9766, "n04070964": 9767, "n04071102": 9768, "n04071263": 9769, "n04071393": 9770, "n04072193": 9771, "n04072551": 9772, "n04072960": 9773, "n04073425": 9774, "n04073948": 9775, "n04074185": 9776, "n04074963": 9777, "n04075291": 9778, "n04075468": 9779, "n04075715": 9780, "n04075813": 9781, "n04075916": 9782, "n04076052": 9783, "n04076284": 9784, "n04076713": 9785, "n04077430": 9786, "n04077594": 9787, "n04077734": 9788, "n04077889": 9789, "n04078002": 9790, "n04078574": 9791, "n04078955": 9792, "n04079106": 9793, "n04079244": 9794, "n04079603": 9795, "n04079933": 9796, "n04080138": 9797, "n04080454": 9798, "n04080705": 9799, "n04080833": 9800, "n04081281": 9801, "n04081699": 9802, "n04081844": 9803, "n04082344": 9804, "n04082562": 9805, "n04082710": 9806, "n04082886": 9807, "n04083113": 9808, "n04083309": 9809, "n04083649": 9810, "n04083800": 9811, "n04084517": 9812, "n04084682": 9813, "n04084889": 9814, "n04085017": 9815, "n04085574": 9816, "n04085873": 9817, "n04086066": 9818, "n04086273": 9819, "n04086446": 9820, "n04086663": 9821, "n04086794": 9822, "n04086937": 9823, "n04087126": 9824, "n04087432": 9825, "n04087709": 9826, "n04087826": 9827, "n04088229": 9828, "n04088343": 9829, "n04088441": 9830, "n04088696": 9831, "n04088797": 9832, "n04089152": 9833, "n04089376": 9834, "n04089666": 9835, "n04089836": 9836, "n04089976": 9837, "n04090263": 9838, "n04090548": 9839, "n04090781": 9840, "n04091097": 9841, "n04091466": 9842, "n04091584": 9843, "n04091693": 9844, "n04092168": 9845, "n04093157": 9846, "n04093223": 9847, "n04093625": 9848, "n04093775": 9849, "n04093915": 9850, "n04094060": 9851, "n04094250": 9852, "n04094438": 9853, "n04094608": 9854, "n04094720": 9855, "n04094859": 9856, "n04095109": 9857, "n04095210": 9858, "n04095342": 9859, "n04095577": 9860, "n04095938": 9861, "n04096066": 9862, "n04096733": 9863, "n04096848": 9864, "n04097085": 9865, "n04097373": 9866, "n04097622": 9867, "n04097760": 9868, "n04097866": 9869, "n04098169": 9870, "n04098260": 9871, "n04098399": 9872, "n04098513": 9873, "n04098795": 9874, "n04099003": 9875, "n04099175": 9876, "n04099429": 9877, "n04099969": 9878, "n04100174": 9879, "n04100519": 9880, "n04101375": 9881, "n04101497": 9882, "n04101701": 9883, "n04101860": 9884, "n04102037": 9885, "n04102162": 9886, "n04102285": 9887, "n04102406": 9888, "n04102618": 9889, "n04102760": 9890, "n04102872": 9891, "n04102962": 9892, "n04103094": 9893, "n04103206": 9894, "n04103364": 9895, "n04103665": 9896, "n04103769": 9897, "n04103918": 9898, "n04104147": 9899, "n04104384": 9900, "n04104500": 9901, "n04104770": 9902, "n04104925": 9903, "n04105068": 9904, "n04105438": 9905, "n04105704": 9906, "n04105893": 9907, "n04107598": 9908, "n04107743": 9909, "n04107984": 9910, "n04108268": 9911, "n04108822": 9912, "n04108999": 9913, "n04110068": 9914, "n04110178": 9915, "n04110281": 9916, "n04110439": 9917, "n04110654": 9918, "n04110841": 9919, "n04110955": 9920, "n04111190": 9921, "n04111414": 9922, "n04111531": 9923, "n04111668": 9924, "n04111962": 9925, "n04112147": 9926, "n04112252": 9927, "n04112430": 9928, "n04112579": 9929, "n04112654": 9930, "n04112752": 9931, "n04112921": 9932, "n04113038": 9933, "n04113194": 9934, "n04113316": 9935, "n04113406": 9936, "n04113641": 9937, "n04113765": 9938, "n04113968": 9939, "n04114069": 9940, "n04114301": 9941, "n04114428": 9942, "n04114719": 9943, "n04114844": 9944, "n04114996": 9945, "n04115144": 9946, "n04115256": 9947, "n04115456": 9948, "n04115542": 9949, "n04115802": 9950, "n04115996": 9951, "n04116098": 9952, "n04116294": 9953, "n04116389": 9954, "n04116512": 9955, "n04117216": 9956, "n04117464": 9957, "n04117639": 9958, "n04118021": 9959, "n04118538": 9960, "n04118635": 9961, "n04118776": 9962, "n04119091": 9963, "n04119230": 9964, "n04119360": 9965, "n04119478": 9966, "n04119630": 9967, "n04119751": 9968, "n04120489": 9969, "n04120695": 9970, "n04120842": 9971, "n04121228": 9972, "n04121342": 9973, "n04121426": 9974, "n04121511": 9975, "n04121728": 9976, "n04122262": 9977, "n04122349": 9978, "n04122492": 9979, "n04122578": 9980, "n04122685": 9981, "n04122825": 9982, "n04123026": 9983, "n04123123": 9984, "n04123228": 9985, "n04123317": 9986, "n04123448": 9987, "n04123567": 9988, "n04123740": 9989, "n04124098": 9990, "n04124202": 9991, "n04124370": 9992, "n04124488": 9993, "n04124573": 9994, "n04124887": 9995, "n04125021": 9996, "n04125116": 9997, "n04125257": 9998, "n04125541": 9999, "n04125692": 10000, "n04125853": 10001, "n04126066": 10002, "n04126244": 10003, "n04126541": 10004, "n04126659": 10005, "n04126852": 10006, "n04126980": 10007, "n04127117": 10008, "n04127249": 10009, "n04127395": 10010, "n04127521": 10011, "n04127633": 10012, "n04127904": 10013, "n04128413": 10014, "n04128499": 10015, "n04128710": 10016, "n04128837": 10017, "n04129490": 10018, "n04129688": 10019, "n04129766": 10020, "n04130143": 10021, "n04130257": 10022, "n04130566": 10023, "n04130907": 10024, "n04131015": 10025, "n04131113": 10026, "n04131208": 10027, "n04131368": 10028, "n04131499": 10029, "n04131690": 10030, "n04131811": 10031, "n04131929": 10032, "n04132158": 10033, "n04132465": 10034, "n04132603": 10035, "n04132829": 10036, "n04132985": 10037, "n04133114": 10038, "n04133789": 10039, "n04134008": 10040, "n04134170": 10041, "n04134523": 10042, "n04134632": 10043, "n04135024": 10044, "n04135118": 10045, "n04135315": 10046, "n04135710": 10047, "n04135933": 10048, "n04136045": 10049, "n04136161": 10050, "n04136333": 10051, "n04136510": 10052, "n04136800": 10053, "n04137089": 10054, "n04137217": 10055, "n04137355": 10056, "n04137444": 10057, "n04137773": 10058, "n04137897": 10059, "n04138131": 10060, "n04138261": 10061, "n04138869": 10062, "n04138977": 10063, "n04139140": 10064, "n04139395": 10065, "n04139859": 10066, "n04140064": 10067, "n04140539": 10068, "n04140631": 10069, "n04140777": 10070, "n04140853": 10071, "n04141076": 10072, "n04141198": 10073, "n04141327": 10074, "n04141712": 10075, "n04141838": 10076, "n04141975": 10077, "n04142175": 10078, "n04142327": 10079, "n04142434": 10080, "n04142731": 10081, "n04142999": 10082, "n04143140": 10083, "n04143365": 10084, "n04143897": 10085, "n04144241": 10086, "n04144539": 10087, "n04144651": 10088, "n04145863": 10089, "n04146050": 10090, "n04146343": 10091, "n04146504": 10092, "n04146614": 10093, "n04146862": 10094, "n04146976": 10095, "n04147183": 10096, "n04147291": 10097, "n04147495": 10098, "n04147793": 10099, "n04147916": 10100, "n04148054": 10101, "n04148285": 10102, "n04148464": 10103, "n04148579": 10104, "n04148703": 10105, "n04149083": 10106, "n04149374": 10107, "n04149813": 10108, "n04150153": 10109, "n04150273": 10110, "n04150371": 10111, "n04150980": 10112, "n04151108": 10113, "n04151581": 10114, "n04151940": 10115, "n04152387": 10116, "n04152593": 10117, "n04153025": 10118, "n04153330": 10119, "n04153751": 10120, "n04154152": 10121, "n04154340": 10122, "n04154565": 10123, "n04154753": 10124, "n04154854": 10125, "n04154938": 10126, "n04155068": 10127, "n04155177": 10128, "n04155457": 10129, "n04155625": 10130, "n04155735": 10131, "n04155889": 10132, "n04156040": 10133, "n04156140": 10134, "n04156297": 10135, "n04156411": 10136, "n04156591": 10137, "n04156814": 10138, "n04156946": 10139, "n04157099": 10140, "n04157320": 10141, "n04158002": 10142, "n04158138": 10143, "n04158250": 10144, "n04158672": 10145, "n04158807": 10146, "n04158956": 10147, "n04160036": 10148, "n04160261": 10149, "n04160372": 10150, "n04160586": 10151, "n04160847": 10152, "n04161010": 10153, "n04161358": 10154, "n04161981": 10155, "n04162433": 10156, "n04162706": 10157, "n04163530": 10158, "n04164002": 10159, "n04164199": 10160, "n04164406": 10161, "n04164757": 10162, "n04164868": 10163, "n04165409": 10164, "n04165675": 10165, "n04165945": 10166, "n04166111": 10167, "n04166281": 10168, "n04166436": 10169, "n04167346": 10170, "n04167489": 10171, "n04167661": 10172, "n04168084": 10173, "n04168199": 10174, "n04168472": 10175, "n04168541": 10176, "n04168840": 10177, "n04169437": 10178, "n04169597": 10179, "n04170037": 10180, "n04170384": 10181, "n04170515": 10182, "n04170694": 10183, "n04170933": 10184, "n04171208": 10185, "n04171459": 10186, "n04171629": 10187, "n04171831": 10188, "n04172107": 10189, "n04172230": 10190, "n04172342": 10191, "n04172512": 10192, "n04172607": 10193, "n04172776": 10194, "n04172904": 10195, "n04173046": 10196, "n04173172": 10197, "n04173511": 10198, "n04173907": 10199, "n04174026": 10200, "n04174101": 10201, "n04174234": 10202, "n04174500": 10203, "n04174705": 10204, "n04175039": 10205, "n04175147": 10206, "n04175574": 10207, "n04176068": 10208, "n04176190": 10209, "n04176295": 10210, "n04176528": 10211, "n04177041": 10212, "n04177329": 10213, "n04177545": 10214, "n04177654": 10215, "n04177755": 10216, "n04177820": 10217, "n04177931": 10218, "n04178190": 10219, "n04178329": 10220, "n04178668": 10221, "n04179126": 10222, "n04179712": 10223, "n04179824": 10224, "n04179913": 10225, "n04180063": 10226, "n04180229": 10227, "n04180888": 10228, "n04181083": 10229, "n04181228": 10230, "n04181561": 10231, "n04181718": 10232, "n04182152": 10233, "n04182322": 10234, "n04183217": 10235, "n04183329": 10236, "n04183957": 10237, "n04184095": 10238, "n04184316": 10239, "n04184435": 10240, "n04184600": 10241, "n04184880": 10242, "n04185071": 10243, "n04185529": 10244, "n04185804": 10245, "n04185946": 10246, "n04186051": 10247, "n04186268": 10248, "n04186455": 10249, "n04186624": 10250, "n04186848": 10251, "n04187061": 10252, "n04187233": 10253, "n04187547": 10254, "n04187751": 10255, "n04187885": 10256, "n04187970": 10257, "n04188064": 10258, "n04188179": 10259, "n04189092": 10260, "n04189282": 10261, "n04189651": 10262, "n04189816": 10263, "n04190052": 10264, "n04190376": 10265, "n04190464": 10266, "n04190747": 10267, "n04190997": 10268, "n04191150": 10269, "n04191595": 10270, "n04191943": 10271, "n04192238": 10272, "n04192361": 10273, "n04192521": 10274, "n04192698": 10275, "n04192858": 10276, "n04193179": 10277, "n04193377": 10278, "n04193742": 10279, "n04193883": 10280, "n04194009": 10281, "n04194127": 10282, "n04194289": 10283, "n04196080": 10284, "n04196502": 10285, "n04196803": 10286, "n04196925": 10287, "n04197110": 10288, "n04197391": 10289, "n04197781": 10290, "n04197878": 10291, "n04198015": 10292, "n04198233": 10293, "n04198355": 10294, "n04198453": 10295, "n04198562": 10296, "n04198722": 10297, "n04198797": 10298, "n04199027": 10299, "n04200000": 10300, "n04200258": 10301, "n04200537": 10302, "n04200800": 10303, "n04200908": 10304, "n04201064": 10305, "n04201297": 10306, "n04201733": 10307, "n04202142": 10308, "n04202282": 10309, "n04202417": 10310, "n04203356": 10311, "n04204081": 10312, "n04204238": 10313, "n04204347": 10314, "n04204755": 10315, "n04205062": 10316, "n04205318": 10317, "n04205505": 10318, "n04205613": 10319, "n04206070": 10320, "n04206225": 10321, "n04206356": 10322, "n04206570": 10323, "n04206790": 10324, "n04207151": 10325, "n04207343": 10326, "n04207596": 10327, "n04207763": 10328, "n04207903": 10329, "n04208065": 10330, "n04208210": 10331, "n04208427": 10332, "n04208582": 10333, "n04208760": 10334, "n04208936": 10335, "n04209133": 10336, "n04209239": 10337, "n04209509": 10338, "n04209613": 10339, "n04209811": 10340, "n04210012": 10341, "n04210120": 10342, "n04210288": 10343, "n04210390": 10344, "n04210591": 10345, "n04210858": 10346, "n04211001": 10347, "n04211219": 10348, "n04211356": 10349, "n04211528": 10350, "n04211857": 10351, "n04211970": 10352, "n04212165": 10353, "n04212282": 10354, "n04212467": 10355, "n04212810": 10356, "n04213105": 10357, "n04213264": 10358, "n04213353": 10359, "n04213530": 10360, "n04214046": 10361, "n04214282": 10362, "n04214413": 10363, "n04214649": 10364, "n04215153": 10365, "n04215402": 10366, "n04215588": 10367, "n04215800": 10368, "n04215910": 10369, "n04216634": 10370, "n04216860": 10371, "n04216963": 10372, "n04217387": 10373, "n04217546": 10374, "n04217718": 10375, "n04217882": 10376, "n04218564": 10377, "n04218921": 10378, "n04219185": 10379, "n04219424": 10380, "n04219580": 10381, "n04220250": 10382, "n04220805": 10383, "n04221076": 10384, "n04221673": 10385, "n04221823": 10386, "n04222210": 10387, "n04222307": 10388, "n04222470": 10389, "n04222723": 10390, "n04222847": 10391, "n04223066": 10392, "n04223170": 10393, "n04223299": 10394, "n04224395": 10395, "n04224543": 10396, "n04224842": 10397, "n04225031": 10398, "n04225222": 10399, "n04225729": 10400, "n04225987": 10401, "n04226322": 10402, "n04226464": 10403, "n04226537": 10404, "n04226826": 10405, "n04226962": 10406, "n04227050": 10407, "n04227144": 10408, "n04227519": 10409, "n04227787": 10410, "n04227900": 10411, "n04228054": 10412, "n04228215": 10413, "n04228422": 10414, "n04228581": 10415, "n04228693": 10416, "n04229007": 10417, "n04229107": 10418, "n04229480": 10419, "n04229620": 10420, "n04229737": 10421, "n04229816": 10422, "n04229959": 10423, "n04230387": 10424, "n04230487": 10425, "n04230603": 10426, "n04230707": 10427, "n04230808": 10428, "n04231272": 10429, "n04231693": 10430, "n04231905": 10431, "n04232153": 10432, "n04232312": 10433, "n04232437": 10434, "n04232800": 10435, "n04233027": 10436, "n04233124": 10437, "n04233295": 10438, "n04233715": 10439, "n04233832": 10440, "n04234160": 10441, "n04234260": 10442, "n04234455": 10443, "n04234670": 10444, "n04234763": 10445, "n04234887": 10446, "n04235291": 10447, "n04235646": 10448, "n04235771": 10449, "n04235860": 10450, "n04236001": 10451, "n04236377": 10452, "n04236702": 10453, "n04236809": 10454, "n04236935": 10455, "n04237174": 10456, "n04237287": 10457, "n04237423": 10458, "n04238128": 10459, "n04238321": 10460, "n04238617": 10461, "n04238763": 10462, "n04238953": 10463, "n04239074": 10464, "n04239218": 10465, "n04239333": 10466, "n04239436": 10467, "n04239639": 10468, "n04239786": 10469, "n04239900": 10470, "n04240434": 10471, "n04240752": 10472, "n04240867": 10473, "n04241042": 10474, "n04241249": 10475, "n04241394": 10476, "n04241573": 10477, "n04242084": 10478, "n04242315": 10479, "n04242408": 10480, "n04242587": 10481, "n04242704": 10482, "n04243003": 10483, "n04243142": 10484, "n04243251": 10485, "n04243546": 10486, "n04243941": 10487, "n04244379": 10488, "n04244847": 10489, "n04244997": 10490, "n04245218": 10491, "n04245412": 10492, "n04245508": 10493, "n04245847": 10494, "n04246060": 10495, "n04246271": 10496, "n04246459": 10497, "n04246731": 10498, "n04246855": 10499, "n04247011": 10500, "n04247440": 10501, "n04247544": 10502, "n04247630": 10503, "n04247736": 10504, "n04247876": 10505, "n04248209": 10506, "n04248396": 10507, "n04248507": 10508, "n04248851": 10509, "n04249415": 10510, "n04249582": 10511, "n04249882": 10512, "n04250224": 10513, "n04250473": 10514, "n04250599": 10515, "n04250692": 10516, "n04250850": 10517, "n04251144": 10518, "n04251701": 10519, "n04251791": 10520, "n04252077": 10521, "n04252225": 10522, "n04252331": 10523, "n04252560": 10524, "n04252653": 10525, "n04253057": 10526, "n04253168": 10527, "n04253304": 10528, "n04253931": 10529, "n04254009": 10530, "n04254120": 10531, "n04254450": 10532, "n04254680": 10533, "n04254777": 10534, "n04255163": 10535, "n04255346": 10536, "n04255499": 10537, "n04255586": 10538, "n04255670": 10539, "n04255768": 10540, "n04255899": 10541, "n04256318": 10542, "n04256520": 10543, "n04256758": 10544, "n04256891": 10545, "n04257223": 10546, "n04257684": 10547, "n04257790": 10548, "n04257986": 10549, "n04258138": 10550, "n04258333": 10551, "n04258438": 10552, "n04258618": 10553, "n04258732": 10554, "n04258859": 10555, "n04259202": 10556, "n04259468": 10557, "n04259630": 10558, "n04260192": 10559, "n04260364": 10560, "n04260589": 10561, "n04261116": 10562, "n04261281": 10563, "n04261369": 10564, "n04261506": 10565, "n04261638": 10566, "n04261767": 10567, "n04261868": 10568, "n04262161": 10569, "n04262530": 10570, "n04262678": 10571, "n04262869": 10572, "n04263257": 10573, "n04263336": 10574, "n04263502": 10575, "n04263760": 10576, "n04263950": 10577, "n04264134": 10578, "n04264233": 10579, "n04264361": 10580, "n04264485": 10581, "n04264628": 10582, "n04264765": 10583, "n04264914": 10584, "n04265275": 10585, "n04265428": 10586, "n04265904": 10587, "n04266014": 10588, "n04266162": 10589, "n04266375": 10590, "n04266486": 10591, "n04266849": 10592, "n04266968": 10593, "n04267091": 10594, "n04267165": 10595, "n04267246": 10596, "n04267435": 10597, "n04267577": 10598, "n04267985": 10599, "n04268142": 10600, "n04268275": 10601, "n04268418": 10602, "n04268565": 10603, "n04268799": 10604, "n04269086": 10605, "n04269270": 10606, "n04269502": 10607, "n04269668": 10608, "n04269822": 10609, "n04269944": 10610, "n04270147": 10611, "n04270371": 10612, "n04270576": 10613, "n04270891": 10614, "n04271148": 10615, "n04271531": 10616, "n04271793": 10617, "n04271891": 10618, "n04272054": 10619, "n04272389": 10620, "n04272782": 10621, "n04272928": 10622, "n04273064": 10623, "n04273285": 10624, "n04273569": 10625, "n04273659": 10626, "n04273796": 10627, "n04273972": 10628, "n04274686": 10629, "n04274985": 10630, "n04275093": 10631, "n04275175": 10632, "n04275283": 10633, "n04275548": 10634, "n04275661": 10635, "n04275904": 10636, "n04277352": 10637, "n04277493": 10638, "n04277669": 10639, "n04277826": 10640, "n04278247": 10641, "n04278353": 10642, "n04278447": 10643, "n04278605": 10644, "n04278932": 10645, "n04279063": 10646, "n04279172": 10647, "n04279353": 10648, "n04279462": 10649, "n04279858": 10650, "n04279987": 10651, "n04280259": 10652, "n04280373": 10653, "n04280487": 10654, "n04280845": 10655, "n04280970": 10656, "n04281260": 10657, "n04281375": 10658, "n04281571": 10659, "n04281998": 10660, "n04282231": 10661, "n04282494": 10662, "n04282872": 10663, "n04282992": 10664, "n04283096": 10665, "n04283255": 10666, "n04283378": 10667, "n04283585": 10668, "n04283784": 10669, "n04283905": 10670, "n04284002": 10671, "n04284341": 10672, "n04284438": 10673, "n04284572": 10674, "n04284869": 10675, "n04285008": 10676, "n04285146": 10677, "n04285622": 10678, "n04285803": 10679, "n04285965": 10680, "n04286128": 10681, "n04286575": 10682, "n04286960": 10683, "n04287351": 10684, "n04287451": 10685, "n04287747": 10686, "n04287898": 10687, "n04287986": 10688, "n04288165": 10689, "n04288272": 10690, "n04288533": 10691, "n04288673": 10692, "n04289027": 10693, "n04289195": 10694, "n04289449": 10695, "n04289576": 10696, "n04289690": 10697, "n04289827": 10698, "n04290079": 10699, "n04290259": 10700, "n04290507": 10701, "n04290615": 10702, "n04290762": 10703, "n04291069": 10704, "n04291242": 10705, "n04291759": 10706, "n04291992": 10707, "n04292080": 10708, "n04292221": 10709, "n04292414": 10710, "n04292572": 10711, "n04292921": 10712, "n04293119": 10713, "n04293258": 10714, "n04293744": 10715, "n04294212": 10716, "n04294426": 10717, "n04294614": 10718, "n04294879": 10719, "n04295081": 10720, "n04295353": 10721, "n04295571": 10722, "n04295777": 10723, "n04295881": 10724, "n04296562": 10725, "n04297098": 10726, "n04297750": 10727, "n04297847": 10728, "n04298053": 10729, "n04298661": 10730, "n04298765": 10731, "n04299215": 10732, "n04299370": 10733, "n04299963": 10734, "n04300358": 10735, "n04300509": 10736, "n04300643": 10737, "n04301000": 10738, "n04301242": 10739, "n04301474": 10740, "n04301760": 10741, "n04302200": 10742, "n04302863": 10743, "n04302988": 10744, "n04303095": 10745, "n04303258": 10746, "n04303357": 10747, "n04303497": 10748, "n04304215": 10749, "n04304375": 10750, "n04304680": 10751, "n04305016": 10752, "n04305210": 10753, "n04305323": 10754, "n04305471": 10755, "n04305572": 10756, "n04305947": 10757, "n04306080": 10758, "n04306592": 10759, "n04306847": 10760, "n04307419": 10761, "n04307767": 10762, "n04307878": 10763, "n04307986": 10764, "n04308084": 10765, "n04308273": 10766, "n04308397": 10767, "n04308583": 10768, "n04308807": 10769, "n04308915": 10770, "n04309049": 10771, "n04309348": 10772, "n04309548": 10773, "n04309833": 10774, "n04310018": 10775, "n04310157": 10776, "n04310507": 10777, "n04310604": 10778, "n04310721": 10779, "n04310904": 10780, "n04311004": 10781, "n04311174": 10782, "n04311595": 10783, "n04312020": 10784, "n04312154": 10785, "n04312432": 10786, "n04312654": 10787, "n04312756": 10788, "n04312916": 10789, "n04313220": 10790, "n04313503": 10791, "n04313628": 10792, "n04314107": 10793, "n04314216": 10794, "n04314522": 10795, "n04314632": 10796, "n04314914": 10797, "n04315342": 10798, "n04315713": 10799, "n04315828": 10800, "n04315948": 10801, "n04316498": 10802, "n04316815": 10803, "n04316924": 10804, "n04317063": 10805, "n04317175": 10806, "n04317325": 10807, "n04317420": 10808, "n04317833": 10809, "n04317976": 10810, "n04318131": 10811, "n04318787": 10812, "n04318892": 10813, "n04318982": 10814, "n04319545": 10815, "n04319774": 10816, "n04319937": 10817, "n04320405": 10818, "n04320598": 10819, "n04320871": 10820, "n04320973": 10821, "n04321121": 10822, "n04321453": 10823, "n04322026": 10824, "n04322531": 10825, "n04322692": 10826, "n04322801": 10827, "n04323519": 10828, "n04323819": 10829, "n04324120": 10830, "n04324297": 10831, "n04324387": 10832, "n04324515": 10833, "n04325041": 10834, "n04325208": 10835, "n04325704": 10836, "n04325804": 10837, "n04325968": 10838, "n04326547": 10839, "n04326676": 10840, "n04326799": 10841, "n04326896": 10842, "n04327204": 10843, "n04327544": 10844, "n04327682": 10845, "n04328054": 10846, "n04328186": 10847, "n04328329": 10848, "n04328580": 10849, "n04328703": 10850, "n04328946": 10851, "n04329477": 10852, "n04329681": 10853, "n04329834": 10854, "n04329958": 10855, "n04330109": 10856, "n04330189": 10857, "n04330267": 10858, "n04330340": 10859, "n04330669": 10860, "n04330746": 10861, "n04330896": 10862, "n04330998": 10863, "n04331277": 10864, "n04331443": 10865, "n04331639": 10866, "n04331765": 10867, "n04331892": 10868, "n04332074": 10869, "n04332243": 10870, "n04332580": 10871, "n04332987": 10872, "n04333129": 10873, "n04333869": 10874, "n04334105": 10875, "n04334365": 10876, "n04334504": 10877, "n04334599": 10878, "n04335209": 10879, "n04335435": 10880, "n04335693": 10881, "n04335886": 10882, "n04336792": 10883, "n04337157": 10884, "n04337287": 10885, "n04337503": 10886, "n04337650": 10887, "n04338517": 10888, "n04338963": 10889, "n04339062": 10890, "n04339191": 10891, "n04339638": 10892, "n04339879": 10893, "n04340019": 10894, "n04340521": 10895, "n04340750": 10896, "n04340935": 10897, "n04341133": 10898, "n04341288": 10899, "n04341414": 10900, "n04341686": 10901, "n04343511": 10902, "n04343630": 10903, "n04343740": 10904, "n04344003": 10905, "n04344734": 10906, "n04344873": 10907, "n04345028": 10908, "n04345201": 10909, "n04345787": 10910, "n04346003": 10911, "n04346157": 10912, "n04346328": 10913, "n04346428": 10914, "n04346511": 10915, "n04346679": 10916, "n04346855": 10917, "n04347119": 10918, "n04347519": 10919, "n04347754": 10920, "n04348070": 10921, "n04348184": 10922, "n04348359": 10923, "n04348988": 10924, "n04349189": 10925, "n04349306": 10926, "n04349401": 10927, "n04349913": 10928, "n04350104": 10929, "n04350235": 10930, "n04350458": 10931, "n04350581": 10932, "n04350688": 10933, "n04350769": 10934, "n04350905": 10935, "n04351550": 10936, "n04351699": 10937, "n04353573": 10938, "n04354026": 10939, "n04354182": 10940, "n04354387": 10941, "n04354487": 10942, "n04354589": 10943, "n04355115": 10944, "n04355267": 10945, "n04355338": 10946, "n04355511": 10947, "n04355684": 10948, "n04355821": 10949, "n04355933": 10950, "n04356056": 10951, "n04356595": 10952, "n04356772": 10953, "n04356925": 10954, "n04357121": 10955, "n04357314": 10956, "n04357531": 10957, "n04357930": 10958, "n04358117": 10959, "n04358256": 10960, "n04358491": 10961, "n04358707": 10962, "n04358874": 10963, "n04359034": 10964, "n04359124": 10965, "n04359217": 10966, "n04359335": 10967, "n04359500": 10968, "n04359589": 10969, "n04360501": 10970, "n04360798": 10971, "n04360914": 10972, "n04361095": 10973, "n04361260": 10974, "n04361937": 10975, "n04362624": 10976, "n04362821": 10977, "n04362972": 10978, "n04363082": 10979, "n04363210": 10980, "n04363412": 10981, "n04363671": 10982, "n04363777": 10983, "n04363874": 10984, "n04363991": 10985, "n04364160": 10986, "n04364397": 10987, "n04364545": 10988, "n04364827": 10989, "n04364994": 10990, "n04365112": 10991, "n04365229": 10992, "n04365328": 10993, "n04365484": 10994, "n04365751": 10995, "n04366033": 10996, "n04366116": 10997, "n04366367": 10998, "n04366832": 10999, "n04367011": 11000, "n04367371": 11001, "n04367480": 11002, "n04367746": 11003, "n04367950": 11004, "n04368109": 11005, "n04368235": 11006, "n04368365": 11007, "n04368496": 11008, "n04368695": 11009, "n04368840": 11010, "n04369025": 11011, "n04369282": 11012, "n04369485": 11013, "n04369618": 11014, "n04370048": 11015, "n04370288": 11016, "n04370456": 11017, "n04370600": 11018, "n04370774": 11019, "n04370955": 11020, "n04371050": 11021, "n04371430": 11022, "n04371563": 11023, "n04371774": 11024, "n04371979": 11025, "n04372370": 11026, "n04373089": 11027, "n04373428": 11028, "n04373563": 11029, "n04373704": 11030, "n04373795": 11031, "n04373894": 11032, "n04374315": 11033, "n04374521": 11034, "n04374735": 11035, "n04374907": 11036, "n04375080": 11037, "n04375241": 11038, "n04375405": 11039, "n04375615": 11040, "n04375775": 11041, "n04375926": 11042, "n04376400": 11043, "n04376876": 11044, "n04377057": 11045, "n04378489": 11046, "n04378651": 11047, "n04378956": 11048, "n04379096": 11049, "n04379243": 11050, "n04379964": 11051, "n04380255": 11052, "n04380346": 11053, "n04380533": 11054, "n04380916": 11055, "n04381073": 11056, "n04381450": 11057, "n04381587": 11058, "n04381724": 11059, "n04381860": 11060, "n04381994": 11061, "n04382334": 11062, "n04382438": 11063, "n04382537": 11064, "n04382695": 11065, "n04382880": 11066, "n04383015": 11067, "n04383130": 11068, "n04383301": 11069, "n04383839": 11070, "n04383923": 11071, "n04384593": 11072, "n04384910": 11073, "n04385079": 11074, "n04385157": 11075, "n04385536": 11076, "n04385799": 11077, "n04386051": 11078, "n04386456": 11079, "n04386664": 11080, "n04386792": 11081, "n04387095": 11082, "n04387201": 11083, "n04387261": 11084, "n04387400": 11085, "n04387531": 11086, "n04387706": 11087, "n04387932": 11088, "n04388040": 11089, "n04388162": 11090, "n04388473": 11091, "n04388574": 11092, "n04388743": 11093, "n04389033": 11094, "n04389430": 11095, "n04389521": 11096, "n04389718": 11097, "n04389854": 11098, "n04389999": 11099, "n04390483": 11100, "n04390577": 11101, "n04390873": 11102, "n04390977": 11103, "n04391445": 11104, "n04391838": 11105, "n04392113": 11106, "n04392526": 11107, "n04392764": 11108, "n04392985": 11109, "n04393095": 11110, "n04393301": 11111, "n04393549": 11112, "n04393808": 11113, "n04393913": 11114, "n04394031": 11115, "n04394261": 11116, "n04394421": 11117, "n04394630": 11118, "n04395024": 11119, "n04395106": 11120, "n04395332": 11121, "n04395651": 11122, "n04395875": 11123, "n04396226": 11124, "n04396335": 11125, "n04396650": 11126, "n04396808": 11127, "n04396902": 11128, "n04397027": 11129, "n04397168": 11130, "n04397261": 11131, "n04397452": 11132, "n04397645": 11133, "n04397768": 11134, "n04397860": 11135, "n04398044": 11136, "n04398497": 11137, "n04398688": 11138, "n04398834": 11139, "n04398951": 11140, "n04399046": 11141, "n04399158": 11142, "n04399537": 11143, "n04399846": 11144, "n04400109": 11145, "n04400289": 11146, "n04400499": 11147, "n04400737": 11148, "n04400899": 11149, "n04401088": 11150, "n04401578": 11151, "n04401680": 11152, "n04401828": 11153, "n04401949": 11154, "n04402057": 11155, "n04402342": 11156, "n04402449": 11157, "n04402580": 11158, "n04402746": 11159, "n04402984": 11160, "n04403413": 11161, "n04403524": 11162, "n04403638": 11163, "n04403925": 11164, "n04404072": 11165, "n04404200": 11166, "n04404412": 11167, "n04404817": 11168, "n04404997": 11169, "n04405540": 11170, "n04405762": 11171, "n04405907": 11172, "n04406239": 11173, "n04406552": 11174, "n04406687": 11175, "n04406817": 11176, "n04407257": 11177, "n04407435": 11178, "n04407686": 11179, "n04408871": 11180, "n04409011": 11181, "n04409128": 11182, "n04409279": 11183, "n04409384": 11184, "n04409515": 11185, "n04409625": 11186, "n04409806": 11187, "n04409911": 11188, "n04410086": 11189, "n04410365": 11190, "n04410485": 11191, "n04410565": 11192, "n04410663": 11193, "n04410760": 11194, "n04410886": 11195, "n04411019": 11196, "n04411264": 11197, "n04411835": 11198, "n04411966": 11199, "n04412097": 11200, "n04412300": 11201, "n04412416": 11202, "n04413151": 11203, "n04413419": 11204, "n04413969": 11205, "n04414101": 11206, "n04414199": 11207, "n04414319": 11208, "n04414476": 11209, "n04414675": 11210, "n04414909": 11211, "n04415257": 11212, "n04415663": 11213, "n04415815": 11214, "n04416005": 11215, "n04416901": 11216, "n04417086": 11217, "n04417180": 11218, "n04417361": 11219, "n04417672": 11220, "n04417809": 11221, "n04418357": 11222, "n04418644": 11223, "n04419073": 11224, "n04419642": 11225, "n04419868": 11226, "n04420024": 11227, "n04420720": 11228, "n04421083": 11229, "n04421258": 11230, "n04421417": 11231, "n04421582": 11232, "n04421740": 11233, "n04421872": 11234, "n04422409": 11235, "n04422566": 11236, "n04422727": 11237, "n04422875": 11238, "n04423552": 11239, "n04423687": 11240, "n04423845": 11241, "n04424692": 11242, "n04425804": 11243, "n04425977": 11244, "n04426184": 11245, "n04426316": 11246, "n04426427": 11247, "n04427216": 11248, "n04427473": 11249, "n04427559": 11250, "n04427715": 11251, "n04427857": 11252, "n04428008": 11253, "n04428191": 11254, "n04428382": 11255, "n04428634": 11256, "n04429038": 11257, "n04429376": 11258, "n04430475": 11259, "n04430605": 11260, "n04430896": 11261, "n04431025": 11262, "n04431436": 11263, "n04431648": 11264, "n04431745": 11265, "n04431925": 11266, "n04432043": 11267, "n04432203": 11268, "n04432662": 11269, "n04432785": 11270, "n04433377": 11271, "n04433585": 11272, "n04434207": 11273, "n04434531": 11274, "n04434932": 11275, "n04435180": 11276, "n04435552": 11277, "n04435653": 11278, "n04435759": 11279, "n04435870": 11280, "n04436012": 11281, "n04436185": 11282, "n04436329": 11283, "n04436401": 11284, "n04436542": 11285, "n04436832": 11286, "n04436992": 11287, "n04437276": 11288, "n04437380": 11289, "n04437670": 11290, "n04437953": 11291, "n04438304": 11292, "n04438507": 11293, "n04438643": 11294, "n04438897": 11295, "n04439505": 11296, "n04439585": 11297, "n04439712": 11298, "n04440597": 11299, "n04440963": 11300, "n04441093": 11301, "n04441528": 11302, "n04441662": 11303, "n04441790": 11304, "n04442312": 11305, "n04442441": 11306, "n04442582": 11307, "n04442741": 11308, "n04443164": 11309, "n04443257": 11310, "n04443433": 11311, "n04443766": 11312, "n04444121": 11313, "n04444218": 11314, "n04444749": 11315, "n04444953": 11316, "n04445040": 11317, "n04445154": 11318, "n04445327": 11319, "n04445610": 11320, "n04445782": 11321, "n04445952": 11322, "n04446162": 11323, "n04446276": 11324, "n04446844": 11325, "n04447028": 11326, "n04447156": 11327, "n04447276": 11328, "n04447443": 11329, "n04447861": 11330, "n04448070": 11331, "n04448185": 11332, "n04448361": 11333, "n04449290": 11334, "n04449449": 11335, "n04449550": 11336, "n04449700": 11337, "n04449966": 11338, "n04450133": 11339, "n04450243": 11340, "n04450465": 11341, "n04450640": 11342, "n04450749": 11343, "n04450994": 11344, "n04451139": 11345, "n04451318": 11346, "n04451636": 11347, "n04451818": 11348, "n04452528": 11349, "n04452615": 11350, "n04452757": 11351, "n04452848": 11352, "n04453037": 11353, "n04453156": 11354, "n04453390": 11355, "n04453666": 11356, "n04453910": 11357, "n04454654": 11358, "n04454792": 11359, "n04454908": 11360, "n04455048": 11361, "n04455250": 11362, "n04455579": 11363, "n04455652": 11364, "n04456011": 11365, "n04456115": 11366, "n04456472": 11367, "n04456734": 11368, "n04457157": 11369, "n04457326": 11370, "n04457474": 11371, "n04457638": 11372, "n04457767": 11373, "n04457910": 11374, "n04458201": 11375, "n04458633": 11376, "n04458843": 11377, "n04459018": 11378, "n04459122": 11379, "n04459243": 11380, "n04459362": 11381, "n04459610": 11382, "n04459773": 11383, "n04459909": 11384, "n04460130": 11385, "n04461437": 11386, "n04461570": 11387, "n04461696": 11388, "n04461879": 11389, "n04462011": 11390, "n04462240": 11391, "n04462576": 11392, "n04463679": 11393, "n04464125": 11394, "n04464615": 11395, "n04464852": 11396, "n04465050": 11397, "n04465203": 11398, "n04465358": 11399, "n04465501": 11400, "n04465666": 11401, "n04466871": 11402, "n04467099": 11403, "n04467307": 11404, "n04467506": 11405, "n04467665": 11406, "n04467899": 11407, "n04468005": 11408, "n04469003": 11409, "n04469251": 11410, "n04469514": 11411, "n04469684": 11412, "n04469813": 11413, "n04470741": 11414, "n04471148": 11415, "n04471315": 11416, "n04471632": 11417, "n04471912": 11418, "n04472243": 11419, "n04472563": 11420, "n04472726": 11421, "n04472961": 11422, "n04473108": 11423, "n04473275": 11424, "n04473884": 11425, "n04474035": 11426, "n04474187": 11427, "n04474466": 11428, "n04475309": 11429, "n04475411": 11430, "n04475496": 11431, "n04475631": 11432, "n04475749": 11433, "n04475900": 11434, "n04476116": 11435, "n04476259": 11436, "n04476526": 11437, "n04476831": 11438, "n04476972": 11439, "n04477219": 11440, "n04477387": 11441, "n04477548": 11442, "n04477725": 11443, "n04478066": 11444, "n04478383": 11445, "n04478512": 11446, "n04478657": 11447, "n04479046": 11448, "n04479287": 11449, "n04479405": 11450, "n04479526": 11451, "n04479694": 11452, "n04479823": 11453, "n04479939": 11454, "n04480033": 11455, "n04480141": 11456, "n04480303": 11457, "n04480527": 11458, "n04480853": 11459, "n04480995": 11460, "n04481524": 11461, "n04481642": 11462, "n04482177": 11463, "n04482297": 11464, "n04482393": 11465, "n04482975": 11466, "n04483073": 11467, "n04483307": 11468, "n04483925": 11469, "n04484024": 11470, "n04484432": 11471, "n04485082": 11472, "n04485423": 11473, "n04485586": 11474, "n04485750": 11475, "n04485884": 11476, "n04486054": 11477, "n04486213": 11478, "n04486322": 11479, "n04486616": 11480, "n04486934": 11481, "n04487081": 11482, "n04487394": 11483, "n04487724": 11484, "n04487894": 11485, "n04488202": 11486, "n04488427": 11487, "n04488530": 11488, "n04488742": 11489, "n04488857": 11490, "n04489008": 11491, "n04489695": 11492, "n04489817": 11493, "n04490091": 11494, "n04491312": 11495, "n04491388": 11496, "n04491638": 11497, "n04491769": 11498, "n04491934": 11499, "n04492060": 11500, "n04492157": 11501, "n04492375": 11502, "n04492749": 11503, "n04493109": 11504, "n04493259": 11505, "n04493381": 11506, "n04494204": 11507, "n04495051": 11508, "n04495183": 11509, "n04495310": 11510, "n04495450": 11511, "n04495555": 11512, "n04495698": 11513, "n04495843": 11514, "n04496614": 11515, "n04496726": 11516, "n04496872": 11517, "n04497249": 11518, "n04497442": 11519, "n04497570": 11520, "n04497801": 11521, "n04498275": 11522, "n04498389": 11523, "n04498523": 11524, "n04498873": 11525, "n04499062": 11526, "n04499300": 11527, "n04499446": 11528, "n04499554": 11529, "n04499810": 11530, "n04500060": 11531, "n04500390": 11532, "n04501127": 11533, "n04501281": 11534, "n04501370": 11535, "n04501550": 11536, "n04501837": 11537, "n04501947": 11538, "n04502059": 11539, "n04502197": 11540, "n04502502": 11541, "n04502670": 11542, "n04502851": 11543, "n04502989": 11544, "n04503073": 11545, "n04503155": 11546, "n04503269": 11547, "n04503413": 11548, "n04503499": 11549, "n04503593": 11550, "n04503705": 11551, "n04504038": 11552, "n04504141": 11553, "n04504770": 11554, "n04505036": 11555, "n04505345": 11556, "n04505470": 11557, "n04505888": 11558, "n04506289": 11559, "n04506402": 11560, "n04506506": 11561, "n04506688": 11562, "n04506895": 11563, "n04506994": 11564, "n04507155": 11565, "n04507326": 11566, "n04507453": 11567, "n04507689": 11568, "n04508163": 11569, "n04508489": 11570, "n04508949": 11571, "n04509171": 11572, "n04509260": 11573, "n04509417": 11574, "n04509592": 11575, "n04510706": 11576, "n04511002": 11577, "n04513827": 11578, "n04513998": 11579, "n04514095": 11580, "n04514241": 11581, "n04514648": 11582, "n04515003": 11583, "n04515444": 11584, "n04515729": 11585, "n04515890": 11586, "n04516116": 11587, "n04516214": 11588, "n04516354": 11589, "n04516672": 11590, "n04517211": 11591, "n04517408": 11592, "n04517823": 11593, "n04517999": 11594, "n04518132": 11595, "n04518343": 11596, "n04518643": 11597, "n04518764": 11598, "n04519153": 11599, "n04519536": 11600, "n04519728": 11601, "n04519887": 11602, "n04520170": 11603, "n04520382": 11604, "n04520784": 11605, "n04520962": 11606, "n04521571": 11607, "n04521863": 11608, "n04521987": 11609, "n04522168": 11610, "n04523525": 11611, "n04523831": 11612, "n04524142": 11613, "n04524313": 11614, "n04524594": 11615, "n04524716": 11616, "n04524941": 11617, "n04525038": 11618, "n04525191": 11619, "n04525305": 11620, "n04525417": 11621, "n04525584": 11622, "n04525821": 11623, "n04526520": 11624, "n04526800": 11625, "n04526964": 11626, "n04527648": 11627, "n04528079": 11628, "n04528968": 11629, "n04529108": 11630, "n04529681": 11631, "n04529962": 11632, "n04530283": 11633, "n04530456": 11634, "n04530566": 11635, "n04531098": 11636, "n04531873": 11637, "n04532022": 11638, "n04532106": 11639, "n04532398": 11640, "n04532504": 11641, "n04532670": 11642, "n04532831": 11643, "n04533042": 11644, "n04533199": 11645, "n04533499": 11646, "n04533594": 11647, "n04533700": 11648, "n04533802": 11649, "n04533946": 11650, "n04534127": 11651, "n04534359": 11652, "n04534520": 11653, "n04534895": 11654, "n04535252": 11655, "n04535370": 11656, "n04535524": 11657, "n04536153": 11658, "n04536335": 11659, "n04536465": 11660, "n04536595": 11661, "n04536765": 11662, "n04536866": 11663, "n04537436": 11664, "n04538249": 11665, "n04538403": 11666, "n04538552": 11667, "n04538878": 11668, "n04539053": 11669, "n04539203": 11670, "n04539407": 11671, "n04539794": 11672, "n04540053": 11673, "n04540255": 11674, "n04540397": 11675, "n04540761": 11676, "n04541136": 11677, "n04541320": 11678, "n04541662": 11679, "n04541777": 11680, "n04541987": 11681, "n04542095": 11682, "n04542329": 11683, "n04542474": 11684, "n04542595": 11685, "n04542715": 11686, "n04542858": 11687, "n04542943": 11688, "n04543158": 11689, "n04543509": 11690, "n04543636": 11691, "n04543772": 11692, "n04543924": 11693, "n04543996": 11694, "n04544325": 11695, "n04544450": 11696, "n04545305": 11697, "n04545471": 11698, "n04545748": 11699, "n04545858": 11700, "n04545984": 11701, "n04546081": 11702, "n04546194": 11703, "n04546340": 11704, "n04546595": 11705, "n04546855": 11706, "n04547592": 11707, "n04548280": 11708, "n04548362": 11709, "n04549028": 11710, "n04549122": 11711, "n04549629": 11712, "n04549721": 11713, "n04549919": 11714, "n04550184": 11715, "n04550676": 11716, "n04551055": 11717, "n04551833": 11718, "n04552097": 11719, "n04552348": 11720, "n04552551": 11721, "n04552696": 11722, "n04553389": 11723, "n04553561": 11724, "n04553703": 11725, "n04554211": 11726, "n04554406": 11727, "n04554684": 11728, "n04554871": 11729, "n04554998": 11730, "n04555291": 11731, "n04555400": 11732, "n04555600": 11733, "n04555700": 11734, "n04555897": 11735, "n04556408": 11736, "n04556533": 11737, "n04556664": 11738, "n04556948": 11739, "n04557308": 11740, "n04557522": 11741, "n04557648": 11742, "n04557751": 11743, "n04558059": 11744, "n04558199": 11745, "n04558478": 11746, "n04558804": 11747, "n04559023": 11748, "n04559166": 11749, "n04559451": 11750, "n04559620": 11751, "n04559730": 11752, "n04559910": 11753, "n04559994": 11754, "n04560113": 11755, "n04560292": 11756, "n04560502": 11757, "n04560619": 11758, "n04560804": 11759, "n04560882": 11760, "n04561010": 11761, "n04561287": 11762, "n04561422": 11763, "n04561734": 11764, "n04561857": 11765, "n04561965": 11766, "n04562122": 11767, "n04562262": 11768, "n04562496": 11769, "n04562935": 11770, "n04563020": 11771, "n04563204": 11772, "n04563413": 11773, "n04563560": 11774, "n04563790": 11775, "n04564278": 11776, "n04564581": 11777, "n04565039": 11778, "n04565375": 11779, "n04566257": 11780, "n04566561": 11781, "n04566756": 11782, "n04567098": 11783, "n04567593": 11784, "n04567746": 11785, "n04568069": 11786, "n04568557": 11787, "n04568713": 11788, "n04568841": 11789, "n04569063": 11790, "n04569520": 11791, "n04569822": 11792, "n04570118": 11793, "n04570214": 11794, "n04570416": 11795, "n04570532": 11796, "n04570815": 11797, "n04570958": 11798, "n04571292": 11799, "n04571566": 11800, "n04571686": 11801, "n04571800": 11802, "n04571958": 11803, "n04572121": 11804, "n04572235": 11805, "n04572935": 11806, "n04573045": 11807, "n04573281": 11808, "n04573379": 11809, "n04573513": 11810, "n04573625": 11811, "n04573832": 11812, "n04573937": 11813, "n04574067": 11814, "n04574348": 11815, "n04574471": 11816, "n04574606": 11817, "n04574999": 11818, "n04575723": 11819, "n04575824": 11820, "n04576002": 11821, "n04576211": 11822, "n04576971": 11823, "n04577139": 11824, "n04577293": 11825, "n04577426": 11826, "n04577567": 11827, "n04577769": 11828, "n04578112": 11829, "n04578329": 11830, "n04578559": 11831, "n04578708": 11832, "n04578801": 11833, "n04578934": 11834, "n04579056": 11835, "n04579145": 11836, "n04579230": 11837, "n04579432": 11838, "n04579667": 11839, "n04579986": 11840, "n04580493": 11841, "n04581102": 11842, "n04581595": 11843, "n04581829": 11844, "n04582205": 11845, "n04582349": 11846, "n04582771": 11847, "n04582869": 11848, "n04583022": 11849, "n04583212": 11850, "n04583620": 11851, "n04583888": 11852, "n04583967": 11853, "n04584056": 11854, "n04584207": 11855, "n04584373": 11856, "n04585128": 11857, "n04585318": 11858, "n04585456": 11859, "n04585626": 11860, "n04585745": 11861, "n04585980": 11862, "n04586072": 11863, "n04586581": 11864, "n04586932": 11865, "n04587327": 11866, "n04587404": 11867, "n04587559": 11868, "n04587648": 11869, "n04588739": 11870, "n04589190": 11871, "n04589325": 11872, "n04589434": 11873, "n04589593": 11874, "n04589890": 11875, "n04590021": 11876, "n04590129": 11877, "n04590263": 11878, "n04590553": 11879, "n04590746": 11880, "n04590933": 11881, "n04591056": 11882, "n04591157": 11883, "n04591249": 11884, "n04591359": 11885, "n04591517": 11886, "n04591631": 11887, "n04591713": 11888, "n04591887": 11889, "n04592005": 11890, "n04592099": 11891, "n04592356": 11892, "n04592465": 11893, "n04592596": 11894, "n04592741": 11895, "n04593077": 11896, "n04593185": 11897, "n04593376": 11898, "n04593524": 11899, "n04593629": 11900, "n04593866": 11901, "n04594114": 11902, "n04594218": 11903, "n04594489": 11904, "n04594742": 11905, "n04594828": 11906, "n04594919": 11907, "n04595028": 11908, "n04595285": 11909, "n04595501": 11910, "n04595611": 11911, "n04595762": 11912, "n04595855": 11913, "n04596116": 11914, "n04596492": 11915, "n04596742": 11916, "n04596852": 11917, "n04597066": 11918, "n04597309": 11919, "n04597400": 11920, "n04597804": 11921, "n04597913": 11922, "n04598136": 11923, "n04598318": 11924, "n04598416": 11925, "n04598582": 11926, "n04598965": 11927, "n04599124": 11928, "n04599235": 11929, "n04600312": 11930, "n04600486": 11931, "n04600912": 11932, "n04601041": 11933, "n04601159": 11934, "n04601938": 11935, "n04602762": 11936, "n04602840": 11937, "n04602956": 11938, "n04603399": 11939, "n04603729": 11940, "n04603872": 11941, "n04604276": 11942, "n04604644": 11943, "n04604806": 11944, "n04605057": 11945, "n04605163": 11946, "n04605321": 11947, "n04605446": 11948, "n04605572": 11949, "n04605726": 11950, "n04606251": 11951, "n04606574": 11952, "n04607035": 11953, "n04607242": 11954, "n04607640": 11955, "n04607759": 11956, "n04607869": 11957, "n04607982": 11958, "n04608329": 11959, "n04608435": 11960, "n04608567": 11961, "n04608809": 11962, "n04608923": 11963, "n04609531": 11964, "n04609651": 11965, "n04609811": 11966, "n04610013": 11967, "n04610176": 11968, "n04610274": 11969, "n04610503": 11970, "n04610676": 11971, "n04611351": 11972, "n04611795": 11973, "n04611916": 11974, "n04612026": 11975, "n04612159": 11976, "n04612257": 11977, "n04612373": 11978, "n04612504": 11979, "n04612840": 11980, "n04613015": 11981, "n04613158": 11982, "n04613696": 11983, "n04613939": 11984, "n04614505": 11985, "n04614655": 11986, "n04614844": 11987, "n04615149": 11988, "n04615226": 11989, "n04615644": 11990, "n04682018": 11991, "n04950713": 11992, "n04950952": 11993, "n04951071": 11994, "n04951186": 11995, "n04951373": 11996, "n04951716": 11997, "n04951875": 11998, "n04953296": 11999, "n04953678": 12000, "n04955160": 12001, "n04957356": 12002, "n04957589": 12003, "n04958634": 12004, "n04958865": 12005, "n04959061": 12006, "n04959230": 12007, "n04959672": 12008, "n04960277": 12009, "n04960582": 12010, "n04961062": 12011, "n04961331": 12012, "n04961691": 12013, "n04962062": 12014, "n04962240": 12015, "n04963111": 12016, "n04963307": 12017, "n04963588": 12018, "n04963740": 12019, "n04964001": 12020, "n04964799": 12021, "n04964878": 12022, "n04965179": 12023, "n04965451": 12024, "n04965661": 12025, "n04966543": 12026, "n04966941": 12027, "n04967191": 12028, "n04967561": 12029, "n04967674": 12030, "n04967801": 12031, "n04967882": 12032, "n04968056": 12033, "n04968139": 12034, "n04968749": 12035, "n04968895": 12036, "n04969242": 12037, "n04969540": 12038, "n04969798": 12039, "n04969952": 12040, "n04970059": 12041, "n04970312": 12042, "n04970398": 12043, "n04970470": 12044, "n04970631": 12045, "n04970916": 12046, "n04971211": 12047, "n04971313": 12048, "n04972350": 12049, "n04972451": 12050, "n04972801": 12051, "n04973020": 12052, "n04973291": 12053, "n04973386": 12054, "n04973585": 12055, "n04973669": 12056, "n04973816": 12057, "n04974145": 12058, "n04974340": 12059, "n04974859": 12060, "n04975739": 12061, "n04976319": 12062, "n04976952": 12063, "n04977412": 12064, "n04978561": 12065, "n04979002": 12066, "n04979307": 12067, "n04981658": 12068, "n05102764": 12069, "n05218119": 12070, "n05233741": 12071, "n05235879": 12072, "n05238282": 12073, "n05239437": 12074, "n05241218": 12075, "n05241485": 12076, "n05241662": 12077, "n05242070": 12078, "n05242239": 12079, "n05242928": 12080, "n05244421": 12081, "n05244755": 12082, "n05244934": 12083, "n05245192": 12084, "n05257476": 12085, "n05257967": 12086, "n05258051": 12087, "n05258627": 12088, "n05259914": 12089, "n05260127": 12090, "n05260240": 12091, "n05261310": 12092, "n05262422": 12093, "n05262534": 12094, "n05262698": 12095, "n05263183": 12096, "n05263316": 12097, "n05263448": 12098, "n05265736": 12099, "n05266096": 12100, "n05266879": 12101, "n05278922": 12102, "n05279953": 12103, "n05282652": 12104, "n05285623": 12105, "n05302499": 12106, "n05314075": 12107, "n05399034": 12108, "n05399243": 12109, "n05399356": 12110, "n05418717": 12111, "n05427346": 12112, "n05442594": 12113, "n05447757": 12114, "n05448704": 12115, "n05448827": 12116, "n05449196": 12117, "n05449661": 12118, "n05449959": 12119, "n05450617": 12120, "n05451099": 12121, "n05451384": 12122, "n05453412": 12123, "n05453657": 12124, "n05453815": 12125, "n05454833": 12126, "n05454978": 12127, "n05455113": 12128, "n05458173": 12129, "n05458576": 12130, "n05459101": 12131, "n05459457": 12132, "n05459769": 12133, "n05460759": 12134, "n05464534": 12135, "n05467054": 12136, "n05467758": 12137, "n05468098": 12138, "n05468739": 12139, "n05469664": 12140, "n05469861": 12141, "n05475397": 12142, "n05482922": 12143, "n05486510": 12144, "n05491154": 12145, "n05526957": 12146, "n05538625": 12147, "n05539947": 12148, "n05541509": 12149, "n05542893": 12150, "n05545879": 12151, "n05571341": 12152, "n05578095": 12153, "n05581932": 12154, "n05584746": 12155, "n05586759": 12156, "n05604434": 12157, "n05716342": 12158, "n06008896": 12159, "n06209940": 12160, "n06254669": 12161, "n06255081": 12162, "n06255613": 12163, "n06259898": 12164, "n06262567": 12165, "n06262943": 12166, "n06263202": 12167, "n06263369": 12168, "n06263609": 12169, "n06263762": 12170, "n06263895": 12171, "n06266417": 12172, "n06266633": 12173, "n06266710": 12174, "n06266878": 12175, "n06266973": 12176, "n06267145": 12177, "n06267564": 12178, "n06267655": 12179, "n06267758": 12180, "n06267893": 12181, "n06267991": 12182, "n06271778": 12183, "n06272290": 12184, "n06272612": 12185, "n06272803": 12186, "n06273207": 12187, "n06273294": 12188, "n06273414": 12189, "n06273555": 12190, "n06273743": 12191, "n06273890": 12192, "n06273986": 12193, "n06274092": 12194, "n06274292": 12195, "n06274546": 12196, "n06274760": 12197, "n06274921": 12198, "n06275095": 12199, "n06275353": 12200, "n06275471": 12201, "n06276501": 12202, "n06276697": 12203, "n06276902": 12204, "n06277025": 12205, "n06277135": 12206, "n06277280": 12207, "n06278338": 12208, "n06278475": 12209, "n06281040": 12210, "n06281175": 12211, "n06340977": 12212, "n06359193": 12213, "n06359467": 12214, "n06359657": 12215, "n06415688": 12216, "n06417096": 12217, "n06418693": 12218, "n06419354": 12219, "n06423496": 12220, "n06470073": 12221, "n06591815": 12222, "n06592078": 12223, "n06592281": 12224, "n06592421": 12225, "n06595351": 12226, "n06596179": 12227, "n06596364": 12228, "n06596474": 12229, "n06596607": 12230, "n06596727": 12231, "n06596845": 12232, "n06613686": 12233, "n06614901": 12234, "n06616216": 12235, "n06618653": 12236, "n06625062": 12237, "n06785654": 12238, "n06793231": 12239, "n06794110": 12240, "n06874185": 12241, "n06883725": 12242, "n06892775": 12243, "n06998748": 12244, "n07005523": 12245, "n07248320": 12246, "n07273802": 12247, "n07461050": 12248, "n07556406": 12249, "n07556637": 12250, "n07556872": 12251, "n07556970": 12252, "n07557165": 12253, "n07557434": 12254, "n07560193": 12255, "n07560331": 12256, "n07560422": 12257, "n07560542": 12258, "n07560652": 12259, "n07560903": 12260, "n07561112": 12261, "n07561590": 12262, "n07561848": 12263, "n07562017": 12264, "n07562172": 12265, "n07562379": 12266, "n07562495": 12267, "n07562651": 12268, "n07562881": 12269, "n07562984": 12270, "n07563207": 12271, "n07563366": 12272, "n07563642": 12273, "n07563800": 12274, "n07564008": 12275, "n07564101": 12276, "n07564292": 12277, "n07564515": 12278, "n07564629": 12279, "n07564796": 12280, "n07564971": 12281, "n07565083": 12282, "n07565161": 12283, "n07565259": 12284, "n07565608": 12285, "n07565725": 12286, "n07565945": 12287, "n07566092": 12288, "n07566231": 12289, "n07566340": 12290, "n07566863": 12291, "n07567039": 12292, "n07567139": 12293, "n07567390": 12294, "n07567611": 12295, "n07567707": 12296, "n07567980": 12297, "n07568095": 12298, "n07568241": 12299, "n07568389": 12300, "n07568502": 12301, "n07568625": 12302, "n07568818": 12303, "n07568991": 12304, "n07569106": 12305, "n07569423": 12306, "n07569543": 12307, "n07569644": 12308, "n07569873": 12309, "n07570021": 12310, "n07570530": 12311, "n07570720": 12312, "n07572353": 12313, "n07572616": 12314, "n07572858": 12315, "n07572957": 12316, "n07573103": 12317, "n07573347": 12318, "n07573453": 12319, "n07573563": 12320, "n07573696": 12321, "n07574176": 12322, "n07574426": 12323, "n07574504": 12324, "n07574602": 12325, "n07574780": 12326, "n07574923": 12327, "n07575076": 12328, "n07575226": 12329, "n07575392": 12330, "n07575510": 12331, "n07575726": 12332, "n07575984": 12333, "n07576182": 12334, "n07576438": 12335, "n07576577": 12336, "n07576781": 12337, "n07576969": 12338, "n07577144": 12339, "n07577374": 12340, "n07577538": 12341, "n07577657": 12342, "n07577772": 12343, "n07577918": 12344, "n07578093": 12345, "n07579575": 12346, "n07579688": 12347, "n07579787": 12348, "n07579917": 12349, "n07580053": 12350, "n07580253": 12351, "n07580359": 12352, "n07580470": 12353, "n07580592": 12354, "n07581249": 12355, "n07581346": 12356, "n07581607": 12357, "n07581775": 12358, "n07581931": 12359, "n07582027": 12360, "n07582152": 12361, "n07582277": 12362, "n07582441": 12363, "n07582609": 12364, "n07582811": 12365, "n07582892": 12366, "n07582970": 12367, "n07583066": 12368, "n07583197": 12369, "n07583865": 12370, "n07583978": 12371, "n07584110": 12372, "n07584228": 12373, "n07584332": 12374, "n07584423": 12375, "n07584593": 12376, "n07584859": 12377, "n07584938": 12378, "n07585015": 12379, "n07585107": 12380, "n07585208": 12381, "n07585474": 12382, "n07585557": 12383, "n07585644": 12384, "n07585758": 12385, "n07585906": 12386, "n07585997": 12387, "n07586099": 12388, "n07586179": 12389, "n07586318": 12390, "n07586485": 12391, "n07586604": 12392, "n07586718": 12393, "n07586894": 12394, "n07587023": 12395, "n07587111": 12396, "n07587206": 12397, "n07587331": 12398, "n07587441": 12399, "n07587618": 12400, "n07587700": 12401, "n07587819": 12402, "n07587962": 12403, "n07588111": 12404, "n07588193": 12405, "n07588299": 12406, "n07588419": 12407, "n07588574": 12408, "n07588688": 12409, "n07588817": 12410, "n07588947": 12411, "n07589458": 12412, "n07589543": 12413, "n07589724": 12414, "n07589872": 12415, "n07589967": 12416, "n07590068": 12417, "n07590177": 12418, "n07590320": 12419, "n07590502": 12420, "n07590611": 12421, "n07590752": 12422, "n07590841": 12423, "n07590974": 12424, "n07591049": 12425, "n07591162": 12426, "n07591236": 12427, "n07591330": 12428, "n07591473": 12429, "n07591586": 12430, "n07591813": 12431, "n07591961": 12432, "n07592094": 12433, "n07592317": 12434, "n07592400": 12435, "n07592481": 12436, "n07592656": 12437, "n07592768": 12438, "n07592922": 12439, "n07593004": 12440, "n07593107": 12441, "n07593199": 12442, "n07593471": 12443, "n07593774": 12444, "n07593972": 12445, "n07594066": 12446, "n07594155": 12447, "n07594250": 12448, "n07594737": 12449, "n07594840": 12450, "n07595051": 12451, "n07595180": 12452, "n07595368": 12453, "n07595649": 12454, "n07595751": 12455, "n07595914": 12456, "n07596046": 12457, "n07596160": 12458, "n07596362": 12459, "n07596452": 12460, "n07596566": 12461, "n07596684": 12462, "n07596967": 12463, "n07597145": 12464, "n07597263": 12465, "n07597365": 12466, "n07598256": 12467, "n07598529": 12468, "n07598622": 12469, "n07598734": 12470, "n07598928": 12471, "n07599068": 12472, "n07599161": 12473, "n07599242": 12474, "n07599383": 12475, "n07599468": 12476, "n07599554": 12477, "n07599649": 12478, "n07599783": 12479, "n07599911": 12480, "n07599998": 12481, "n07600177": 12482, "n07600285": 12483, "n07600394": 12484, "n07600506": 12485, "n07600696": 12486, "n07600895": 12487, "n07601025": 12488, "n07601175": 12489, "n07601290": 12490, "n07601407": 12491, "n07601572": 12492, "n07601686": 12493, "n07601809": 12494, "n07602650": 12495, "n07604956": 12496, "n07605040": 12497, "n07605198": 12498, "n07605282": 12499, "n07605380": 12500, "n07605474": 12501, "n07605597": 12502, "n07605693": 12503, "n07605804": 12504, "n07605944": 12505, "n07606058": 12506, "n07606191": 12507, "n07606278": 12508, "n07606419": 12509, "n07606538": 12510, "n07606669": 12511, "n07606764": 12512, "n07606933": 12513, "n07607027": 12514, "n07607138": 12515, "n07607361": 12516, "n07607492": 12517, "n07607605": 12518, "n07607707": 12519, "n07607832": 12520, "n07607967": 12521, "n07608098": 12522, "n07608245": 12523, "n07608339": 12524, "n07608429": 12525, "n07608533": 12526, "n07608641": 12527, "n07608721": 12528, "n07608866": 12529, "n07608980": 12530, "n07609083": 12531, "n07609215": 12532, "n07609316": 12533, "n07609407": 12534, "n07609549": 12535, "n07609632": 12536, "n07609728": 12537, "n07609840": 12538, "n07610295": 12539, "n07610502": 12540, "n07610620": 12541, "n07610746": 12542, "n07610890": 12543, "n07611046": 12544, "n07611148": 12545, "n07611267": 12546, "n07611358": 12547, "n07611733": 12548, "n07611839": 12549, "n07611991": 12550, "n07612137": 12551, "n07612273": 12552, "n07612367": 12553, "n07612530": 12554, "n07612632": 12555, "n07612996": 12556, "n07613158": 12557, "n07613266": 12558, "n07613480": 12559, "n07613671": 12560, "n07613815": 12561, "n07614103": 12562, "n07614198": 12563, "n07614348": 12564, "n07614500": 12565, "n07614730": 12566, "n07614825": 12567, "n07615052": 12568, "n07615190": 12569, "n07615289": 12570, "n07615460": 12571, "n07615569": 12572, "n07615671": 12573, "n07615774": 12574, "n07615954": 12575, "n07616046": 12576, "n07616174": 12577, "n07616265": 12578, "n07616386": 12579, "n07616487": 12580, "n07616590": 12581, "n07616748": 12582, "n07616906": 12583, "n07617051": 12584, "n07617188": 12585, "n07617344": 12586, "n07617447": 12587, "n07617526": 12588, "n07617611": 12589, "n07617708": 12590, "n07617839": 12591, "n07617932": 12592, "n07618029": 12593, "n07618119": 12594, "n07618281": 12595, "n07618432": 12596, "n07618587": 12597, "n07618684": 12598, "n07618871": 12599, "n07619004": 12600, "n07619208": 12601, "n07619301": 12602, "n07619409": 12603, "n07619508": 12604, "n07619881": 12605, "n07620047": 12606, "n07620145": 12607, "n07620327": 12608, "n07620597": 12609, "n07620689": 12610, "n07621264": 12611, "n07621497": 12612, "n07621618": 12613, "n07623136": 12614, "n07624466": 12615, "n07624666": 12616, "n07624757": 12617, "n07624924": 12618, "n07625061": 12619, "n07625324": 12620, "n07627931": 12621, "n07628068": 12622, "n07628181": 12623, "n07631926": 12624, "n07639069": 12625, "n07641928": 12626, "n07642361": 12627, "n07642471": 12628, "n07642742": 12629, "n07642833": 12630, "n07642933": 12631, "n07643026": 12632, "n07643200": 12633, "n07643306": 12634, "n07643474": 12635, "n07643577": 12636, "n07643679": 12637, "n07643764": 12638, "n07643891": 12639, "n07643981": 12640, "n07644244": 12641, "n07648913": 12642, "n07648997": 12643, "n07650792": 12644, "n07650903": 12645, "n07651025": 12646, "n07654148": 12647, "n07654298": 12648, "n07655067": 12649, "n07655263": 12650, "n07663899": 12651, "n07665438": 12652, "n07666176": 12653, "n07672914": 12654, "n07678586": 12655, "n07678729": 12656, "n07678953": 12657, "n07679034": 12658, "n07679140": 12659, "n07679356": 12660, "n07680168": 12661, "n07680313": 12662, "n07680416": 12663, "n07680517": 12664, "n07680655": 12665, "n07680761": 12666, "n07680932": 12667, "n07681264": 12668, "n07681355": 12669, "n07681450": 12670, "n07681691": 12671, "n07681805": 12672, "n07681926": 12673, "n07682197": 12674, "n07682316": 12675, "n07682477": 12676, "n07682624": 12677, "n07682808": 12678, "n07682952": 12679, "n07683039": 12680, "n07683138": 12681, "n07683265": 12682, "n07683360": 12683, "n07683490": 12684, "n07683617": 12685, "n07683786": 12686, "n07684084": 12687, "n07684164": 12688, "n07684289": 12689, "n07684422": 12690, "n07684517": 12691, "n07684600": 12692, "n07684938": 12693, "n07685031": 12694, "n07685118": 12695, "n07685218": 12696, "n07685303": 12697, "n07685399": 12698, "n07685546": 12699, "n07685730": 12700, "n07685918": 12701, "n07686021": 12702, "n07686202": 12703, "n07686299": 12704, "n07686461": 12705, "n07686634": 12706, "n07686720": 12707, "n07686873": 12708, "n07687053": 12709, "n07687211": 12710, "n07687381": 12711, "n07687469": 12712, "n07687626": 12713, "n07687789": 12714, "n07688021": 12715, "n07688130": 12716, "n07688265": 12717, "n07688412": 12718, "n07688624": 12719, "n07688757": 12720, "n07688898": 12721, "n07689003": 12722, "n07689217": 12723, "n07689313": 12724, "n07689490": 12725, "n07689624": 12726, "n07689757": 12727, "n07689842": 12728, "n07690019": 12729, "n07690152": 12730, "n07690273": 12731, "n07690431": 12732, "n07690511": 12733, "n07690585": 12734, "n07690739": 12735, "n07690892": 12736, "n07691091": 12737, "n07691237": 12738, "n07691539": 12739, "n07691650": 12740, "n07691758": 12741, "n07691863": 12742, "n07691954": 12743, "n07692114": 12744, "n07692248": 12745, "n07692405": 12746, "n07692517": 12747, "n07692614": 12748, "n07692887": 12749, "n07693048": 12750, "n07693223": 12751, "n07693439": 12752, "n07693590": 12753, "n07693725": 12754, "n07693889": 12755, "n07693972": 12756, "n07694169": 12757, "n07694403": 12758, "n07694516": 12759, "n07694659": 12760, "n07694839": 12761, "n07695187": 12762, "n07695284": 12763, "n07695410": 12764, "n07695504": 12765, "n07695652": 12766, "n07695742": 12767, "n07695878": 12768, "n07695965": 12769, "n07696403": 12770, "n07696527": 12771, "n07696625": 12772, "n07696728": 12773, "n07696839": 12774, "n07696977": 12775, "n07697100": 12776, "n07697313": 12777, "n07697408": 12778, "n07697537": 12779, "n07697699": 12780, "n07697825": 12781, "n07698250": 12782, "n07698401": 12783, "n07698543": 12784, "n07698672": 12785, "n07698782": 12786, "n07700003": 12787, "n07703889": 12788, "n07704054": 12789, "n07704205": 12790, "n07704305": 12791, "n07705931": 12792, "n07707451": 12793, "n07708124": 12794, "n07708398": 12795, "n07708512": 12796, "n07708685": 12797, "n07708798": 12798, "n07709046": 12799, "n07709172": 12800, "n07709333": 12801, "n07709701": 12802, "n07709881": 12803, "n07710007": 12804, "n07710283": 12805, "n07710616": 12806, "n07710952": 12807, "n07711080": 12808, "n07711232": 12809, "n07711371": 12810, "n07711569": 12811, "n07711683": 12812, "n07711799": 12813, "n07711907": 12814, "n07712063": 12815, "n07712267": 12816, "n07712382": 12817, "n07712559": 12818, "n07712748": 12819, "n07712856": 12820, "n07712959": 12821, "n07713074": 12822, "n07713267": 12823, "n07713395": 12824, "n07713763": 12825, "n07713895": 12826, "n07714078": 12827, "n07714188": 12828, "n07714287": 12829, "n07714448": 12830, "n07714571": 12831, "n07714802": 12832, "n07714895": 12833, "n07714990": 12834, "n07715103": 12835, "n07715221": 12836, "n07715407": 12837, "n07715561": 12838, "n07715721": 12839, "n07716034": 12840, "n07716203": 12841, "n07716358": 12842, "n07716504": 12843, "n07716649": 12844, "n07716750": 12845, "n07716906": 12846, "n07717070": 12847, "n07717410": 12848, "n07717556": 12849, "n07717714": 12850, "n07717858": 12851, "n07718068": 12852, "n07718195": 12853, "n07718329": 12854, "n07718472": 12855, "n07718671": 12856, "n07718747": 12857, "n07718920": 12858, "n07719058": 12859, "n07719213": 12860, "n07719330": 12861, "n07719437": 12862, "n07719616": 12863, "n07719756": 12864, "n07719839": 12865, "n07719980": 12866, "n07720084": 12867, "n07720185": 12868, "n07720277": 12869, "n07720442": 12870, "n07720615": 12871, "n07720875": 12872, "n07721018": 12873, "n07721118": 12874, "n07721195": 12875, "n07721325": 12876, "n07721456": 12877, "n07721678": 12878, "n07721833": 12879, "n07721942": 12880, "n07722052": 12881, "n07722217": 12882, "n07722390": 12883, "n07722485": 12884, "n07722666": 12885, "n07722763": 12886, "n07722888": 12887, "n07723039": 12888, "n07723177": 12889, "n07723330": 12890, "n07723559": 12891, "n07723753": 12892, "n07723968": 12893, "n07724078": 12894, "n07724173": 12895, "n07724269": 12896, "n07724492": 12897, "n07724654": 12898, "n07724819": 12899, "n07724943": 12900, "n07725158": 12901, "n07725255": 12902, "n07725376": 12903, "n07725531": 12904, "n07725663": 12905, "n07725789": 12906, "n07725888": 12907, "n07726009": 12908, "n07726095": 12909, "n07726230": 12910, "n07726386": 12911, "n07726525": 12912, "n07726672": 12913, "n07726796": 12914, "n07727048": 12915, "n07727140": 12916, "n07727252": 12917, "n07727377": 12918, "n07727458": 12919, "n07727578": 12920, "n07727741": 12921, "n07727868": 12922, "n07728053": 12923, "n07728181": 12924, "n07728284": 12925, "n07728391": 12926, "n07728585": 12927, "n07728708": 12928, "n07728804": 12929, "n07729000": 12930, "n07729142": 12931, "n07729225": 12932, "n07729384": 12933, "n07729485": 12934, "n07729828": 12935, "n07729926": 12936, "n07730033": 12937, "n07730207": 12938, "n07730320": 12939, "n07730406": 12940, "n07730562": 12941, "n07730708": 12942, "n07730855": 12943, "n07731006": 12944, "n07731122": 12945, "n07731284": 12946, "n07731436": 12947, "n07731587": 12948, "n07731767": 12949, "n07731952": 12950, "n07732168": 12951, "n07732302": 12952, "n07732433": 12953, "n07732525": 12954, "n07732636": 12955, "n07732747": 12956, "n07732904": 12957, "n07733005": 12958, "n07733124": 12959, "n07733217": 12960, "n07733394": 12961, "n07733567": 12962, "n07733712": 12963, "n07733847": 12964, "n07734017": 12965, "n07734183": 12966, "n07734292": 12967, "n07734417": 12968, "n07734555": 12969, "n07734744": 12970, "n07734879": 12971, "n07735052": 12972, "n07735179": 12973, "n07735294": 12974, "n07735404": 12975, "n07735510": 12976, "n07735687": 12977, "n07735803": 12978, "n07735981": 12979, "n07736087": 12980, "n07736256": 12981, "n07736371": 12982, "n07736527": 12983, "n07736692": 12984, "n07736813": 12985, "n07736971": 12986, "n07737081": 12987, "n07737594": 12988, "n07737745": 12989, "n07738105": 12990, "n07738224": 12991, "n07739035": 12992, "n07739125": 12993, "n07739344": 12994, "n07739506": 12995, "n07739923": 12996, "n07740033": 12997, "n07740115": 12998, "n07740220": 12999, "n07740342": 13000, "n07740461": 13001, "n07740597": 13002, "n07740744": 13003, "n07740855": 13004, "n07740954": 13005, "n07741138": 13006, "n07741235": 13007, "n07741357": 13008, "n07741461": 13009, "n07741623": 13010, "n07741706": 13011, "n07741804": 13012, "n07741888": 13013, "n07742012": 13014, "n07742224": 13015, "n07742313": 13016, "n07742415": 13017, "n07742513": 13018, "n07742605": 13019, "n07742704": 13020, "n07743224": 13021, "n07743384": 13022, "n07743544": 13023, "n07743723": 13024, "n07743902": 13025, "n07744057": 13026, "n07744246": 13027, "n07744430": 13028, "n07744559": 13029, "n07744682": 13030, "n07744811": 13031, "n07745046": 13032, "n07745197": 13033, "n07745357": 13034, "n07745466": 13035, "n07745661": 13036, "n07745940": 13037, "n07746038": 13038, "n07746186": 13039, "n07746334": 13040, "n07746551": 13041, "n07746749": 13042, "n07746910": 13043, "n07747055": 13044, "n07747607": 13045, "n07747811": 13046, "n07747951": 13047, "n07748157": 13048, "n07748276": 13049, "n07748416": 13050, "n07748574": 13051, "n07748753": 13052, "n07748912": 13053, "n07749095": 13054, "n07749192": 13055, "n07749312": 13056, "n07749446": 13057, "n07749582": 13058, "n07749731": 13059, "n07749870": 13060, "n07749969": 13061, "n07750146": 13062, "n07750299": 13063, "n07750449": 13064, "n07750586": 13065, "n07750736": 13066, "n07750872": 13067, "n07751004": 13068, "n07751148": 13069, "n07751280": 13070, "n07751451": 13071, "n07751737": 13072, "n07751858": 13073, "n07751977": 13074, "n07752109": 13075, "n07752264": 13076, "n07752377": 13077, "n07752514": 13078, "n07752602": 13079, "n07752664": 13080, "n07752782": 13081, "n07752874": 13082, "n07752966": 13083, "n07753113": 13084, "n07753275": 13085, "n07753448": 13086, "n07753592": 13087, "n07753743": 13088, "n07753980": 13089, "n07754155": 13090, "n07754279": 13091, "n07754451": 13092, "n07754684": 13093, "n07754894": 13094, "n07755089": 13095, "n07755262": 13096, "n07755411": 13097, "n07755619": 13098, "n07755707": 13099, "n07755929": 13100, "n07756096": 13101, "n07756325": 13102, "n07756499": 13103, "n07756641": 13104, "n07756838": 13105, "n07756951": 13106, "n07757132": 13107, "n07757312": 13108, "n07757511": 13109, "n07757602": 13110, "n07757753": 13111, "n07757874": 13112, "n07757990": 13113, "n07758125": 13114, "n07758260": 13115, "n07758407": 13116, "n07758582": 13117, "n07758680": 13118, "n07758950": 13119, "n07759194": 13120, "n07759324": 13121, "n07759424": 13122, "n07759576": 13123, "n07759691": 13124, "n07759816": 13125, "n07760070": 13126, "n07760153": 13127, "n07760297": 13128, "n07760395": 13129, "n07760501": 13130, "n07760673": 13131, "n07760755": 13132, "n07760859": 13133, "n07761141": 13134, "n07761309": 13135, "n07761611": 13136, "n07761777": 13137, "n07761954": 13138, "n07762114": 13139, "n07762244": 13140, "n07762373": 13141, "n07762534": 13142, "n07762740": 13143, "n07762913": 13144, "n07763107": 13145, "n07763290": 13146, "n07763483": 13147, "n07763629": 13148, "n07763792": 13149, "n07763987": 13150, "n07764155": 13151, "n07764315": 13152, "n07764486": 13153, "n07764630": 13154, "n07764847": 13155, "n07765073": 13156, "n07765208": 13157, "n07765361": 13158, "n07765517": 13159, "n07765612": 13160, "n07765728": 13161, "n07765862": 13162, "n07765999": 13163, "n07766173": 13164, "n07766409": 13165, "n07766530": 13166, "n07766723": 13167, "n07766891": 13168, "n07767002": 13169, "n07767171": 13170, "n07767344": 13171, "n07767549": 13172, "n07767709": 13173, "n07767847": 13174, "n07768068": 13175, "n07768139": 13176, "n07768230": 13177, "n07768318": 13178, "n07768423": 13179, "n07768590": 13180, "n07768694": 13181, "n07768858": 13182, "n07769102": 13183, "n07769306": 13184, "n07769465": 13185, "n07769584": 13186, "n07769731": 13187, "n07769886": 13188, "n07770034": 13189, "n07770180": 13190, "n07770439": 13191, "n07770571": 13192, "n07770763": 13193, "n07770869": 13194, "n07771082": 13195, "n07771212": 13196, "n07771405": 13197, "n07771539": 13198, "n07771731": 13199, "n07771891": 13200, "n07772026": 13201, "n07772147": 13202, "n07772274": 13203, "n07772413": 13204, "n07772788": 13205, "n07772935": 13206, "n07773428": 13207, "n07774182": 13208, "n07774295": 13209, "n07774479": 13210, "n07774596": 13211, "n07774719": 13212, "n07774842": 13213, "n07775050": 13214, "n07775197": 13215, "n07783827": 13216, "n07785487": 13217, "n07800091": 13218, "n07800487": 13219, "n07800636": 13220, "n07800740": 13221, "n07801007": 13222, "n07801091": 13223, "n07801342": 13224, "n07801508": 13225, "n07801709": 13226, "n07801779": 13227, "n07801892": 13228, "n07802026": 13229, "n07802152": 13230, "n07802246": 13231, "n07802417": 13232, "n07802767": 13233, "n07802863": 13234, "n07802963": 13235, "n07803093": 13236, "n07803213": 13237, "n07803310": 13238, "n07803408": 13239, "n07803545": 13240, "n07803779": 13241, "n07803895": 13242, "n07803992": 13243, "n07804152": 13244, "n07804323": 13245, "n07804543": 13246, "n07804657": 13247, "n07804771": 13248, "n07804900": 13249, "n07805006": 13250, "n07805254": 13251, "n07805389": 13252, "n07805478": 13253, "n07805594": 13254, "n07805731": 13255, "n07805966": 13256, "n07806043": 13257, "n07806120": 13258, "n07806221": 13259, "n07806633": 13260, "n07806774": 13261, "n07806879": 13262, "n07807002": 13263, "n07807171": 13264, "n07807317": 13265, "n07807472": 13266, "n07807594": 13267, "n07807710": 13268, "n07807834": 13269, "n07807922": 13270, "n07808022": 13271, "n07808166": 13272, "n07808268": 13273, "n07808352": 13274, "n07808479": 13275, "n07808587": 13276, "n07808675": 13277, "n07808806": 13278, "n07808904": 13279, "n07809096": 13280, "n07809368": 13281, "n07810531": 13282, "n07810907": 13283, "n07811416": 13284, "n07812046": 13285, "n07812184": 13286, "n07812662": 13287, "n07812790": 13288, "n07812913": 13289, "n07813107": 13290, "n07813324": 13291, "n07813495": 13292, "n07813579": 13293, "n07813717": 13294, "n07813833": 13295, "n07814007": 13296, "n07814203": 13297, "n07814390": 13298, "n07814487": 13299, "n07814634": 13300, "n07814790": 13301, "n07814925": 13302, "n07815163": 13303, "n07815294": 13304, "n07815424": 13305, "n07815588": 13306, "n07815839": 13307, "n07815956": 13308, "n07816052": 13309, "n07816164": 13310, "n07816296": 13311, "n07816398": 13312, "n07816575": 13313, "n07816726": 13314, "n07816839": 13315, "n07817024": 13316, "n07817160": 13317, "n07817315": 13318, "n07817465": 13319, "n07817599": 13320, "n07817758": 13321, "n07817871": 13322, "n07818029": 13323, "n07818133": 13324, "n07818277": 13325, "n07818422": 13326, "n07818572": 13327, "n07818689": 13328, "n07818825": 13329, "n07818995": 13330, "n07819166": 13331, "n07819303": 13332, "n07819480": 13333, "n07819682": 13334, "n07819769": 13335, "n07819896": 13336, "n07820036": 13337, "n07820145": 13338, "n07820297": 13339, "n07820497": 13340, "n07820683": 13341, "n07820814": 13342, "n07820960": 13343, "n07821107": 13344, "n07821260": 13345, "n07821404": 13346, "n07821610": 13347, "n07821758": 13348, "n07821919": 13349, "n07822053": 13350, "n07822197": 13351, "n07822323": 13352, "n07822518": 13353, "n07822687": 13354, "n07822845": 13355, "n07823105": 13356, "n07823280": 13357, "n07823369": 13358, "n07823460": 13359, "n07823591": 13360, "n07823698": 13361, "n07823814": 13362, "n07823951": 13363, "n07824191": 13364, "n07824268": 13365, "n07824383": 13366, "n07824502": 13367, "n07824702": 13368, "n07824863": 13369, "n07824988": 13370, "n07825194": 13371, "n07825399": 13372, "n07825496": 13373, "n07825597": 13374, "n07825717": 13375, "n07825850": 13376, "n07825972": 13377, "n07826091": 13378, "n07826250": 13379, "n07826340": 13380, "n07826453": 13381, "n07826544": 13382, "n07826653": 13383, "n07826930": 13384, "n07827130": 13385, "n07827284": 13386, "n07827410": 13387, "n07827554": 13388, "n07827750": 13389, "n07827896": 13390, "n07828041": 13391, "n07828156": 13392, "n07828275": 13393, "n07828378": 13394, "n07828642": 13395, "n07828987": 13396, "n07829248": 13397, "n07829331": 13398, "n07829412": 13399, "n07830493": 13400, "n07830593": 13401, "n07830690": 13402, "n07830841": 13403, "n07830986": 13404, "n07831146": 13405, "n07831267": 13406, "n07831450": 13407, "n07831663": 13408, "n07831821": 13409, "n07831955": 13410, "n07832099": 13411, "n07832202": 13412, "n07832307": 13413, "n07832416": 13414, "n07832592": 13415, "n07832741": 13416, "n07832902": 13417, "n07833333": 13418, "n07833535": 13419, "n07833672": 13420, "n07833816": 13421, "n07833951": 13422, "n07834065": 13423, "n07834160": 13424, "n07834286": 13425, "n07834507": 13426, "n07834618": 13427, "n07834774": 13428, "n07834872": 13429, "n07835051": 13430, "n07835173": 13431, "n07835331": 13432, "n07835457": 13433, "n07835547": 13434, "n07835701": 13435, "n07835823": 13436, "n07835921": 13437, "n07836077": 13438, "n07836269": 13439, "n07836456": 13440, "n07836600": 13441, "n07836731": 13442, "n07836838": 13443, "n07837002": 13444, "n07837110": 13445, "n07837234": 13446, "n07837362": 13447, "n07837545": 13448, "n07837630": 13449, "n07837755": 13450, "n07837912": 13451, "n07838073": 13452, "n07838233": 13453, "n07838441": 13454, "n07838551": 13455, "n07838659": 13456, "n07838811": 13457, "n07838905": 13458, "n07839055": 13459, "n07839172": 13460, "n07839312": 13461, "n07839478": 13462, "n07839593": 13463, "n07839730": 13464, "n07839864": 13465, "n07840027": 13466, "n07840124": 13467, "n07840219": 13468, "n07840304": 13469, "n07840395": 13470, "n07840520": 13471, "n07840672": 13472, "n07840804": 13473, "n07841037": 13474, "n07841345": 13475, "n07841495": 13476, "n07841639": 13477, "n07841800": 13478, "n07841907": 13479, "n07842044": 13480, "n07842130": 13481, "n07842202": 13482, "n07842308": 13483, "n07842433": 13484, "n07842605": 13485, "n07842753": 13486, "n07842972": 13487, "n07843117": 13488, "n07843220": 13489, "n07843348": 13490, "n07843464": 13491, "n07843636": 13492, "n07843775": 13493, "n07844042": 13494, "n07844604": 13495, "n07844786": 13496, "n07844867": 13497, "n07845087": 13498, "n07845166": 13499, "n07845335": 13500, "n07845421": 13501, "n07845495": 13502, "n07845571": 13503, "n07845702": 13504, "n07845775": 13505, "n07845863": 13506, "n07846014": 13507, "n07846143": 13508, "n07846274": 13509, "n07846359": 13510, "n07846471": 13511, "n07846557": 13512, "n07846688": 13513, "n07846802": 13514, "n07846938": 13515, "n07847047": 13516, "n07847198": 13517, "n07847453": 13518, "n07847585": 13519, "n07847706": 13520, "n07847827": 13521, "n07847917": 13522, "n07848093": 13523, "n07848196": 13524, "n07848338": 13525, "n07848771": 13526, "n07848936": 13527, "n07849026": 13528, "n07849186": 13529, "n07849336": 13530, "n07849506": 13531, "n07849619": 13532, "n07849733": 13533, "n07849912": 13534, "n07850083": 13535, "n07850219": 13536, "n07850329": 13537, "n07851054": 13538, "n07851298": 13539, "n07851443": 13540, "n07851554": 13541, "n07851641": 13542, "n07851767": 13543, "n07851926": 13544, "n07852045": 13545, "n07852229": 13546, "n07852302": 13547, "n07852376": 13548, "n07852452": 13549, "n07852532": 13550, "n07852614": 13551, "n07852712": 13552, "n07852833": 13553, "n07852919": 13554, "n07853125": 13555, "n07853232": 13556, "n07853345": 13557, "n07853445": 13558, "n07853560": 13559, "n07853648": 13560, "n07853762": 13561, "n07853852": 13562, "n07853946": 13563, "n07854066": 13564, "n07854184": 13565, "n07854266": 13566, "n07854348": 13567, "n07854455": 13568, "n07854614": 13569, "n07854707": 13570, "n07854813": 13571, "n07854982": 13572, "n07855105": 13573, "n07855188": 13574, "n07855317": 13575, "n07855413": 13576, "n07855510": 13577, "n07855603": 13578, "n07855721": 13579, "n07855812": 13580, "n07855907": 13581, "n07856045": 13582, "n07856186": 13583, "n07856270": 13584, "n07856756": 13585, "n07856895": 13586, "n07856992": 13587, "n07857076": 13588, "n07857170": 13589, "n07857356": 13590, "n07857598": 13591, "n07857731": 13592, "n07857959": 13593, "n07858114": 13594, "n07858197": 13595, "n07858336": 13596, "n07858484": 13597, "n07858595": 13598, "n07858841": 13599, "n07858978": 13600, "n07859142": 13601, "n07859284": 13602, "n07859583": 13603, "n07859796": 13604, "n07859951": 13605, "n07860103": 13606, "n07860208": 13607, "n07860331": 13608, "n07860447": 13609, "n07860548": 13610, "n07860629": 13611, "n07860805": 13612, "n07860988": 13613, "n07861158": 13614, "n07861247": 13615, "n07861334": 13616, "n07861557": 13617, "n07861681": 13618, "n07861813": 13619, "n07861983": 13620, "n07862095": 13621, "n07862244": 13622, "n07862348": 13623, "n07862461": 13624, "n07862611": 13625, "n07862770": 13626, "n07862946": 13627, "n07863107": 13628, "n07863229": 13629, "n07863374": 13630, "n07863547": 13631, "n07863644": 13632, "n07863802": 13633, "n07863935": 13634, "n07864065": 13635, "n07864198": 13636, "n07864317": 13637, "n07864475": 13638, "n07864638": 13639, "n07864756": 13640, "n07864934": 13641, "n07865105": 13642, "n07865196": 13643, "n07865484": 13644, "n07865575": 13645, "n07865700": 13646, "n07865788": 13647, "n07866015": 13648, "n07866151": 13649, "n07866277": 13650, "n07866409": 13651, "n07866571": 13652, "n07866723": 13653, "n07866868": 13654, "n07867021": 13655, "n07867164": 13656, "n07867324": 13657, "n07867421": 13658, "n07867616": 13659, "n07867751": 13660, "n07867883": 13661, "n07868045": 13662, "n07868200": 13663, "n07868340": 13664, "n07868508": 13665, "n07868684": 13666, "n07868830": 13667, "n07868955": 13668, "n07869111": 13669, "n07869291": 13670, "n07869391": 13671, "n07869522": 13672, "n07869611": 13673, "n07869775": 13674, "n07869937": 13675, "n07870069": 13676, "n07870167": 13677, "n07870313": 13678, "n07870478": 13679, "n07870620": 13680, "n07870734": 13681, "n07870894": 13682, "n07871065": 13683, "n07871234": 13684, "n07871335": 13685, "n07871436": 13686, "n07871588": 13687, "n07871720": 13688, "n07871810": 13689, "n07872593": 13690, "n07872748": 13691, "n07873057": 13692, "n07873198": 13693, "n07873348": 13694, "n07873464": 13695, "n07873679": 13696, "n07873807": 13697, "n07874063": 13698, "n07874159": 13699, "n07874259": 13700, "n07874343": 13701, "n07874441": 13702, "n07874531": 13703, "n07874674": 13704, "n07874780": 13705, "n07874995": 13706, "n07875086": 13707, "n07875152": 13708, "n07875267": 13709, "n07875436": 13710, "n07875560": 13711, "n07875693": 13712, "n07875835": 13713, "n07875926": 13714, "n07876026": 13715, "n07876189": 13716, "n07876281": 13717, "n07876460": 13718, "n07876550": 13719, "n07876651": 13720, "n07876775": 13721, "n07876893": 13722, "n07877187": 13723, "n07877299": 13724, "n07877675": 13725, "n07877849": 13726, "n07877961": 13727, "n07878145": 13728, "n07878283": 13729, "n07878479": 13730, "n07878647": 13731, "n07878785": 13732, "n07878926": 13733, "n07879072": 13734, "n07879174": 13735, "n07879350": 13736, "n07879450": 13737, "n07879560": 13738, "n07879659": 13739, "n07879821": 13740, "n07879953": 13741, "n07880080": 13742, "n07880213": 13743, "n07880325": 13744, "n07880458": 13745, "n07880751": 13746, "n07880880": 13747, "n07880968": 13748, "n07881117": 13749, "n07881205": 13750, "n07881404": 13751, "n07881525": 13752, "n07881625": 13753, "n07881800": 13754, "n07882420": 13755, "n07882497": 13756, "n07882886": 13757, "n07883031": 13758, "n07883156": 13759, "n07883251": 13760, "n07883384": 13761, "n07883510": 13762, "n07883661": 13763, "n07884567": 13764, "n07885705": 13765, "n07886057": 13766, "n07886176": 13767, "n07886317": 13768, "n07886463": 13769, "n07886572": 13770, "n07886849": 13771, "n07887099": 13772, "n07887192": 13773, "n07887304": 13774, "n07887461": 13775, "n07887634": 13776, "n07887967": 13777, "n07888058": 13778, "n07888229": 13779, "n07888378": 13780, "n07888465": 13781, "n07888816": 13782, "n07888909": 13783, "n07889193": 13784, "n07889274": 13785, "n07889510": 13786, "n07889814": 13787, "n07889990": 13788, "n07890068": 13789, "n07890226": 13790, "n07890352": 13791, "n07890540": 13792, "n07890617": 13793, "n07890750": 13794, "n07890890": 13795, "n07890970": 13796, "n07891095": 13797, "n07891189": 13798, "n07891309": 13799, "n07891433": 13800, "n07891726": 13801, "n07892418": 13802, "n07892512": 13803, "n07892813": 13804, "n07893253": 13805, "n07893425": 13806, "n07893528": 13807, "n07893642": 13808, "n07893792": 13809, "n07893891": 13810, "n07894102": 13811, "n07894298": 13812, "n07894451": 13813, "n07894551": 13814, "n07894703": 13815, "n07894799": 13816, "n07894965": 13817, "n07895100": 13818, "n07895237": 13819, "n07895435": 13820, "n07895595": 13821, "n07895710": 13822, "n07895839": 13823, "n07895962": 13824, "n07896060": 13825, "n07896165": 13826, "n07896287": 13827, "n07896422": 13828, "n07896560": 13829, "n07896661": 13830, "n07896765": 13831, "n07896893": 13832, "n07896994": 13833, "n07897116": 13834, "n07897200": 13835, "n07897438": 13836, "n07897600": 13837, "n07897750": 13838, "n07897865": 13839, "n07897975": 13840, "n07898117": 13841, "n07898247": 13842, "n07898333": 13843, "n07898443": 13844, "n07898617": 13845, "n07898745": 13846, "n07898895": 13847, "n07899003": 13848, "n07899108": 13849, "n07899292": 13850, "n07899434": 13851, "n07899533": 13852, "n07899660": 13853, "n07899769": 13854, "n07899899": 13855, "n07899976": 13856, "n07900225": 13857, "n07900406": 13858, "n07900616": 13859, "n07900734": 13860, "n07900825": 13861, "n07900958": 13862, "n07901355": 13863, "n07901457": 13864, "n07901587": 13865, "n07902121": 13866, "n07902336": 13867, "n07902443": 13868, "n07902520": 13869, "n07902698": 13870, "n07902799": 13871, "n07902937": 13872, "n07903101": 13873, "n07903208": 13874, "n07903543": 13875, "n07903643": 13876, "n07903731": 13877, "n07903841": 13878, "n07903962": 13879, "n07904072": 13880, "n07904293": 13881, "n07904395": 13882, "n07904637": 13883, "n07904760": 13884, "n07904865": 13885, "n07904934": 13886, "n07905038": 13887, "n07905296": 13888, "n07905386": 13889, "n07905474": 13890, "n07905618": 13891, "n07905770": 13892, "n07905979": 13893, "n07906111": 13894, "n07906284": 13895, "n07906572": 13896, "n07906718": 13897, "n07906877": 13898, "n07907037": 13899, "n07907161": 13900, "n07907342": 13901, "n07907429": 13902, "n07907548": 13903, "n07907831": 13904, "n07907943": 13905, "n07908411": 13906, "n07908567": 13907, "n07908647": 13908, "n07908812": 13909, "n07908923": 13910, "n07909129": 13911, "n07909231": 13912, "n07909362": 13913, "n07909504": 13914, "n07909593": 13915, "n07909714": 13916, "n07909811": 13917, "n07909954": 13918, "n07910048": 13919, "n07910152": 13920, "n07910245": 13921, "n07910379": 13922, "n07910538": 13923, "n07910656": 13924, "n07910799": 13925, "n07910970": 13926, "n07911061": 13927, "n07911249": 13928, "n07911371": 13929, "n07911677": 13930, "n07912093": 13931, "n07912211": 13932, "n07913180": 13933, "n07913300": 13934, "n07913393": 13935, "n07913537": 13936, "n07913644": 13937, "n07913774": 13938, "n07913882": 13939, "n07914006": 13940, "n07914128": 13941, "n07914271": 13942, "n07914413": 13943, "n07914586": 13944, "n07914686": 13945, "n07914777": 13946, "n07914887": 13947, "n07914995": 13948, "n07915094": 13949, "n07915213": 13950, "n07915366": 13951, "n07915491": 13952, "n07915618": 13953, "n07915800": 13954, "n07915918": 13955, "n07916041": 13956, "n07916183": 13957, "n07916319": 13958, "n07916437": 13959, "n07916582": 13960, "n07917133": 13961, "n07917272": 13962, "n07917392": 13963, "n07917507": 13964, "n07917618": 13965, "n07917791": 13966, "n07917874": 13967, "n07917951": 13968, "n07918028": 13969, "n07918193": 13970, "n07918309": 13971, "n07918706": 13972, "n07918879": 13973, "n07919165": 13974, "n07919310": 13975, "n07919441": 13976, "n07919572": 13977, "n07919665": 13978, "n07919787": 13979, "n07919894": 13980, "n07920052": 13981, "n07920222": 13982, "n07920349": 13983, "n07920540": 13984, "n07920663": 13985, "n07920872": 13986, "n07920989": 13987, "n07921090": 13988, "n07921239": 13989, "n07921360": 13990, "n07921455": 13991, "n07921615": 13992, "n07921834": 13993, "n07921948": 13994, "n07922041": 13995, "n07922147": 13996, "n07922512": 13997, "n07922607": 13998, "n07922764": 13999, "n07922955": 14000, "n07923748": 14001, "n07924033": 14002, "n07924276": 14003, "n07924366": 14004, "n07924443": 14005, "n07924560": 14006, "n07924655": 14007, "n07924747": 14008, "n07924834": 14009, "n07924955": 14010, "n07925116": 14011, "n07925229": 14012, "n07925327": 14013, "n07925423": 14014, "n07925500": 14015, "n07925608": 14016, "n07925708": 14017, "n07925808": 14018, "n07925966": 14019, "n07926250": 14020, "n07926346": 14021, "n07926442": 14022, "n07926540": 14023, "n07926785": 14024, "n07926920": 14025, "n07927070": 14026, "n07927197": 14027, "n07927512": 14028, "n07927716": 14029, "n07927836": 14030, "n07927931": 14031, "n07928163": 14032, "n07928264": 14033, "n07928367": 14034, "n07928488": 14035, "n07928578": 14036, "n07928696": 14037, "n07928790": 14038, "n07928887": 14039, "n07928998": 14040, "n07929172": 14041, "n07929351": 14042, "n07929519": 14043, "n07929940": 14044, "n07930062": 14045, "n07930205": 14046, "n07930315": 14047, "n07930433": 14048, "n07930554": 14049, "n07930864": 14050, "n07931001": 14051, "n07931096": 14052, "n07931280": 14053, "n07931452": 14054, "n07931612": 14055, "n07931733": 14056, "n07931870": 14057, "n07932039": 14058, "n07932323": 14059, "n07932454": 14060, "n07932614": 14061, "n07932762": 14062, "n07932841": 14063, "n07933154": 14064, "n07933274": 14065, "n07933530": 14066, "n07933652": 14067, "n07933799": 14068, "n07933891": 14069, "n07934032": 14070, "n07934152": 14071, "n07934282": 14072, "n07934373": 14073, "n07934530": 14074, "n07934678": 14075, "n07934800": 14076, "n07934908": 14077, "n07935043": 14078, "n07935152": 14079, "n07935288": 14080, "n07935379": 14081, "n07935504": 14082, "n07935737": 14083, "n07935878": 14084, "n07936015": 14085, "n07936093": 14086, "n07936263": 14087, "n07936459": 14088, "n07936548": 14089, "n07936745": 14090, "n07936979": 14091, "n07937069": 14092, "n07937344": 14093, "n07937461": 14094, "n07937621": 14095, "n07938007": 14096, "n07938149": 14097, "n07938313": 14098, "n07938594": 14099, "n07942152": 14100, "n07951464": 14101, "n07954211": 14102, "n07977870": 14103, "n08079613": 14104, "n08182379": 14105, "n08238463": 14106, "n08242223": 14107, "n08249459": 14108, "n08253141": 14109, "n08256735": 14110, "n08376250": 14111, "n08385989": 14112, "n08492354": 14113, "n08492461": 14114, "n08494231": 14115, "n08495908": 14116, "n08496334": 14117, "n08500819": 14118, "n08500989": 14119, "n08501887": 14120, "n08505018": 14121, "n08506347": 14122, "n08511017": 14123, "n08517010": 14124, "n08517676": 14125, "n08518171": 14126, "n08519299": 14127, "n08521623": 14128, "n08523340": 14129, "n08524735": 14130, "n08539072": 14131, "n08539276": 14132, "n08540532": 14133, "n08547468": 14134, "n08547544": 14135, "n08551296": 14136, "n08554440": 14137, "n08555333": 14138, "n08555710": 14139, "n08558770": 14140, "n08558963": 14141, "n08559155": 14142, "n08560295": 14143, "n08569482": 14144, "n08571275": 14145, "n08571642": 14146, "n08571898": 14147, "n08573674": 14148, "n08573842": 14149, "n08578517": 14150, "n08579266": 14151, "n08579352": 14152, "n08580944": 14153, "n08583292": 14154, "n08583455": 14155, "n08583554": 14156, "n08583682": 14157, "n08584914": 14158, "n08586978": 14159, "n08589670": 14160, "n08596076": 14161, "n08597579": 14162, "n08598301": 14163, "n08598568": 14164, "n08599174": 14165, "n08599292": 14166, "n08611339": 14167, "n08611421": 14168, "n08613733": 14169, "n08614632": 14170, "n08616050": 14171, "n08618831": 14172, "n08619112": 14173, "n08623676": 14174, "n08628141": 14175, "n08633683": 14176, "n08640531": 14177, "n08640739": 14178, "n08640962": 14179, "n08643267": 14180, "n08644045": 14181, "n08645104": 14182, "n08645212": 14183, "n08645318": 14184, "n08647264": 14185, "n08648917": 14186, "n08649711": 14187, "n08651104": 14188, "n08652376": 14189, "n08658309": 14190, "n08658918": 14191, "n08659242": 14192, "n08659331": 14193, "n08659446": 14194, "n08659861": 14195, "n08661878": 14196, "n08662427": 14197, "n08663051": 14198, "n08663703": 14199, "n08663860": 14200, "n08673039": 14201, "n08674344": 14202, "n08676253": 14203, "n08677424": 14204, "n08677801": 14205, "n08678783": 14206, "n08679167": 14207, "n08679269": 14208, "n08679562": 14209, "n08685188": 14210, "n08782627": 14211, "n08896327": 14212, "n09032191": 14213, "n09186592": 14214, "n09189157": 14215, "n09191635": 14216, "n09193551": 14217, "n09193705": 14218, "n09194227": 14219, "n09199101": 14220, "n09201998": 14221, "n09203827": 14222, "n09205509": 14223, "n09206896": 14224, "n09206985": 14225, "n09208496": 14226, "n09209025": 14227, "n09210862": 14228, "n09213434": 14229, "n09213565": 14230, "n09214060": 14231, "n09214269": 14232, "n09214916": 14233, "n09215023": 14234, "n09215437": 14235, "n09217230": 14236, "n09218315": 14237, "n09218494": 14238, "n09218641": 14239, "n09219233": 14240, "n09223487": 14241, "n09224725": 14242, "n09226869": 14243, "n09228055": 14244, "n09229709": 14245, "n09230041": 14246, "n09230202": 14247, "n09231117": 14248, "n09233446": 14249, "n09233603": 14250, "n09238926": 14251, "n09239302": 14252, "n09242389": 14253, "n09245515": 14254, "n09246464": 14255, "n09247410": 14256, "n09248153": 14257, "n09248399": 14258, "n09249034": 14259, "n09249155": 14260, "n09251407": 14261, "n09255070": 14262, "n09256479": 14263, "n09257843": 14264, "n09259025": 14265, "n09259219": 14266, "n09260907": 14267, "n09262690": 14268, "n09263912": 14269, "n09264803": 14270, "n09265620": 14271, "n09266604": 14272, "n09267854": 14273, "n09268007": 14274, "n09269341": 14275, "n09269472": 14276, "n09269882": 14277, "n09270160": 14278, "n09270657": 14279, "n09270735": 14280, "n09274152": 14281, "n09274305": 14282, "n09279986": 14283, "n09281252": 14284, "n09282208": 14285, "n09283193": 14286, "n09283405": 14287, "n09283514": 14288, "n09283767": 14289, "n09283866": 14290, "n09287415": 14291, "n09287968": 14292, "n09288635": 14293, "n09289331": 14294, "n09289596": 14295, "n09290350": 14296, "n09290444": 14297, "n09294877": 14298, "n09295210": 14299, "n09295946": 14300, "n09300306": 14301, "n09300905": 14302, "n09302616": 14303, "n09303008": 14304, "n09303528": 14305, "n09304750": 14306, "n09305031": 14307, "n09305898": 14308, "n09308572": 14309, "n09308743": 14310, "n09309046": 14311, "n09309168": 14312, "n09309292": 14313, "n09310616": 14314, "n09315159": 14315, "n09319604": 14316, "n09325824": 14317, "n09326662": 14318, "n09327077": 14319, "n09327538": 14320, "n09330378": 14321, "n09331251": 14322, "n09332890": 14323, "n09335693": 14324, "n09335809": 14325, "n09336555": 14326, "n09337048": 14327, "n09337253": 14328, "n09338013": 14329, "n09339810": 14330, "n09344198": 14331, "n09344324": 14332, "n09344724": 14333, "n09348460": 14334, "n09349648": 14335, "n09351905": 14336, "n09352849": 14337, "n09353815": 14338, "n09354511": 14339, "n09357346": 14340, "n09357447": 14341, "n09359803": 14342, "n09361517": 14343, "n09362316": 14344, "n09362945": 14345, "n09366017": 14346, "n09366317": 14347, "n09375606": 14348, "n09376198": 14349, "n09376526": 14350, "n09376786": 14351, "n09381242": 14352, "n09382099": 14353, "n09384106": 14354, "n09389867": 14355, "n09391386": 14356, "n09391644": 14357, "n09391774": 14358, "n09392402": 14359, "n09393524": 14360, "n09393605": 14361, "n09396465": 14362, "n09396608": 14363, "n09398076": 14364, "n09398677": 14365, "n09399592": 14366, "n09400584": 14367, "n09400987": 14368, "n09402944": 14369, "n09403086": 14370, "n09403211": 14371, "n09403427": 14372, "n09403734": 14373, "n09405078": 14374, "n09405787": 14375, "n09406793": 14376, "n09409512": 14377, "n09409752": 14378, "n09410224": 14379, "n09411189": 14380, "n09411295": 14381, "n09415584": 14382, "n09415671": 14383, "n09416076": 14384, "n09416890": 14385, "n09421031": 14386, "n09421799": 14387, "n09421951": 14388, "n09422190": 14389, "n09422631": 14390, "n09425019": 14391, "n09425344": 14392, "n09428293": 14393, "n09428628": 14394, "n09429630": 14395, "n09432283": 14396, "n09432990": 14397, "n09433312": 14398, "n09433442": 14399, "n09433839": 14400, "n09435739": 14401, "n09436444": 14402, "n09436708": 14403, "n09437454": 14404, "n09438844": 14405, "n09438940": 14406, "n09439032": 14407, "n09439213": 14408, "n09442595": 14409, "n09443281": 14410, "n09443641": 14411, "n09444783": 14412, "n09445008": 14413, "n09445289": 14414, "n09447666": 14415, "n09448690": 14416, "n09450163": 14417, "n09451237": 14418, "n09452291": 14419, "n09452395": 14420, "n09452760": 14421, "n09453008": 14422, "n09454153": 14423, "n09454412": 14424, "n09454744": 14425, "n09456207": 14426, "n09457979": 14427, "n09458269": 14428, "n09459979": 14429, "n09460046": 14430, "n09461069": 14431, "n09462600": 14432, "n09463226": 14433, "n09464486": 14434, "n09466678": 14435, "n09467696": 14436, "n09468604": 14437, "n09470027": 14438, "n09470222": 14439, "n09472413": 14440, "n09472597": 14441, "n09474010": 14442, "n09474412": 14443, "n09474765": 14444, "n09475044": 14445, "n09475179": 14446, "n09475925": 14447, "n09476123": 14448, "n09478210": 14449, "n09480959": 14450, "n09481120": 14451, "n09493983": 14452, "n09495962": 14453, "n09505153": 14454, "n09537660": 14455, "n09556121": 14456, "n09605110": 14457, "n09606009": 14458, "n09606527": 14459, "n09607630": 14460, "n09607782": 14461, "n09607903": 14462, "n09608709": 14463, "n09610255": 14464, "n09610405": 14465, "n09611722": 14466, "n09612700": 14467, "n09613118": 14468, "n09613191": 14469, "n09613690": 14470, "n09615336": 14471, "n09616573": 14472, "n09616922": 14473, "n09617161": 14474, "n09617435": 14475, "n09617577": 14476, "n09617696": 14477, "n09618760": 14478, "n09618880": 14479, "n09618957": 14480, "n09619168": 14481, "n09619452": 14482, "n09620078": 14483, "n09620794": 14484, "n09621232": 14485, "n09622049": 14486, "n09622302": 14487, "n09624168": 14488, "n09624559": 14489, "n09624899": 14490, "n09625401": 14491, "n09626238": 14492, "n09627807": 14493, "n09627906": 14494, "n09629065": 14495, "n09629246": 14496, "n09629752": 14497, "n09631129": 14498, "n09632274": 14499, "n09632518": 14500, "n09633969": 14501, "n09635534": 14502, "n09635635": 14503, "n09635973": 14504, "n09636339": 14505, "n09637339": 14506, "n09638454": 14507, "n09638875": 14508, "n09639382": 14509, "n09639919": 14510, "n09640327": 14511, "n09640715": 14512, "n09641002": 14513, "n09641578": 14514, "n09643799": 14515, "n09644152": 14516, "n09644657": 14517, "n09648743": 14518, "n09648911": 14519, "n09649067": 14520, "n09650729": 14521, "n09650839": 14522, "n09650989": 14523, "n09651123": 14524, "n09651968": 14525, "n09652149": 14526, "n09653144": 14527, "n09653438": 14528, "n09654079": 14529, "n09654518": 14530, "n09654898": 14531, "n09655213": 14532, "n09655466": 14533, "n09656077": 14534, "n09657206": 14535, "n09657748": 14536, "n09658254": 14537, "n09658398": 14538, "n09658815": 14539, "n09658921": 14540, "n09659039": 14541, "n09659188": 14542, "n09660010": 14543, "n09660240": 14544, "n09661873": 14545, "n09662038": 14546, "n09662661": 14547, "n09662951": 14548, "n09663248": 14549, "n09663786": 14550, "n09663999": 14551, "n09664556": 14552, "n09664908": 14553, "n09665367": 14554, "n09665545": 14555, "n09666349": 14556, "n09666476": 14557, "n09666883": 14558, "n09667358": 14559, "n09668199": 14560, "n09668437": 14561, "n09668562": 14562, "n09668988": 14563, "n09669631": 14564, "n09670280": 14565, "n09670521": 14566, "n09670909": 14567, "n09671089": 14568, "n09672590": 14569, "n09672725": 14570, "n09672840": 14571, "n09673091": 14572, "n09674412": 14573, "n09674786": 14574, "n09675045": 14575, "n09675673": 14576, "n09675799": 14577, "n09675922": 14578, "n09676021": 14579, "n09676247": 14580, "n09676884": 14581, "n09677427": 14582, "n09678747": 14583, "n09679028": 14584, "n09679170": 14585, "n09679925": 14586, "n09680908": 14587, "n09681107": 14588, "n09681234": 14589, "n09681973": 14590, "n09683180": 14591, "n09683757": 14592, "n09683924": 14593, "n09684082": 14594, "n09684901": 14595, "n09685233": 14596, "n09685806": 14597, "n09686262": 14598, "n09686401": 14599, "n09688233": 14600, "n09688804": 14601, "n09689435": 14602, "n09689958": 14603, "n09690083": 14604, "n09690208": 14605, "n09690496": 14606, "n09690621": 14607, "n09690864": 14608, "n09691604": 14609, "n09691729": 14610, "n09691858": 14611, "n09692125": 14612, "n09692915": 14613, "n09693244": 14614, "n09693982": 14615, "n09694664": 14616, "n09694771": 14617, "n09695019": 14618, "n09695132": 14619, "n09695514": 14620, "n09695620": 14621, "n09695979": 14622, "n09696456": 14623, "n09696585": 14624, "n09696763": 14625, "n09697401": 14626, "n09697986": 14627, "n09698644": 14628, "n09699020": 14629, "n09699642": 14630, "n09700125": 14631, "n09700964": 14632, "n09701148": 14633, "n09701833": 14634, "n09702134": 14635, "n09702673": 14636, "n09703101": 14637, "n09703344": 14638, "n09703485": 14639, "n09703708": 14640, "n09703809": 14641, "n09703932": 14642, "n09704057": 14643, "n09704157": 14644, "n09704283": 14645, "n09705003": 14646, "n09705124": 14647, "n09705671": 14648, "n09705784": 14649, "n09706029": 14650, "n09706255": 14651, "n09707061": 14652, "n09707289": 14653, "n09707735": 14654, "n09708750": 14655, "n09708889": 14656, "n09709531": 14657, "n09709673": 14658, "n09710041": 14659, "n09710164": 14660, "n09710886": 14661, "n09711132": 14662, "n09711435": 14663, "n09712324": 14664, "n09712448": 14665, "n09712696": 14666, "n09712967": 14667, "n09713108": 14668, "n09714120": 14669, "n09714694": 14670, "n09715165": 14671, "n09715303": 14672, "n09715427": 14673, "n09716047": 14674, "n09716933": 14675, "n09717233": 14676, "n09718217": 14677, "n09718811": 14678, "n09718936": 14679, "n09719309": 14680, "n09719794": 14681, "n09720033": 14682, "n09720256": 14683, "n09720595": 14684, "n09720702": 14685, "n09720842": 14686, "n09721244": 14687, "n09721444": 14688, "n09722064": 14689, "n09722658": 14690, "n09722817": 14691, "n09723067": 14692, "n09723819": 14693, "n09723944": 14694, "n09724234": 14695, "n09724533": 14696, "n09724656": 14697, "n09724785": 14698, "n09725000": 14699, "n09725229": 14700, "n09725546": 14701, "n09725653": 14702, "n09725772": 14703, "n09725935": 14704, "n09726621": 14705, "n09726811": 14706, "n09727440": 14707, "n09727826": 14708, "n09728137": 14709, "n09728285": 14710, "n09729062": 14711, "n09729156": 14712, "n09730077": 14713, "n09730204": 14714, "n09730824": 14715, "n09731343": 14716, "n09731436": 14717, "n09731571": 14718, "n09732170": 14719, "n09733459": 14720, "n09733793": 14721, "n09734185": 14722, "n09734450": 14723, "n09734535": 14724, "n09734639": 14725, "n09735258": 14726, "n09735654": 14727, "n09736485": 14728, "n09736798": 14729, "n09736945": 14730, "n09737050": 14731, "n09737161": 14732, "n09737453": 14733, "n09738121": 14734, "n09738400": 14735, "n09740724": 14736, "n09741074": 14737, "n09741331": 14738, "n09741722": 14739, "n09741816": 14740, "n09741904": 14741, "n09741999": 14742, "n09742101": 14743, "n09742315": 14744, "n09742927": 14745, "n09743487": 14746, "n09743601": 14747, "n09743792": 14748, "n09744161": 14749, "n09744346": 14750, "n09744462": 14751, "n09744679": 14752, "n09744834": 14753, "n09745229": 14754, "n09745324": 14755, "n09745834": 14756, "n09745933": 14757, "n09746936": 14758, "n09747191": 14759, "n09747495": 14760, "n09748101": 14761, "n09748408": 14762, "n09748648": 14763, "n09748889": 14764, "n09749386": 14765, "n09750282": 14766, "n09750641": 14767, "n09750770": 14768, "n09750891": 14769, "n09751076": 14770, "n09751496": 14771, "n09751622": 14772, "n09751895": 14773, "n09752023": 14774, "n09752519": 14775, "n09753348": 14776, "n09753792": 14777, "n09754152": 14778, "n09754217": 14779, "n09754633": 14780, "n09754907": 14781, "n09755086": 14782, "n09755241": 14783, "n09755555": 14784, "n09755788": 14785, "n09755893": 14786, "n09756049": 14787, "n09756195": 14788, "n09756961": 14789, "n09757449": 14790, "n09758173": 14791, "n09758885": 14792, "n09759501": 14793, "n09760290": 14794, "n09760609": 14795, "n09760913": 14796, "n09761068": 14797, "n09761753": 14798, "n09762011": 14799, "n09762385": 14800, "n09763272": 14801, "n09763784": 14802, "n09764201": 14803, "n09764598": 14804, "n09764732": 14805, "n09764900": 14806, "n09765118": 14807, "n09765278": 14808, "n09767197": 14809, "n09769076": 14810, "n09769525": 14811, "n09769929": 14812, "n09770179": 14813, "n09770359": 14814, "n09771435": 14815, "n09772330": 14816, "n09772746": 14817, "n09772930": 14818, "n09773962": 14819, "n09774167": 14820, "n09774783": 14821, "n09775907": 14822, "n09776346": 14823, "n09776642": 14824, "n09776807": 14825, "n09777870": 14826, "n09778266": 14827, "n09778537": 14828, "n09778783": 14829, "n09778927": 14830, "n09779124": 14831, "n09779280": 14832, "n09779461": 14833, "n09779790": 14834, "n09780395": 14835, "n09780828": 14836, "n09780984": 14837, "n09781398": 14838, "n09781504": 14839, "n09781650": 14840, "n09782167": 14841, "n09782397": 14842, "n09782855": 14843, "n09783537": 14844, "n09783776": 14845, "n09783884": 14846, "n09784043": 14847, "n09784160": 14848, "n09784564": 14849, "n09785236": 14850, "n09785659": 14851, "n09785891": 14852, "n09786115": 14853, "n09787534": 14854, "n09787765": 14855, "n09788073": 14856, "n09788237": 14857, "n09789150": 14858, "n09789566": 14859, "n09789898": 14860, "n09790047": 14861, "n09790482": 14862, "n09791014": 14863, "n09791419": 14864, "n09791816": 14865, "n09792125": 14866, "n09792555": 14867, "n09792969": 14868, "n09793141": 14869, "n09793352": 14870, "n09793946": 14871, "n09794550": 14872, "n09794668": 14873, "n09795010": 14874, "n09795124": 14875, "n09795334": 14876, "n09796809": 14877, "n09796974": 14878, "n09797742": 14879, "n09797873": 14880, "n09797998": 14881, "n09798096": 14882, "n09800469": 14883, "n09800964": 14884, "n09801102": 14885, "n09801275": 14886, "n09801533": 14887, "n09802445": 14888, "n09802641": 14889, "n09802951": 14890, "n09804230": 14891, "n09805151": 14892, "n09805324": 14893, "n09805475": 14894, "n09806944": 14895, "n09807075": 14896, "n09808080": 14897, "n09808591": 14898, "n09809279": 14899, "n09809538": 14900, "n09809749": 14901, "n09809925": 14902, "n09810166": 14903, "n09811568": 14904, "n09811712": 14905, "n09811852": 14906, "n09813219": 14907, "n09814252": 14908, "n09814381": 14909, "n09814488": 14910, "n09814567": 14911, "n09814660": 14912, "n09815455": 14913, "n09815790": 14914, "n09816654": 14915, "n09816771": 14916, "n09817174": 14917, "n09817386": 14918, "n09818022": 14919, "n09819477": 14920, "n09820044": 14921, "n09820263": 14922, "n09821831": 14923, "n09822830": 14924, "n09823153": 14925, "n09823287": 14926, "n09823502": 14927, "n09823832": 14928, "n09824135": 14929, "n09824609": 14930, "n09825096": 14931, "n09825750": 14932, "n09826204": 14933, "n09826605": 14934, "n09826821": 14935, "n09827246": 14936, "n09827363": 14937, "n09828216": 14938, "n09828403": 14939, "n09828988": 14940, "n09830194": 14941, "n09830400": 14942, "n09830629": 14943, "n09830759": 14944, "n09830926": 14945, "n09831962": 14946, "n09832456": 14947, "n09832633": 14948, "n09832978": 14949, "n09833111": 14950, "n09833275": 14951, "n09833441": 14952, "n09833536": 14953, "n09833751": 14954, "n09833997": 14955, "n09834258": 14956, "n09834378": 14957, "n09834699": 14958, "n09834885": 14959, "n09835017": 14960, "n09835153": 14961, "n09835230": 14962, "n09835348": 14963, "n09835506": 14964, "n09836160": 14965, "n09836343": 14966, "n09836519": 14967, "n09836786": 14968, "n09837459": 14969, "n09837720": 14970, "n09838295": 14971, "n09838370": 14972, "n09838621": 14973, "n09839702": 14974, "n09840217": 14975, "n09840435": 14976, "n09840520": 14977, "n09841188": 14978, "n09841515": 14979, "n09841696": 14980, "n09842047": 14981, "n09842288": 14982, "n09842395": 14983, "n09842528": 14984, "n09842823": 14985, "n09843443": 14986, "n09843602": 14987, "n09843716": 14988, "n09843824": 14989, "n09844457": 14990, "n09844898": 14991, "n09845401": 14992, "n09845849": 14993, "n09846142": 14994, "n09846469": 14995, "n09846586": 14996, "n09846755": 14997, "n09846894": 14998, "n09847267": 14999, "n09847344": 15000, "n09847543": 15001, "n09848110": 15002, "n09848489": 15003, "n09849167": 15004, "n09849990": 15005, "n09850760": 15006, "n09850974": 15007, "n09851165": 15008, "n09851575": 15009, "n09853541": 15010, "n09853645": 15011, "n09853881": 15012, "n09854218": 15013, "n09854421": 15014, "n09854915": 15015, "n09855433": 15016, "n09856401": 15017, "n09856671": 15018, "n09856827": 15019, "n09857007": 15020, "n09858165": 15021, "n09858299": 15022, "n09858733": 15023, "n09859152": 15024, "n09859285": 15025, "n09859975": 15026, "n09861287": 15027, "n09861599": 15028, "n09861863": 15029, "n09861946": 15030, "n09862183": 15031, "n09862621": 15032, "n09863031": 15033, "n09863339": 15034, "n09863749": 15035, "n09863936": 15036, "n09864632": 15037, "n09864968": 15038, "n09865068": 15039, "n09865162": 15040, "n09865398": 15041, "n09865672": 15042, "n09865744": 15043, "n09866115": 15044, "n09866354": 15045, "n09866559": 15046, "n09866661": 15047, "n09866817": 15048, "n09866922": 15049, "n09867069": 15050, "n09867154": 15051, "n09867311": 15052, "n09868270": 15053, "n09868782": 15054, "n09868899": 15055, "n09869317": 15056, "n09869447": 15057, "n09869578": 15058, "n09870096": 15059, "n09871095": 15060, "n09871229": 15061, "n09871681": 15062, "n09871867": 15063, "n09871952": 15064, "n09872066": 15065, "n09872557": 15066, "n09873348": 15067, "n09873473": 15068, "n09873769": 15069, "n09873899": 15070, "n09874428": 15071, "n09874725": 15072, "n09874862": 15073, "n09875025": 15074, "n09875979": 15075, "n09876701": 15076, "n09877288": 15077, "n09877587": 15078, "n09877750": 15079, "n09877951": 15080, "n09878921": 15081, "n09879552": 15082, "n09880189": 15083, "n09880741": 15084, "n09881265": 15085, "n09881358": 15086, "n09881895": 15087, "n09883047": 15088, "n09883452": 15089, "n09883807": 15090, "n09885059": 15091, "n09885866": 15092, "n09886403": 15093, "n09886540": 15094, "n09888635": 15095, "n09889065": 15096, "n09889170": 15097, "n09889691": 15098, "n09889941": 15099, "n09890192": 15100, "n09890749": 15101, "n09891730": 15102, "n09892262": 15103, "n09892513": 15104, "n09892693": 15105, "n09893191": 15106, "n09893344": 15107, "n09893502": 15108, "n09893600": 15109, "n09894143": 15110, "n09894445": 15111, "n09894654": 15112, "n09894909": 15113, "n09895222": 15114, "n09895480": 15115, "n09895561": 15116, "n09895701": 15117, "n09895902": 15118, "n09896170": 15119, "n09896311": 15120, "n09896401": 15121, "n09896685": 15122, "n09896826": 15123, "n09898020": 15124, "n09899289": 15125, "n09899671": 15126, "n09899782": 15127, "n09899929": 15128, "n09901337": 15129, "n09901502": 15130, "n09901642": 15131, "n09901786": 15132, "n09901921": 15133, "n09902128": 15134, "n09902353": 15135, "n09902731": 15136, "n09902851": 15137, "n09902954": 15138, "n09903153": 15139, "n09903501": 15140, "n09903639": 15141, "n09903936": 15142, "n09904208": 15143, "n09904837": 15144, "n09905050": 15145, "n09905185": 15146, "n09905530": 15147, "n09906293": 15148, "n09906449": 15149, "n09906704": 15150, "n09907804": 15151, "n09908769": 15152, "n09909660": 15153, "n09909929": 15154, "n09910222": 15155, "n09910374": 15156, "n09910556": 15157, "n09910840": 15158, "n09911226": 15159, "n09912431": 15160, "n09912681": 15161, "n09912907": 15162, "n09912995": 15163, "n09913329": 15164, "n09913455": 15165, "n09913593": 15166, "n09915434": 15167, "n09915651": 15168, "n09916348": 15169, "n09917214": 15170, "n09917345": 15171, "n09917481": 15172, "n09917593": 15173, "n09918248": 15174, "n09918554": 15175, "n09918867": 15176, "n09919061": 15177, "n09919200": 15178, "n09919451": 15179, "n09919899": 15180, "n09920106": 15181, "n09920283": 15182, "n09920901": 15183, "n09921034": 15184, "n09923003": 15185, "n09923186": 15186, "n09923418": 15187, "n09923561": 15188, "n09923673": 15189, "n09923996": 15190, "n09924106": 15191, "n09924195": 15192, "n09924313": 15193, "n09924437": 15194, "n09924996": 15195, "n09927089": 15196, "n09927451": 15197, "n09928136": 15198, "n09928451": 15199, "n09928845": 15200, "n09929202": 15201, "n09929298": 15202, "n09929577": 15203, "n09930257": 15204, "n09930628": 15205, "n09930876": 15206, "n09931165": 15207, "n09931418": 15208, "n09931640": 15209, "n09932098": 15210, "n09932336": 15211, "n09932508": 15212, "n09932788": 15213, "n09933020": 15214, "n09933098": 15215, "n09933842": 15216, "n09933972": 15217, "n09934337": 15218, "n09934488": 15219, "n09934774": 15220, "n09935107": 15221, "n09935434": 15222, "n09936825": 15223, "n09936892": 15224, "n09937056": 15225, "n09937688": 15226, "n09937802": 15227, "n09937903": 15228, "n09938080": 15229, "n09938449": 15230, "n09938991": 15231, "n09940725": 15232, "n09940818": 15233, "n09941089": 15234, "n09941571": 15235, "n09941787": 15236, "n09941964": 15237, "n09942697": 15238, "n09942970": 15239, "n09943239": 15240, "n09943811": 15241, "n09944022": 15242, "n09944160": 15243, "n09944430": 15244, "n09945021": 15245, "n09945223": 15246, "n09945319": 15247, "n09945603": 15248, "n09945745": 15249, "n09946814": 15250, "n09947127": 15251, "n09950457": 15252, "n09950728": 15253, "n09951070": 15254, "n09951274": 15255, "n09951524": 15256, "n09951616": 15257, "n09952163": 15258, "n09953052": 15259, "n09953350": 15260, "n09953615": 15261, "n09954355": 15262, "n09954639": 15263, "n09955406": 15264, "n09955944": 15265, "n09956578": 15266, "n09957523": 15267, "n09958133": 15268, "n09958292": 15269, "n09958447": 15270, "n09958569": 15271, "n09959142": 15272, "n09959658": 15273, "n09960688": 15274, "n09961198": 15275, "n09961331": 15276, "n09961469": 15277, "n09961605": 15278, "n09961739": 15279, "n09962966": 15280, "n09964202": 15281, "n09964411": 15282, "n09965515": 15283, "n09965787": 15284, "n09966470": 15285, "n09966554": 15286, "n09967063": 15287, "n09967406": 15288, "n09967555": 15289, "n09967816": 15290, "n09967967": 15291, "n09968259": 15292, "n09968652": 15293, "n09968741": 15294, "n09968845": 15295, "n09970088": 15296, "n09970192": 15297, "n09970402": 15298, "n09970822": 15299, "n09971273": 15300, "n09971385": 15301, "n09971839": 15302, "n09972010": 15303, "n09972458": 15304, "n09972587": 15305, "n09974648": 15306, "n09975425": 15307, "n09976024": 15308, "n09976283": 15309, "n09976429": 15310, "n09976728": 15311, "n09976917": 15312, "n09978442": 15313, "n09979321": 15314, "n09979913": 15315, "n09980458": 15316, "n09980805": 15317, "n09980985": 15318, "n09981092": 15319, "n09981278": 15320, "n09981540": 15321, "n09981939": 15322, "n09982152": 15323, "n09982525": 15324, "n09983314": 15325, "n09983572": 15326, "n09983889": 15327, "n09984960": 15328, "n09985470": 15329, "n09985809": 15330, "n09985978": 15331, "n09986450": 15332, "n09986700": 15333, "n09986904": 15334, "n09987045": 15335, "n09987161": 15336, "n09987239": 15337, "n09988063": 15338, "n09988311": 15339, "n09988493": 15340, "n09988703": 15341, "n09989502": 15342, "n09990415": 15343, "n09990690": 15344, "n09990777": 15345, "n09991740": 15346, "n09991867": 15347, "n09992538": 15348, "n09992837": 15349, "n09993252": 15350, "n09993651": 15351, "n09994400": 15352, "n09994673": 15353, "n09994808": 15354, "n09994878": 15355, "n09995829": 15356, "n09996039": 15357, "n09996304": 15358, "n09996481": 15359, "n09997622": 15360, "n09998788": 15361, "n09999135": 15362, "n10000294": 15363, "n10000459": 15364, "n10000787": 15365, "n10001217": 15366, "n10001481": 15367, "n10001764": 15368, "n10002257": 15369, "n10002760": 15370, "n10003476": 15371, "n10004718": 15372, "n10005006": 15373, "n10005934": 15374, "n10006177": 15375, "n10006748": 15376, "n10007684": 15377, "n10007809": 15378, "n10007995": 15379, "n10008123": 15380, "n10008254": 15381, "n10009162": 15382, "n10009276": 15383, "n10009484": 15384, "n10009671": 15385, "n10010062": 15386, "n10010243": 15387, "n10010632": 15388, "n10010767": 15389, "n10010864": 15390, "n10011360": 15391, "n10011486": 15392, "n10012484": 15393, "n10013811": 15394, "n10015215": 15395, "n10015485": 15396, "n10015792": 15397, "n10015897": 15398, "n10017272": 15399, "n10017422": 15400, "n10018747": 15401, "n10018861": 15402, "n10019072": 15403, "n10019187": 15404, "n10019406": 15405, "n10020366": 15406, "n10020533": 15407, "n10020670": 15408, "n10020807": 15409, "n10020890": 15410, "n10022908": 15411, "n10023264": 15412, "n10023506": 15413, "n10023656": 15414, "n10024025": 15415, "n10024362": 15416, "n10024937": 15417, "n10025060": 15418, "n10025295": 15419, "n10025391": 15420, "n10025635": 15421, "n10026976": 15422, "n10027246": 15423, "n10027590": 15424, "n10028402": 15425, "n10028541": 15426, "n10029068": 15427, "n10030277": 15428, "n10032987": 15429, "n10033412": 15430, "n10033572": 15431, "n10033663": 15432, "n10033888": 15433, "n10034201": 15434, "n10034614": 15435, "n10035952": 15436, "n10036266": 15437, "n10036444": 15438, "n10036692": 15439, "n10036929": 15440, "n10037080": 15441, "n10037385": 15442, "n10037588": 15443, "n10037922": 15444, "n10038119": 15445, "n10038409": 15446, "n10038620": 15447, "n10039271": 15448, "n10039946": 15449, "n10040240": 15450, "n10040698": 15451, "n10040945": 15452, "n10041373": 15453, "n10041887": 15454, "n10042690": 15455, "n10042845": 15456, "n10043024": 15457, "n10043491": 15458, "n10043643": 15459, "n10044682": 15460, "n10044879": 15461, "n10047199": 15462, "n10047459": 15463, "n10048117": 15464, "n10048367": 15465, "n10048612": 15466, "n10048836": 15467, "n10049363": 15468, "n10050043": 15469, "n10050880": 15470, "n10051026": 15471, "n10051761": 15472, "n10051861": 15473, "n10051975": 15474, "n10052694": 15475, "n10053439": 15476, "n10053808": 15477, "n10054657": 15478, "n10055297": 15479, "n10055410": 15480, "n10055566": 15481, "n10055730": 15482, "n10055847": 15483, "n10056103": 15484, "n10056611": 15485, "n10056719": 15486, "n10057271": 15487, "n10058411": 15488, "n10058962": 15489, "n10059067": 15490, "n10060075": 15491, "n10060175": 15492, "n10060352": 15493, "n10061043": 15494, "n10061195": 15495, "n10061431": 15496, "n10061882": 15497, "n10062042": 15498, "n10062176": 15499, "n10062275": 15500, "n10062492": 15501, "n10062594": 15502, "n10062716": 15503, "n10062905": 15504, "n10062996": 15505, "n10063635": 15506, "n10063919": 15507, "n10064831": 15508, "n10064977": 15509, "n10065758": 15510, "n10066206": 15511, "n10066314": 15512, "n10067011": 15513, "n10067305": 15514, "n10067600": 15515, "n10067968": 15516, "n10068234": 15517, "n10068425": 15518, "n10069296": 15519, "n10069981": 15520, "n10070108": 15521, "n10070377": 15522, "n10070449": 15523, "n10070563": 15524, "n10070711": 15525, "n10071332": 15526, "n10071557": 15527, "n10072054": 15528, "n10074249": 15529, "n10074578": 15530, "n10074735": 15531, "n10074841": 15532, "n10075299": 15533, "n10075693": 15534, "n10076224": 15535, "n10076483": 15536, "n10076604": 15537, "n10076957": 15538, "n10077106": 15539, "n10077593": 15540, "n10077879": 15541, "n10078131": 15542, "n10078719": 15543, "n10078806": 15544, "n10079399": 15545, "n10079893": 15546, "n10080117": 15547, "n10080508": 15548, "n10080869": 15549, "n10081204": 15550, "n10081842": 15551, "n10082043": 15552, "n10082299": 15553, "n10082423": 15554, "n10082562": 15555, "n10082687": 15556, "n10082997": 15557, "n10083677": 15558, "n10083823": 15559, "n10084043": 15560, "n10084295": 15561, "n10085101": 15562, "n10085869": 15563, "n10086383": 15564, "n10086744": 15565, "n10087434": 15566, "n10087736": 15567, "n10088200": 15568, "n10090745": 15569, "n10091349": 15570, "n10091450": 15571, "n10091564": 15572, "n10091651": 15573, "n10091861": 15574, "n10091997": 15575, "n10092488": 15576, "n10092643": 15577, "n10092794": 15578, "n10092978": 15579, "n10093167": 15580, "n10093475": 15581, "n10093818": 15582, "n10094320": 15583, "n10094584": 15584, "n10094782": 15585, "n10095265": 15586, "n10095420": 15587, "n10095769": 15588, "n10095869": 15589, "n10096126": 15590, "n10096508": 15591, "n10097262": 15592, "n10097477": 15593, "n10097590": 15594, "n10097842": 15595, "n10097995": 15596, "n10098245": 15597, "n10098388": 15598, "n10098517": 15599, "n10098624": 15600, "n10098710": 15601, "n10098862": 15602, "n10099002": 15603, "n10099375": 15604, "n10101308": 15605, "n10101634": 15606, "n10101981": 15607, "n10102800": 15608, "n10103155": 15609, "n10103228": 15610, "n10103921": 15611, "n10104064": 15612, "n10104487": 15613, "n10104756": 15614, "n10104888": 15615, "n10105085": 15616, "n10105733": 15617, "n10105906": 15618, "n10106387": 15619, "n10106509": 15620, "n10106995": 15621, "n10107173": 15622, "n10107303": 15623, "n10108018": 15624, "n10108089": 15625, "n10108464": 15626, "n10108832": 15627, "n10109443": 15628, "n10109662": 15629, "n10109826": 15630, "n10110093": 15631, "n10110731": 15632, "n10110893": 15633, "n10111358": 15634, "n10111779": 15635, "n10111903": 15636, "n10112129": 15637, "n10113249": 15638, "n10113583": 15639, "n10113869": 15640, "n10114476": 15641, "n10114550": 15642, "n10114662": 15643, "n10115430": 15644, "n10115946": 15645, "n10116370": 15646, "n10116478": 15647, "n10116702": 15648, "n10117017": 15649, "n10117267": 15650, "n10117415": 15651, "n10117739": 15652, "n10117851": 15653, "n10118301": 15654, "n10118743": 15655, "n10118844": 15656, "n10119609": 15657, "n10120330": 15658, "n10120671": 15659, "n10121026": 15660, "n10121246": 15661, "n10121714": 15662, "n10121800": 15663, "n10122300": 15664, "n10122531": 15665, "n10123122": 15666, "n10123844": 15667, "n10126177": 15668, "n10126424": 15669, "n10126708": 15670, "n10127186": 15671, "n10127689": 15672, "n10128519": 15673, "n10128748": 15674, "n10129338": 15675, "n10129825": 15676, "n10130686": 15677, "n10130877": 15678, "n10131151": 15679, "n10131268": 15680, "n10131590": 15681, "n10131815": 15682, "n10132035": 15683, "n10132502": 15684, "n10134178": 15685, "n10134396": 15686, "n10134760": 15687, "n10134982": 15688, "n10135129": 15689, "n10135197": 15690, "n10135297": 15691, "n10136615": 15692, "n10136959": 15693, "n10137825": 15694, "n10138369": 15695, "n10138472": 15696, "n10139077": 15697, "n10139651": 15698, "n10140051": 15699, "n10140597": 15700, "n10140683": 15701, "n10140783": 15702, "n10140929": 15703, "n10141364": 15704, "n10141732": 15705, "n10142166": 15706, "n10142391": 15707, "n10142537": 15708, "n10142747": 15709, "n10142946": 15710, "n10143172": 15711, "n10143595": 15712, "n10143725": 15713, "n10144338": 15714, "n10145239": 15715, "n10145340": 15716, "n10145480": 15717, "n10145590": 15718, "n10145774": 15719, "n10145902": 15720, "n10146002": 15721, "n10146104": 15722, "n10146416": 15723, "n10146816": 15724, "n10146927": 15725, "n10147121": 15726, "n10147262": 15727, "n10147710": 15728, "n10147935": 15729, "n10148035": 15730, "n10148305": 15731, "n10148825": 15732, "n10149436": 15733, "n10149867": 15734, "n10150071": 15735, "n10150794": 15736, "n10150940": 15737, "n10151133": 15738, "n10151261": 15739, "n10151367": 15740, "n10151570": 15741, "n10151760": 15742, "n10152306": 15743, "n10152616": 15744, "n10152763": 15745, "n10153155": 15746, "n10153414": 15747, "n10153594": 15748, "n10153865": 15749, "n10154013": 15750, "n10154186": 15751, "n10154601": 15752, "n10155222": 15753, "n10155600": 15754, "n10155849": 15755, "n10156629": 15756, "n10156831": 15757, "n10157016": 15758, "n10157128": 15759, "n10157271": 15760, "n10158506": 15761, "n10159045": 15762, "n10159289": 15763, "n10159533": 15764, "n10160188": 15765, "n10160280": 15766, "n10160412": 15767, "n10161622": 15768, "n10162016": 15769, "n10162194": 15770, "n10162354": 15771, "n10164025": 15772, "n10164233": 15773, "n10164492": 15774, "n10165448": 15775, "n10166189": 15776, "n10166394": 15777, "n10167152": 15778, "n10167361": 15779, "n10167565": 15780, "n10167838": 15781, "n10168012": 15782, "n10168183": 15783, "n10168584": 15784, "n10168837": 15785, "n10169147": 15786, "n10169241": 15787, "n10169419": 15788, "n10169796": 15789, "n10170060": 15790, "n10170681": 15791, "n10170866": 15792, "n10171219": 15793, "n10171456": 15794, "n10171567": 15795, "n10172080": 15796, "n10173410": 15797, "n10173579": 15798, "n10173665": 15799, "n10173771": 15800, "n10174253": 15801, "n10174330": 15802, "n10174445": 15803, "n10174589": 15804, "n10174695": 15805, "n10174971": 15806, "n10175248": 15807, "n10175725": 15808, "n10176913": 15809, "n10177150": 15810, "n10178077": 15811, "n10178216": 15812, "n10179069": 15813, "n10180580": 15814, "n10180791": 15815, "n10180923": 15816, "n10181445": 15817, "n10181547": 15818, "n10181799": 15819, "n10181878": 15820, "n10182190": 15821, "n10182402": 15822, "n10183347": 15823, "n10183931": 15824, "n10184505": 15825, "n10185148": 15826, "n10185483": 15827, "n10185793": 15828, "n10186068": 15829, "n10186143": 15830, "n10186216": 15831, "n10186350": 15832, "n10186686": 15833, "n10186774": 15834, "n10187130": 15835, "n10187491": 15836, "n10187990": 15837, "n10188715": 15838, "n10188856": 15839, "n10188957": 15840, "n10189278": 15841, "n10189597": 15842, "n10190122": 15843, "n10190516": 15844, "n10191001": 15845, "n10191388": 15846, "n10191613": 15847, "n10192839": 15848, "n10193650": 15849, "n10194231": 15850, "n10194775": 15851, "n10195056": 15852, "n10195155": 15853, "n10195261": 15854, "n10195593": 15855, "n10196404": 15856, "n10196725": 15857, "n10197392": 15858, "n10198437": 15859, "n10198832": 15860, "n10199251": 15861, "n10200246": 15862, "n10200781": 15863, "n10202225": 15864, "n10202624": 15865, "n10202763": 15866, "n10203949": 15867, "n10204177": 15868, "n10204833": 15869, "n10205231": 15870, "n10205344": 15871, "n10205457": 15872, "n10205714": 15873, "n10206173": 15874, "n10206506": 15875, "n10206629": 15876, "n10207077": 15877, "n10207169": 15878, "n10208189": 15879, "n10208847": 15880, "n10208950": 15881, "n10209082": 15882, "n10209731": 15883, "n10210137": 15884, "n10210512": 15885, "n10210648": 15886, "n10210911": 15887, "n10211036": 15888, "n10211666": 15889, "n10211830": 15890, "n10212231": 15891, "n10212501": 15892, "n10212780": 15893, "n10213034": 15894, "n10213429": 15895, "n10214062": 15896, "n10214390": 15897, "n10215623": 15898, "n10216106": 15899, "n10216403": 15900, "n10217208": 15901, "n10218043": 15902, "n10218164": 15903, "n10218292": 15904, "n10219240": 15905, "n10219453": 15906, "n10219879": 15907, "n10220080": 15908, "n10220924": 15909, "n10221312": 15910, "n10221520": 15911, "n10222170": 15912, "n10222259": 15913, "n10222497": 15914, "n10222716": 15915, "n10223069": 15916, "n10223177": 15917, "n10223606": 15918, "n10224578": 15919, "n10225219": 15920, "n10225931": 15921, "n10226413": 15922, "n10227166": 15923, "n10227266": 15924, "n10227393": 15925, "n10227490": 15926, "n10227698": 15927, "n10227793": 15928, "n10227985": 15929, "n10228278": 15930, "n10228468": 15931, "n10228592": 15932, "n10228712": 15933, "n10229883": 15934, "n10230216": 15935, "n10233248": 15936, "n10235024": 15937, "n10235269": 15938, "n10235385": 15939, "n10236304": 15940, "n10236521": 15941, "n10236842": 15942, "n10237069": 15943, "n10237196": 15944, "n10237464": 15945, "n10237556": 15946, "n10237676": 15947, "n10237799": 15948, "n10238272": 15949, "n10238375": 15950, "n10239928": 15951, "n10240082": 15952, "n10240235": 15953, "n10240417": 15954, "n10240821": 15955, "n10241024": 15956, "n10241300": 15957, "n10242328": 15958, "n10243137": 15959, "n10243273": 15960, "n10243483": 15961, "n10243664": 15962, "n10243872": 15963, "n10244108": 15964, "n10244359": 15965, "n10244913": 15966, "n10245029": 15967, "n10245341": 15968, "n10245507": 15969, "n10245639": 15970, "n10245863": 15971, "n10246317": 15972, "n10246395": 15973, "n10246703": 15974, "n10247358": 15975, "n10247880": 15976, "n10248008": 15977, "n10248198": 15978, "n10248377": 15979, "n10249191": 15980, "n10249270": 15981, "n10249459": 15982, "n10249869": 15983, "n10249950": 15984, "n10250712": 15985, "n10251329": 15986, "n10251612": 15987, "n10252075": 15988, "n10252222": 15989, "n10252354": 15990, "n10252547": 15991, "n10253122": 15992, "n10253296": 15993, "n10253479": 15994, "n10253611": 15995, "n10253703": 15996, "n10255459": 15997, "n10257221": 15998, "n10258602": 15999, "n10258786": 16000, "n10259348": 16001, "n10259780": 16002, "n10259997": 16003, "n10260473": 16004, "n10260706": 16005, "n10260800": 16006, "n10261211": 16007, "n10261511": 16008, "n10261624": 16009, "n10261862": 16010, "n10262343": 16011, "n10262445": 16012, "n10262561": 16013, "n10262655": 16014, "n10262880": 16015, "n10263146": 16016, "n10263411": 16017, "n10263790": 16018, "n10265281": 16019, "n10265801": 16020, "n10265891": 16021, "n10266016": 16022, "n10266328": 16023, "n10266848": 16024, "n10267166": 16025, "n10267311": 16026, "n10267865": 16027, "n10268629": 16028, "n10269199": 16029, "n10269289": 16030, "n10271677": 16031, "n10272782": 16032, "n10272913": 16033, "n10273064": 16034, "n10274173": 16035, "n10274318": 16036, "n10274815": 16037, "n10275249": 16038, "n10275395": 16039, "n10275848": 16040, "n10276045": 16041, "n10276477": 16042, "n10276942": 16043, "n10277027": 16044, "n10277638": 16045, "n10277815": 16046, "n10277912": 16047, "n10278456": 16048, "n10279018": 16049, "n10279778": 16050, "n10280034": 16051, "n10280130": 16052, "n10280598": 16053, "n10280674": 16054, "n10281546": 16055, "n10281770": 16056, "n10281896": 16057, "n10282482": 16058, "n10282672": 16059, "n10283170": 16060, "n10283366": 16061, "n10283546": 16062, "n10284064": 16063, "n10284871": 16064, "n10284965": 16065, "n10286282": 16066, "n10286539": 16067, "n10286749": 16068, "n10288964": 16069, "n10289039": 16070, "n10289176": 16071, "n10289462": 16072, "n10289766": 16073, "n10290422": 16074, "n10290541": 16075, "n10290813": 16076, "n10290919": 16077, "n10291110": 16078, "n10291469": 16079, "n10291822": 16080, "n10291942": 16081, "n10292316": 16082, "n10293332": 16083, "n10293590": 16084, "n10293861": 16085, "n10294020": 16086, "n10294139": 16087, "n10295371": 16088, "n10295479": 16089, "n10296176": 16090, "n10296444": 16091, "n10297234": 16092, "n10297367": 16093, "n10297531": 16094, "n10297841": 16095, "n10298202": 16096, "n10298271": 16097, "n10298647": 16098, "n10298912": 16099, "n10299125": 16100, "n10299250": 16101, "n10299700": 16102, "n10299875": 16103, "n10300041": 16104, "n10300154": 16105, "n10300303": 16106, "n10300500": 16107, "n10300654": 16108, "n10300829": 16109, "n10302576": 16110, "n10302700": 16111, "n10302905": 16112, "n10303037": 16113, "n10303814": 16114, "n10304086": 16115, "n10304650": 16116, "n10304914": 16117, "n10305635": 16118, "n10305802": 16119, "n10306004": 16120, "n10306279": 16121, "n10306496": 16122, "n10306595": 16123, "n10306890": 16124, "n10307114": 16125, "n10308066": 16126, "n10308168": 16127, "n10308275": 16128, "n10308504": 16129, "n10308653": 16130, "n10308732": 16131, "n10310783": 16132, "n10311506": 16133, "n10311661": 16134, "n10312287": 16135, "n10312491": 16136, "n10312600": 16137, "n10313000": 16138, "n10313239": 16139, "n10313441": 16140, "n10313724": 16141, "n10314054": 16142, "n10314182": 16143, "n10314517": 16144, "n10314836": 16145, "n10315217": 16146, "n10315456": 16147, "n10315561": 16148, "n10315730": 16149, "n10316360": 16150, "n10316527": 16151, "n10316862": 16152, "n10317007": 16153, "n10317500": 16154, "n10317963": 16155, "n10318293": 16156, "n10318607": 16157, "n10318686": 16158, "n10319313": 16159, "n10320484": 16160, "n10320863": 16161, "n10321126": 16162, "n10321340": 16163, "n10321632": 16164, "n10321882": 16165, "n10322238": 16166, "n10323634": 16167, "n10323752": 16168, "n10323999": 16169, "n10324560": 16170, "n10325549": 16171, "n10325774": 16172, "n10326776": 16173, "n10327143": 16174, "n10327987": 16175, "n10328123": 16176, "n10328328": 16177, "n10328437": 16178, "n10328696": 16179, "n10328941": 16180, "n10329035": 16181, "n10330593": 16182, "n10330931": 16183, "n10331098": 16184, "n10331167": 16185, "n10331258": 16186, "n10331347": 16187, "n10331841": 16188, "n10332110": 16189, "n10332385": 16190, "n10332861": 16191, "n10332953": 16192, "n10333044": 16193, "n10333165": 16194, "n10333317": 16195, "n10333439": 16196, "n10333601": 16197, "n10333838": 16198, "n10334009": 16199, "n10334461": 16200, "n10334782": 16201, "n10335246": 16202, "n10335801": 16203, "n10335931": 16204, "n10336411": 16205, "n10336904": 16206, "n10337488": 16207, "n10338231": 16208, "n10338391": 16209, "n10339179": 16210, "n10339251": 16211, "n10339717": 16212, "n10340312": 16213, "n10341243": 16214, "n10341343": 16215, "n10341446": 16216, "n10341573": 16217, "n10341955": 16218, "n10342180": 16219, "n10342367": 16220, "n10342543": 16221, "n10342893": 16222, "n10342992": 16223, "n10343088": 16224, "n10343355": 16225, "n10343449": 16226, "n10343554": 16227, "n10343869": 16228, "n10344121": 16229, "n10344203": 16230, "n10344319": 16231, "n10344656": 16232, "n10344774": 16233, "n10345015": 16234, "n10345100": 16235, "n10345302": 16236, "n10345422": 16237, "n10345659": 16238, "n10346015": 16239, "n10347204": 16240, "n10347446": 16241, "n10348526": 16242, "n10349243": 16243, "n10349750": 16244, "n10349836": 16245, "n10350220": 16246, "n10350774": 16247, "n10351064": 16248, "n10353016": 16249, "n10353355": 16250, "n10353928": 16251, "n10354265": 16252, "n10354754": 16253, "n10355142": 16254, "n10355306": 16255, "n10355449": 16256, "n10355688": 16257, "n10355806": 16258, "n10356450": 16259, "n10356877": 16260, "n10357012": 16261, "n10357613": 16262, "n10357737": 16263, "n10358032": 16264, "n10358124": 16265, "n10358575": 16266, "n10359117": 16267, "n10359422": 16268, "n10359546": 16269, "n10359659": 16270, "n10360366": 16271, "n10360747": 16272, "n10361060": 16273, "n10361194": 16274, "n10361296": 16275, "n10361525": 16276, "n10362003": 16277, "n10362319": 16278, "n10362557": 16279, "n10363445": 16280, "n10363573": 16281, "n10364198": 16282, "n10364502": 16283, "n10365514": 16284, "n10366145": 16285, "n10366276": 16286, "n10366966": 16287, "n10368291": 16288, "n10368528": 16289, "n10368624": 16290, "n10368711": 16291, "n10368798": 16292, "n10369095": 16293, "n10369317": 16294, "n10369417": 16295, "n10369528": 16296, "n10369699": 16297, "n10369955": 16298, "n10370381": 16299, "n10370955": 16300, "n10371052": 16301, "n10371221": 16302, "n10371330": 16303, "n10371450": 16304, "n10373390": 16305, "n10373525": 16306, "n10374541": 16307, "n10374849": 16308, "n10374943": 16309, "n10375052": 16310, "n10375314": 16311, "n10375402": 16312, "n10376523": 16313, "n10376890": 16314, "n10377021": 16315, "n10377185": 16316, "n10377291": 16317, "n10377542": 16318, "n10377633": 16319, "n10378026": 16320, "n10378113": 16321, "n10378780": 16322, "n10379376": 16323, "n10380126": 16324, "n10380499": 16325, "n10380672": 16326, "n10381804": 16327, "n10381981": 16328, "n10382157": 16329, "n10382302": 16330, "n10382480": 16331, "n10382710": 16332, "n10382825": 16333, "n10383094": 16334, "n10383237": 16335, "n10383505": 16336, "n10383816": 16337, "n10384214": 16338, "n10384392": 16339, "n10384496": 16340, "n10385566": 16341, "n10386196": 16342, "n10386754": 16343, "n10386874": 16344, "n10386984": 16345, "n10387196": 16346, "n10387324": 16347, "n10387836": 16348, "n10389865": 16349, "n10389976": 16350, "n10390600": 16351, "n10390698": 16352, "n10390807": 16353, "n10391416": 16354, "n10393909": 16355, "n10394434": 16356, "n10394786": 16357, "n10395073": 16358, "n10395209": 16359, "n10395390": 16360, "n10395828": 16361, "n10396106": 16362, "n10396337": 16363, "n10396727": 16364, "n10396908": 16365, "n10397001": 16366, "n10397142": 16367, "n10397392": 16368, "n10399130": 16369, "n10400003": 16370, "n10400108": 16371, "n10400205": 16372, "n10400437": 16373, "n10400618": 16374, "n10400998": 16375, "n10401204": 16376, "n10401331": 16377, "n10401639": 16378, "n10402709": 16379, "n10402824": 16380, "n10403633": 16381, "n10403876": 16382, "n10404426": 16383, "n10404998": 16384, "n10405540": 16385, "n10405694": 16386, "n10406266": 16387, "n10406391": 16388, "n10406765": 16389, "n10407310": 16390, "n10407954": 16391, "n10408809": 16392, "n10409459": 16393, "n10409752": 16394, "n10410246": 16395, "n10410996": 16396, "n10411356": 16397, "n10411551": 16398, "n10411867": 16399, "n10414239": 16400, "n10414768": 16401, "n10414865": 16402, "n10415037": 16403, "n10416567": 16404, "n10417288": 16405, "n10417424": 16406, "n10417551": 16407, "n10417682": 16408, "n10417843": 16409, "n10417969": 16410, "n10418101": 16411, "n10418735": 16412, "n10419047": 16413, "n10419472": 16414, "n10419630": 16415, "n10419785": 16416, "n10420031": 16417, "n10420277": 16418, "n10420507": 16419, "n10420649": 16420, "n10421016": 16421, "n10421470": 16422, "n10421956": 16423, "n10422405": 16424, "n10425946": 16425, "n10426454": 16426, "n10426630": 16427, "n10427223": 16428, "n10427359": 16429, "n10427764": 16430, "n10428004": 16431, "n10431122": 16432, "n10431625": 16433, "n10432189": 16434, "n10432441": 16435, "n10432875": 16436, "n10432957": 16437, "n10433077": 16438, "n10433452": 16439, "n10433610": 16440, "n10433737": 16441, "n10435169": 16442, "n10435251": 16443, "n10435716": 16444, "n10435988": 16445, "n10436334": 16446, "n10437014": 16447, "n10437137": 16448, "n10437262": 16449, "n10437698": 16450, "n10438172": 16451, "n10438619": 16452, "n10438842": 16453, "n10439373": 16454, "n10439523": 16455, "n10439727": 16456, "n10439851": 16457, "n10441037": 16458, "n10441124": 16459, "n10441694": 16460, "n10441962": 16461, "n10442093": 16462, "n10442232": 16463, "n10442417": 16464, "n10442573": 16465, "n10443032": 16466, "n10443659": 16467, "n10443830": 16468, "n10444194": 16469, "n10448322": 16470, "n10448455": 16471, "n10449664": 16472, "n10450038": 16473, "n10450161": 16474, "n10450303": 16475, "n10451450": 16476, "n10451590": 16477, "n10451858": 16478, "n10453184": 16479, "n10455619": 16480, "n10456070": 16481, "n10456138": 16482, "n10456696": 16483, "n10457214": 16484, "n10457444": 16485, "n10457903": 16486, "n10458111": 16487, "n10458356": 16488, "n10458596": 16489, "n10459882": 16490, "n10460033": 16491, "n10461060": 16492, "n10462588": 16493, "n10462751": 16494, "n10462860": 16495, "n10464052": 16496, "n10464542": 16497, "n10464711": 16498, "n10464870": 16499, "n10465002": 16500, "n10465451": 16501, "n10465831": 16502, "n10466198": 16503, "n10466564": 16504, "n10466918": 16505, "n10467179": 16506, "n10467395": 16507, "n10468750": 16508, "n10469611": 16509, "n10469874": 16510, "n10470779": 16511, "n10471640": 16512, "n10471732": 16513, "n10471859": 16514, "n10472129": 16515, "n10472447": 16516, "n10473453": 16517, "n10473562": 16518, "n10473789": 16519, "n10473917": 16520, "n10474064": 16521, "n10474343": 16522, "n10474446": 16523, "n10474645": 16524, "n10475835": 16525, "n10475940": 16526, "n10476467": 16527, "n10477713": 16528, "n10477955": 16529, "n10478118": 16530, "n10478293": 16531, "n10478462": 16532, "n10478827": 16533, "n10478960": 16534, "n10479135": 16535, "n10479328": 16536, "n10481167": 16537, "n10481268": 16538, "n10482054": 16539, "n10482220": 16540, "n10482587": 16541, "n10482921": 16542, "n10483138": 16543, "n10483395": 16544, "n10483799": 16545, "n10483890": 16546, "n10484858": 16547, "n10485298": 16548, "n10485883": 16549, "n10486166": 16550, "n10486236": 16551, "n10486561": 16552, "n10487182": 16553, "n10487363": 16554, "n10487592": 16555, "n10488016": 16556, "n10488309": 16557, "n10488656": 16558, "n10489426": 16559, "n10490421": 16560, "n10491998": 16561, "n10492086": 16562, "n10492727": 16563, "n10493199": 16564, "n10493419": 16565, "n10493685": 16566, "n10493835": 16567, "n10493922": 16568, "n10494195": 16569, "n10494373": 16570, "n10495167": 16571, "n10495421": 16572, "n10495555": 16573, "n10495756": 16574, "n10496393": 16575, "n10496489": 16576, "n10497135": 16577, "n10497534": 16578, "n10497645": 16579, "n10498046": 16580, "n10498699": 16581, "n10498816": 16582, "n10498986": 16583, "n10499110": 16584, "n10499232": 16585, "n10499355": 16586, "n10499631": 16587, "n10499857": 16588, "n10500217": 16589, "n10500419": 16590, "n10500603": 16591, "n10500824": 16592, "n10500942": 16593, "n10501453": 16594, "n10501635": 16595, "n10502046": 16596, "n10502329": 16597, "n10502950": 16598, "n10503818": 16599, "n10504090": 16600, "n10504206": 16601, "n10505347": 16602, "n10505613": 16603, "n10505732": 16604, "n10505942": 16605, "n10506336": 16606, "n10506544": 16607, "n10506915": 16608, "n10507070": 16609, "n10507380": 16610, "n10507482": 16611, "n10507565": 16612, "n10507692": 16613, "n10508141": 16614, "n10508379": 16615, "n10508710": 16616, "n10509063": 16617, "n10509161": 16618, "n10509810": 16619, "n10510245": 16620, "n10510974": 16621, "n10511771": 16622, "n10512201": 16623, "n10512372": 16624, "n10512708": 16625, "n10512859": 16626, "n10513509": 16627, "n10513823": 16628, "n10513938": 16629, "n10514051": 16630, "n10514121": 16631, "n10514255": 16632, "n10514429": 16633, "n10514784": 16634, "n10515863": 16635, "n10516527": 16636, "n10517137": 16637, "n10517283": 16638, "n10518349": 16639, "n10519126": 16640, "n10519494": 16641, "n10519984": 16642, "n10520286": 16643, "n10520544": 16644, "n10520964": 16645, "n10521100": 16646, "n10521662": 16647, "n10521853": 16648, "n10522035": 16649, "n10522324": 16650, "n10522759": 16651, "n10523341": 16652, "n10524076": 16653, "n10524223": 16654, "n10524869": 16655, "n10525134": 16656, "n10525436": 16657, "n10525617": 16658, "n10525878": 16659, "n10526534": 16660, "n10527147": 16661, "n10527334": 16662, "n10528023": 16663, "n10528148": 16664, "n10528493": 16665, "n10529231": 16666, "n10530150": 16667, "n10530383": 16668, "n10530571": 16669, "n10530959": 16670, "n10531109": 16671, "n10531445": 16672, "n10531838": 16673, "n10533874": 16674, "n10533983": 16675, "n10536134": 16676, "n10536274": 16677, "n10536416": 16678, "n10537708": 16679, "n10537906": 16680, "n10538629": 16681, "n10538733": 16682, "n10538853": 16683, "n10539015": 16684, "n10539160": 16685, "n10539278": 16686, "n10540114": 16687, "n10540252": 16688, "n10540656": 16689, "n10541833": 16690, "n10542608": 16691, "n10542761": 16692, "n10542888": 16693, "n10543161": 16694, "n10543937": 16695, "n10544232": 16696, "n10544748": 16697, "n10545792": 16698, "n10546428": 16699, "n10546633": 16700, "n10548419": 16701, "n10548537": 16702, "n10548681": 16703, "n10549510": 16704, "n10550252": 16705, "n10550369": 16706, "n10550468": 16707, "n10551576": 16708, "n10552393": 16709, "n10553140": 16710, "n10553235": 16711, "n10554024": 16712, "n10554141": 16713, "n10554846": 16714, "n10555059": 16715, "n10555430": 16716, "n10556033": 16717, "n10556518": 16718, "n10556704": 16719, "n10556825": 16720, "n10557246": 16721, "n10557854": 16722, "n10559009": 16723, "n10559288": 16724, "n10559508": 16725, "n10559683": 16726, "n10559996": 16727, "n10560106": 16728, "n10560637": 16729, "n10561222": 16730, "n10561320": 16731, "n10561736": 16732, "n10562135": 16733, "n10562283": 16734, "n10562509": 16735, "n10562968": 16736, "n10563314": 16737, "n10563403": 16738, "n10563711": 16739, "n10564098": 16740, "n10565502": 16741, "n10565667": 16742, "n10566072": 16743, "n10567613": 16744, "n10567722": 16745, "n10567848": 16746, "n10568200": 16747, "n10568358": 16748, "n10568443": 16749, "n10568608": 16750, "n10568915": 16751, "n10569011": 16752, "n10569179": 16753, "n10570019": 16754, "n10570704": 16755, "n10571907": 16756, "n10572706": 16757, "n10572889": 16758, "n10573957": 16759, "n10574311": 16760, "n10574538": 16761, "n10574840": 16762, "n10575463": 16763, "n10575594": 16764, "n10575787": 16765, "n10576223": 16766, "n10576316": 16767, "n10576676": 16768, "n10576818": 16769, "n10576962": 16770, "n10577182": 16771, "n10577284": 16772, "n10577710": 16773, "n10577820": 16774, "n10578021": 16775, "n10578162": 16776, "n10578471": 16777, "n10578656": 16778, "n10579062": 16779, "n10579549": 16780, "n10580030": 16781, "n10580437": 16782, "n10580535": 16783, "n10581648": 16784, "n10581890": 16785, "n10582604": 16786, "n10582746": 16787, "n10583387": 16788, "n10583790": 16789, "n10585077": 16790, "n10585217": 16791, "n10585628": 16792, "n10586166": 16793, "n10586265": 16794, "n10586444": 16795, "n10586903": 16796, "n10586998": 16797, "n10588074": 16798, "n10588357": 16799, "n10588724": 16800, "n10588965": 16801, "n10589666": 16802, "n10590146": 16803, "n10590239": 16804, "n10590452": 16805, "n10590903": 16806, "n10591072": 16807, "n10591811": 16808, "n10592049": 16809, "n10592811": 16810, "n10593521": 16811, "n10594147": 16812, "n10594523": 16813, "n10594857": 16814, "n10595164": 16815, "n10595647": 16816, "n10596517": 16817, "n10596899": 16818, "n10597505": 16819, "n10597745": 16820, "n10597889": 16821, "n10598013": 16822, "n10598181": 16823, "n10598459": 16824, "n10598904": 16825, "n10599215": 16826, "n10599806": 16827, "n10601234": 16828, "n10601362": 16829, "n10602119": 16830, "n10602470": 16831, "n10602985": 16832, "n10603528": 16833, "n10603851": 16834, "n10604275": 16835, "n10604380": 16836, "n10604634": 16837, "n10604880": 16838, "n10604979": 16839, "n10605253": 16840, "n10605737": 16841, "n10607291": 16842, "n10607478": 16843, "n10609092": 16844, "n10609198": 16845, "n10610465": 16846, "n10610850": 16847, "n10611267": 16848, "n10611613": 16849, "n10612210": 16850, "n10612373": 16851, "n10612518": 16852, "n10613996": 16853, "n10614507": 16854, "n10614629": 16855, "n10615179": 16856, "n10615334": 16857, "n10616578": 16858, "n10617024": 16859, "n10617193": 16860, "n10617397": 16861, "n10618234": 16862, "n10618342": 16863, "n10618465": 16864, "n10618685": 16865, "n10618848": 16866, "n10619492": 16867, "n10619642": 16868, "n10619888": 16869, "n10620212": 16870, "n10620586": 16871, "n10620758": 16872, "n10621294": 16873, "n10621400": 16874, "n10621514": 16875, "n10622053": 16876, "n10624074": 16877, "n10624310": 16878, "n10624437": 16879, "n10624540": 16880, "n10625860": 16881, "n10626630": 16882, "n10627252": 16883, "n10628097": 16884, "n10628644": 16885, "n10629329": 16886, "n10629647": 16887, "n10629939": 16888, "n10630093": 16889, "n10630188": 16890, "n10631131": 16891, "n10631309": 16892, "n10631654": 16893, "n10632576": 16894, "n10633298": 16895, "n10633450": 16896, "n10634464": 16897, "n10634849": 16898, "n10634990": 16899, "n10635788": 16900, "n10636488": 16901, "n10637483": 16902, "n10638922": 16903, "n10639238": 16904, "n10639359": 16905, "n10639637": 16906, "n10639817": 16907, "n10641223": 16908, "n10642596": 16909, "n10642705": 16910, "n10643095": 16911, "n10643837": 16912, "n10643937": 16913, "n10644598": 16914, "n10645017": 16915, "n10645223": 16916, "n10646032": 16917, "n10646140": 16918, "n10646433": 16919, "n10646641": 16920, "n10646780": 16921, "n10646942": 16922, "n10647745": 16923, "n10648237": 16924, "n10648696": 16925, "n10649197": 16926, "n10649308": 16927, "n10650162": 16928, "n10652605": 16929, "n10652703": 16930, "n10654015": 16931, "n10654211": 16932, "n10654321": 16933, "n10654827": 16934, "n10654932": 16935, "n10655169": 16936, "n10655442": 16937, "n10655594": 16938, "n10655730": 16939, "n10655986": 16940, "n10656120": 16941, "n10656223": 16942, "n10656969": 16943, "n10657306": 16944, "n10657556": 16945, "n10657835": 16946, "n10658304": 16947, "n10659042": 16948, "n10659762": 16949, "n10660128": 16950, "n10660621": 16951, "n10660883": 16952, "n10661002": 16953, "n10661216": 16954, "n10661563": 16955, "n10661732": 16956, "n10663315": 16957, "n10663549": 16958, "n10665302": 16959, "n10665587": 16960, "n10665698": 16961, "n10666752": 16962, "n10667477": 16963, "n10667709": 16964, "n10667863": 16965, "n10668450": 16966, "n10668666": 16967, "n10669991": 16968, "n10671042": 16969, "n10671613": 16970, "n10671736": 16971, "n10671898": 16972, "n10672371": 16973, "n10672540": 16974, "n10672662": 16975, "n10673296": 16976, "n10673776": 16977, "n10674130": 16978, "n10674713": 16979, "n10675010": 16980, "n10675142": 16981, "n10675609": 16982, "n10676018": 16983, "n10676434": 16984, "n10676569": 16985, "n10678937": 16986, "n10679174": 16987, "n10679503": 16988, "n10679610": 16989, "n10679723": 16990, "n10680609": 16991, "n10680796": 16992, "n10681194": 16993, "n10681557": 16994, "n10682713": 16995, "n10682953": 16996, "n10683675": 16997, "n10684146": 16998, "n10684630": 16999, "n10684827": 17000, "n10685398": 17001, "n10686073": 17002, "n10686517": 17003, "n10686694": 17004, "n10686885": 17005, "n10688356": 17006, "n10688811": 17007, "n10689306": 17008, "n10690268": 17009, "n10690421": 17010, "n10690648": 17011, "n10691318": 17012, "n10691937": 17013, "n10692090": 17014, "n10692482": 17015, "n10692883": 17016, "n10693235": 17017, "n10693334": 17018, "n10693824": 17019, "n10694258": 17020, "n10694939": 17021, "n10695450": 17022, "n10696101": 17023, "n10696508": 17024, "n10697135": 17025, "n10697282": 17026, "n10698368": 17027, "n10699558": 17028, "n10699752": 17029, "n10699981": 17030, "n10700105": 17031, "n10700201": 17032, "n10700640": 17033, "n10700963": 17034, "n10701180": 17035, "n10701644": 17036, "n10701962": 17037, "n10702167": 17038, "n10702615": 17039, "n10703221": 17040, "n10703336": 17041, "n10703480": 17042, "n10703692": 17043, "n10704238": 17044, "n10704712": 17045, "n10704886": 17046, "n10705448": 17047, "n10705615": 17048, "n10706812": 17049, "n10707134": 17050, "n10707233": 17051, "n10707707": 17052, "n10708292": 17053, "n10708454": 17054, "n10709529": 17055, "n10710171": 17056, "n10710259": 17057, "n10710778": 17058, "n10710913": 17059, "n10711483": 17060, "n10711766": 17061, "n10712229": 17062, "n10712374": 17063, "n10712474": 17064, "n10712690": 17065, "n10712835": 17066, "n10713254": 17067, "n10713686": 17068, "n10713843": 17069, "n10714195": 17070, "n10715030": 17071, "n10715347": 17072, "n10715789": 17073, "n10716576": 17074, "n10716864": 17075, "n10717055": 17076, "n10717196": 17077, "n10717337": 17078, "n10718131": 17079, "n10718349": 17080, "n10718509": 17081, "n10718665": 17082, "n10718952": 17083, "n10719036": 17084, "n10719132": 17085, "n10719267": 17086, "n10719807": 17087, "n10720197": 17088, "n10720453": 17089, "n10720964": 17090, "n10721124": 17091, "n10721321": 17092, "n10721612": 17093, "n10721708": 17094, "n10721819": 17095, "n10722029": 17096, "n10722575": 17097, "n10722965": 17098, "n10723230": 17099, "n10723597": 17100, "n10724132": 17101, "n10724372": 17102, "n10724570": 17103, "n10725280": 17104, "n10726031": 17105, "n10726786": 17106, "n10727016": 17107, "n10727171": 17108, "n10727458": 17109, "n10728117": 17110, "n10728233": 17111, "n10728624": 17112, "n10728998": 17113, "n10729330": 17114, "n10730542": 17115, "n10730728": 17116, "n10731013": 17117, "n10731732": 17118, "n10732010": 17119, "n10732521": 17120, "n10732854": 17121, "n10732967": 17122, "n10733820": 17123, "n10734394": 17124, "n10734741": 17125, "n10734891": 17126, "n10734963": 17127, "n10735173": 17128, "n10735298": 17129, "n10735984": 17130, "n10737103": 17131, "n10737264": 17132, "n10738111": 17133, "n10738215": 17134, "n10738670": 17135, "n10738871": 17136, "n10739135": 17137, "n10739297": 17138, "n10739391": 17139, "n10740594": 17140, "n10740732": 17141, "n10740868": 17142, "n10741152": 17143, "n10741367": 17144, "n10741493": 17145, "n10742005": 17146, "n10742111": 17147, "n10742546": 17148, "n10742997": 17149, "n10743124": 17150, "n10743356": 17151, "n10744078": 17152, "n10744164": 17153, "n10745006": 17154, "n10745770": 17155, "n10746931": 17156, "n10747119": 17157, "n10747424": 17158, "n10747548": 17159, "n10747965": 17160, "n10748142": 17161, "n10748506": 17162, "n10748620": 17163, "n10749928": 17164, "n10750031": 17165, "n10750188": 17166, "n10750640": 17167, "n10751026": 17168, "n10751152": 17169, "n10751265": 17170, "n10751710": 17171, "n10752480": 17172, "n10753061": 17173, "n10753182": 17174, "n10753339": 17175, "n10753442": 17176, "n10753989": 17177, "n10754189": 17178, "n10754281": 17179, "n10754449": 17180, "n10755080": 17181, "n10755164": 17182, "n10755394": 17183, "n10755648": 17184, "n10756061": 17185, "n10756148": 17186, "n10756261": 17187, "n10756641": 17188, "n10756837": 17189, "n10757050": 17190, "n10757492": 17191, "n10758337": 17192, "n10758445": 17193, "n10758949": 17194, "n10759151": 17195, "n10759331": 17196, "n10759982": 17197, "n10760199": 17198, "n10760622": 17199, "n10760951": 17200, "n10761190": 17201, "n10761326": 17202, "n10761519": 17203, "n10762212": 17204, "n10762480": 17205, "n10763075": 17206, "n10763245": 17207, "n10763383": 17208, "n10763620": 17209, "n10764465": 17210, "n10764622": 17211, "n10764719": 17212, "n10765305": 17213, "n10765587": 17214, "n10765679": 17215, "n10765885": 17216, "n10766260": 17217, "n10768148": 17218, "n10768272": 17219, "n10768903": 17220, "n10769084": 17221, "n10769188": 17222, "n10769321": 17223, "n10769459": 17224, "n10771066": 17225, "n10772092": 17226, "n10772580": 17227, "n10772937": 17228, "n10773665": 17229, "n10773800": 17230, "n10774329": 17231, "n10774756": 17232, "n10775003": 17233, "n10775128": 17234, "n10776052": 17235, "n10776339": 17236, "n10776887": 17237, "n10777299": 17238, "n10778044": 17239, "n10778148": 17240, "n10778711": 17241, "n10778999": 17242, "n10779610": 17243, "n10779897": 17244, "n10779995": 17245, "n10780284": 17246, "n10780632": 17247, "n10781236": 17248, "n10781817": 17249, "n10782362": 17250, "n10782471": 17251, "n10782791": 17252, "n10782940": 17253, "n10783240": 17254, "n10783539": 17255, "n10783646": 17256, "n10783734": 17257, "n10784113": 17258, "n10784544": 17259, "n10784922": 17260, "n10785480": 17261, "n10787470": 17262, "n10788852": 17263, "n10789415": 17264, "n10789709": 17265, "n10791115": 17266, "n10791221": 17267, "n10791820": 17268, "n10791890": 17269, "n10792335": 17270, "n10792506": 17271, "n10792856": 17272, "n10793570": 17273, "n10793799": 17274, "n10794014": 17275, "n10801561": 17276, "n10801802": 17277, "n10802507": 17278, "n10802621": 17279, "n10802953": 17280, "n10803031": 17281, "n10803282": 17282, "n10803978": 17283, "n10804287": 17284, "n10804636": 17285, "n10804732": 17286, "n10805501": 17287, "n10806113": 17288, "n10994097": 17289, "n11100798": 17290, "n11196627": 17291, "n11242849": 17292, "n11318824": 17293, "n11346873": 17294, "n11448153": 17295, "n11487732": 17296, "n11508382": 17297, "n11511327": 17298, "n11524451": 17299, "n11530008": 17300, "n11531193": 17301, "n11531334": 17302, "n11532682": 17303, "n11533212": 17304, "n11533999": 17305, "n11536567": 17306, "n11536673": 17307, "n11537327": 17308, "n11539289": 17309, "n11542137": 17310, "n11542640": 17311, "n11544015": 17312, "n11545350": 17313, "n11545524": 17314, "n11545714": 17315, "n11547562": 17316, "n11547855": 17317, "n11548728": 17318, "n11548870": 17319, "n11549009": 17320, "n11549245": 17321, "n11549779": 17322, "n11549895": 17323, "n11552133": 17324, "n11552386": 17325, "n11552594": 17326, "n11552806": 17327, "n11552976": 17328, "n11553240": 17329, "n11553522": 17330, "n11596108": 17331, "n11597657": 17332, "n11598287": 17333, "n11598686": 17334, "n11598886": 17335, "n11599324": 17336, "n11600372": 17337, "n11601177": 17338, "n11601333": 17339, "n11601918": 17340, "n11602091": 17341, "n11602478": 17342, "n11602873": 17343, "n11603246": 17344, "n11603462": 17345, "n11603835": 17346, "n11604046": 17347, "n11608250": 17348, "n11609475": 17349, "n11609684": 17350, "n11609862": 17351, "n11610047": 17352, "n11610215": 17353, "n11610437": 17354, "n11610602": 17355, "n11610823": 17356, "n11611087": 17357, "n11611233": 17358, "n11611356": 17359, "n11611561": 17360, "n11611758": 17361, "n11612018": 17362, "n11612235": 17363, "n11612349": 17364, "n11612575": 17365, "n11612923": 17366, "n11613219": 17367, "n11613459": 17368, "n11613692": 17369, "n11613867": 17370, "n11614039": 17371, "n11614250": 17372, "n11614420": 17373, "n11614713": 17374, "n11615026": 17375, "n11615259": 17376, "n11615387": 17377, "n11615607": 17378, "n11615812": 17379, "n11615967": 17380, "n11616260": 17381, "n11616486": 17382, "n11616662": 17383, "n11616852": 17384, "n11617090": 17385, "n11617272": 17386, "n11617631": 17387, "n11617878": 17388, "n11618079": 17389, "n11618290": 17390, "n11618525": 17391, "n11618861": 17392, "n11619227": 17393, "n11619455": 17394, "n11619687": 17395, "n11619845": 17396, "n11620016": 17397, "n11620389": 17398, "n11620673": 17399, "n11621029": 17400, "n11621281": 17401, "n11621547": 17402, "n11621727": 17403, "n11621950": 17404, "n11622184": 17405, "n11622368": 17406, "n11622591": 17407, "n11622771": 17408, "n11623105": 17409, "n11623815": 17410, "n11623967": 17411, "n11624192": 17412, "n11624531": 17413, "n11625003": 17414, "n11625223": 17415, "n11625391": 17416, "n11625632": 17417, "n11625804": 17418, "n11626010": 17419, "n11626152": 17420, "n11626409": 17421, "n11626585": 17422, "n11626826": 17423, "n11627168": 17424, "n11627512": 17425, "n11627714": 17426, "n11627908": 17427, "n11628087": 17428, "n11628456": 17429, "n11628793": 17430, "n11629047": 17431, "n11629354": 17432, "n11630017": 17433, "n11630489": 17434, "n11631159": 17435, "n11631405": 17436, "n11631619": 17437, "n11631854": 17438, "n11631985": 17439, "n11632167": 17440, "n11632376": 17441, "n11632619": 17442, "n11632929": 17443, "n11633284": 17444, "n11634736": 17445, "n11635152": 17446, "n11635433": 17447, "n11635830": 17448, "n11636204": 17449, "n11636835": 17450, "n11639084": 17451, "n11639306": 17452, "n11639445": 17453, "n11640132": 17454, "n11643835": 17455, "n11644046": 17456, "n11644226": 17457, "n11644462": 17458, "n11644872": 17459, "n11645163": 17460, "n11645590": 17461, "n11645914": 17462, "n11646167": 17463, "n11646344": 17464, "n11646517": 17465, "n11646694": 17466, "n11646955": 17467, "n11647306": 17468, "n11647703": 17469, "n11647868": 17470, "n11648039": 17471, "n11648268": 17472, "n11648776": 17473, "n11649150": 17474, "n11649359": 17475, "n11649878": 17476, "n11650160": 17477, "n11650307": 17478, "n11650430": 17479, "n11650558": 17480, "n11650759": 17481, "n11652039": 17482, "n11652217": 17483, "n11652376": 17484, "n11652578": 17485, "n11652753": 17486, "n11652966": 17487, "n11653126": 17488, "n11653570": 17489, "n11653904": 17490, "n11654293": 17491, "n11654438": 17492, "n11654984": 17493, "n11655152": 17494, "n11655592": 17495, "n11655974": 17496, "n11656123": 17497, "n11656549": 17498, "n11656771": 17499, "n11657585": 17500, "n11658331": 17501, "n11658544": 17502, "n11658709": 17503, "n11659248": 17504, "n11659627": 17505, "n11660300": 17506, "n11661372": 17507, "n11661909": 17508, "n11662128": 17509, "n11662371": 17510, "n11662585": 17511, "n11662937": 17512, "n11663263": 17513, "n11664418": 17514, "n11665372": 17515, "n11666854": 17516, "n11668117": 17517, "n11669786": 17518, "n11669921": 17519, "n11672269": 17520, "n11672400": 17521, "n11674019": 17522, "n11674332": 17523, "n11675025": 17524, "n11675404": 17525, "n11675738": 17526, "n11676500": 17527, "n11676743": 17528, "n11676850": 17529, "n11677485": 17530, "n11677902": 17531, "n11678010": 17532, "n11678299": 17533, "n11678377": 17534, "n11679378": 17535, "n11680457": 17536, "n11680596": 17537, "n11682659": 17538, "n11683216": 17539, "n11683838": 17540, "n11684264": 17541, "n11684499": 17542, "n11684654": 17543, "n11685091": 17544, "n11685621": 17545, "n11686195": 17546, "n11686652": 17547, "n11686780": 17548, "n11686912": 17549, "n11687071": 17550, "n11687432": 17551, "n11687789": 17552, "n11687964": 17553, "n11688069": 17554, "n11688378": 17555, "n11689197": 17556, "n11689367": 17557, "n11689483": 17558, "n11689678": 17559, "n11689815": 17560, "n11689957": 17561, "n11690088": 17562, "n11690254": 17563, "n11690455": 17564, "n11691046": 17565, "n11691857": 17566, "n11692265": 17567, "n11692792": 17568, "n11693981": 17569, "n11694300": 17570, "n11694469": 17571, "n11694664": 17572, "n11694866": 17573, "n11695085": 17574, "n11695285": 17575, "n11695599": 17576, "n11695974": 17577, "n11696450": 17578, "n11696935": 17579, "n11697560": 17580, "n11697802": 17581, "n11698042": 17582, "n11698245": 17583, "n11699442": 17584, "n11699751": 17585, "n11700058": 17586, "n11700279": 17587, "n11700864": 17588, "n11701066": 17589, "n11701302": 17590, "n11702713": 17591, "n11703669": 17592, "n11704093": 17593, "n11704620": 17594, "n11704791": 17595, "n11705171": 17596, "n11705387": 17597, "n11705573": 17598, "n11705776": 17599, "n11706325": 17600, "n11706761": 17601, "n11706942": 17602, "n11707229": 17603, "n11707827": 17604, "n11708658": 17605, "n11708857": 17606, "n11709045": 17607, "n11709205": 17608, "n11709674": 17609, "n11710136": 17610, "n11710393": 17611, "n11710658": 17612, "n11710827": 17613, "n11710987": 17614, "n11711289": 17615, "n11711537": 17616, "n11711764": 17617, "n11711971": 17618, "n11712282": 17619, "n11713164": 17620, "n11713370": 17621, "n11713763": 17622, "n11714382": 17623, "n11715430": 17624, "n11715678": 17625, "n11716698": 17626, "n11717399": 17627, "n11717577": 17628, "n11718296": 17629, "n11718681": 17630, "n11719286": 17631, "n11720353": 17632, "n11720643": 17633, "n11720891": 17634, "n11721337": 17635, "n11721642": 17636, "n11722036": 17637, "n11722342": 17638, "n11722466": 17639, "n11722621": 17640, "n11722982": 17641, "n11723227": 17642, "n11723452": 17643, "n11723770": 17644, "n11723986": 17645, "n11724109": 17646, "n11724660": 17647, "n11725015": 17648, "n11725311": 17649, "n11725480": 17650, "n11725623": 17651, "n11725821": 17652, "n11725973": 17653, "n11726145": 17654, "n11726269": 17655, "n11726433": 17656, "n11726707": 17657, "n11727091": 17658, "n11727358": 17659, "n11727540": 17660, "n11727738": 17661, "n11728099": 17662, "n11728769": 17663, "n11728945": 17664, "n11729142": 17665, "n11729478": 17666, "n11729860": 17667, "n11730015": 17668, "n11730458": 17669, "n11730602": 17670, "n11730750": 17671, "n11730933": 17672, "n11731157": 17673, "n11731659": 17674, "n11732052": 17675, "n11732567": 17676, "n11733054": 17677, "n11733312": 17678, "n11733548": 17679, "n11734493": 17680, "n11734698": 17681, "n11735053": 17682, "n11735570": 17683, "n11735977": 17684, "n11736362": 17685, "n11736694": 17686, "n11736851": 17687, "n11737009": 17688, "n11737125": 17689, "n11737534": 17690, "n11738547": 17691, "n11738997": 17692, "n11739365": 17693, "n11739978": 17694, "n11740414": 17695, "n11741175": 17696, "n11741350": 17697, "n11741575": 17698, "n11741797": 17699, "n11742310": 17700, "n11742878": 17701, "n11744011": 17702, "n11744108": 17703, "n11744471": 17704, "n11745817": 17705, "n11746600": 17706, "n11747468": 17707, "n11748002": 17708, "n11748811": 17709, "n11749112": 17710, "n11749603": 17711, "n11750173": 17712, "n11750508": 17713, "n11750989": 17714, "n11751765": 17715, "n11751974": 17716, "n11752578": 17717, "n11752798": 17718, "n11752937": 17719, "n11753143": 17720, "n11753355": 17721, "n11753562": 17722, "n11753700": 17723, "n11754893": 17724, "n11756092": 17725, "n11756329": 17726, "n11756669": 17727, "n11756870": 17728, "n11757017": 17729, "n11757190": 17730, "n11757653": 17731, "n11757851": 17732, "n11758122": 17733, "n11758276": 17734, "n11758483": 17735, "n11758799": 17736, "n11759224": 17737, "n11759404": 17738, "n11759609": 17739, "n11759853": 17740, "n11760785": 17741, "n11761202": 17742, "n11761650": 17743, "n11761836": 17744, "n11762018": 17745, "n11762433": 17746, "n11762927": 17747, "n11763142": 17748, "n11763625": 17749, "n11763874": 17750, "n11764478": 17751, "n11764814": 17752, "n11765568": 17753, "n11766046": 17754, "n11766189": 17755, "n11766432": 17756, "n11767354": 17757, "n11767877": 17758, "n11768816": 17759, "n11769176": 17760, "n11769621": 17761, "n11769803": 17762, "n11770256": 17763, "n11771147": 17764, "n11771539": 17765, "n11771746": 17766, "n11771924": 17767, "n11772408": 17768, "n11772879": 17769, "n11773408": 17770, "n11773628": 17771, "n11773987": 17772, "n11774513": 17773, "n11774972": 17774, "n11775340": 17775, "n11775626": 17776, "n11776234": 17777, "n11777080": 17778, "n11778092": 17779, "n11778257": 17780, "n11779300": 17781, "n11780148": 17782, "n11780424": 17783, "n11781176": 17784, "n11782036": 17785, "n11782266": 17786, "n11782761": 17787, "n11782878": 17788, "n11783162": 17789, "n11783920": 17790, "n11784126": 17791, "n11784497": 17792, "n11785276": 17793, "n11785668": 17794, "n11785875": 17795, "n11786131": 17796, "n11786539": 17797, "n11786843": 17798, "n11787190": 17799, "n11788039": 17800, "n11788727": 17801, "n11789066": 17802, "n11789438": 17803, "n11789589": 17804, "n11789962": 17805, "n11790089": 17806, "n11790788": 17807, "n11790936": 17808, "n11791341": 17809, "n11791569": 17810, "n11792029": 17811, "n11792341": 17812, "n11792742": 17813, "n11793403": 17814, "n11793779": 17815, "n11794024": 17816, "n11794139": 17817, "n11794519": 17818, "n11795049": 17819, "n11795216": 17820, "n11795580": 17821, "n11796005": 17822, "n11796188": 17823, "n11797321": 17824, "n11797508": 17825, "n11797981": 17826, "n11798270": 17827, "n11798496": 17828, "n11798688": 17829, "n11798978": 17830, "n11799331": 17831, "n11799732": 17832, "n11800236": 17833, "n11800565": 17834, "n11801392": 17835, "n11801665": 17836, "n11801891": 17837, "n11802410": 17838, "n11802586": 17839, "n11802800": 17840, "n11802995": 17841, "n11805255": 17842, "n11805544": 17843, "n11805956": 17844, "n11806219": 17845, "n11806369": 17846, "n11806521": 17847, "n11806679": 17848, "n11806814": 17849, "n11807108": 17850, "n11807525": 17851, "n11807696": 17852, "n11807979": 17853, "n11808299": 17854, "n11808468": 17855, "n11808721": 17856, "n11808932": 17857, "n11809094": 17858, "n11809271": 17859, "n11809437": 17860, "n11809594": 17861, "n11809754": 17862, "n11810030": 17863, "n11810358": 17864, "n11811059": 17865, "n11811473": 17866, "n11811706": 17867, "n11811921": 17868, "n11812094": 17869, "n11812910": 17870, "n11813077": 17871, "n11814584": 17872, "n11814996": 17873, "n11815491": 17874, "n11815721": 17875, "n11815918": 17876, "n11816121": 17877, "n11816336": 17878, "n11816649": 17879, "n11816829": 17880, "n11817160": 17881, "n11817501": 17882, "n11817914": 17883, "n11818069": 17884, "n11818636": 17885, "n11819509": 17886, "n11819912": 17887, "n11820965": 17888, "n11821184": 17889, "n11822300": 17890, "n11823043": 17891, "n11823305": 17892, "n11823436": 17893, "n11823756": 17894, "n11824146": 17895, "n11824344": 17896, "n11824747": 17897, "n11825351": 17898, "n11825749": 17899, "n11826198": 17900, "n11826569": 17901, "n11827541": 17902, "n11828577": 17903, "n11828973": 17904, "n11829205": 17905, "n11829672": 17906, "n11829922": 17907, "n11830045": 17908, "n11830252": 17909, "n11830400": 17910, "n11830714": 17911, "n11830906": 17912, "n11831100": 17913, "n11831297": 17914, "n11831521": 17915, "n11832214": 17916, "n11832480": 17917, "n11832671": 17918, "n11832899": 17919, "n11833373": 17920, "n11833749": 17921, "n11834272": 17922, "n11834654": 17923, "n11834890": 17924, "n11835251": 17925, "n11836327": 17926, "n11836722": 17927, "n11837204": 17928, "n11837351": 17929, "n11837562": 17930, "n11837743": 17931, "n11837970": 17932, "n11838413": 17933, "n11838916": 17934, "n11839460": 17935, "n11839568": 17936, "n11839823": 17937, "n11840067": 17938, "n11840246": 17939, "n11840476": 17940, "n11840764": 17941, "n11841247": 17942, "n11843441": 17943, "n11844371": 17944, "n11844892": 17945, "n11845557": 17946, "n11845793": 17947, "n11845913": 17948, "n11846312": 17949, "n11846425": 17950, "n11846765": 17951, "n11847169": 17952, "n11848479": 17953, "n11848867": 17954, "n11849271": 17955, "n11849467": 17956, "n11849871": 17957, "n11849983": 17958, "n11850521": 17959, "n11850918": 17960, "n11851258": 17961, "n11851578": 17962, "n11851839": 17963, "n11852028": 17964, "n11852148": 17965, "n11852531": 17966, "n11853079": 17967, "n11853356": 17968, "n11853813": 17969, "n11854479": 17970, "n11855274": 17971, "n11855435": 17972, "n11855553": 17973, "n11855842": 17974, "n11856573": 17975, "n11857696": 17976, "n11857875": 17977, "n11858077": 17978, "n11858703": 17979, "n11858814": 17980, "n11859275": 17981, "n11859472": 17982, "n11859737": 17983, "n11860208": 17984, "n11860555": 17985, "n11861238": 17986, "n11861487": 17987, "n11861641": 17988, "n11861853": 17989, "n11862835": 17990, "n11863467": 17991, "n11863877": 17992, "n11865071": 17993, "n11865276": 17994, "n11865429": 17995, "n11865574": 17996, "n11865874": 17997, "n11866248": 17998, "n11866706": 17999, "n11867311": 18000, "n11868814": 18001, "n11869351": 18002, "n11869689": 18003, "n11870044": 18004, "n11870418": 18005, "n11870747": 18006, "n11871059": 18007, "n11871496": 18008, "n11871748": 18009, "n11872146": 18010, "n11872324": 18011, "n11872658": 18012, "n11873182": 18013, "n11873612": 18014, "n11874081": 18015, "n11874423": 18016, "n11874878": 18017, "n11875523": 18018, "n11875691": 18019, "n11875938": 18020, "n11876204": 18021, "n11876432": 18022, "n11876634": 18023, "n11876803": 18024, "n11877193": 18025, "n11877283": 18026, "n11877473": 18027, "n11877646": 18028, "n11877860": 18029, "n11878101": 18030, "n11878283": 18031, "n11878633": 18032, "n11879054": 18033, "n11879722": 18034, "n11879895": 18035, "n11881189": 18036, "n11882074": 18037, "n11882237": 18038, "n11882426": 18039, "n11882636": 18040, "n11882821": 18041, "n11882972": 18042, "n11883328": 18043, "n11883628": 18044, "n11883945": 18045, "n11884384": 18046, "n11884967": 18047, "n11885856": 18048, "n11887119": 18049, "n11887310": 18050, "n11887476": 18051, "n11887750": 18052, "n11888061": 18053, "n11888424": 18054, "n11888800": 18055, "n11889205": 18056, "n11889619": 18057, "n11890022": 18058, "n11890150": 18059, "n11890884": 18060, "n11891175": 18061, "n11892029": 18062, "n11892181": 18063, "n11892637": 18064, "n11892817": 18065, "n11893640": 18066, "n11893916": 18067, "n11894327": 18068, "n11894558": 18069, "n11894770": 18070, "n11895092": 18071, "n11895472": 18072, "n11895714": 18073, "n11896141": 18074, "n11896722": 18075, "n11897116": 18076, "n11897466": 18077, "n11898639": 18078, "n11898775": 18079, "n11899223": 18080, "n11899762": 18081, "n11899921": 18082, "n11900569": 18083, "n11901294": 18084, "n11901452": 18085, "n11901597": 18086, "n11901759": 18087, "n11901977": 18088, "n11902200": 18089, "n11902389": 18090, "n11902709": 18091, "n11902982": 18092, "n11903333": 18093, "n11903671": 18094, "n11904109": 18095, "n11904274": 18096, "n11905392": 18097, "n11905749": 18098, "n11906127": 18099, "n11906514": 18100, "n11906917": 18101, "n11907100": 18102, "n11907405": 18103, "n11907689": 18104, "n11908549": 18105, "n11908846": 18106, "n11909864": 18107, "n11910271": 18108, "n11910460": 18109, "n11910666": 18110, "n11915214": 18111, "n11915658": 18112, "n11915899": 18113, "n11916467": 18114, "n11916696": 18115, "n11917407": 18116, "n11917835": 18117, "n11918286": 18118, "n11918473": 18119, "n11918808": 18120, "n11919447": 18121, "n11919761": 18122, "n11919975": 18123, "n11920133": 18124, "n11920498": 18125, "n11920663": 18126, "n11920998": 18127, "n11921395": 18128, "n11921792": 18129, "n11922661": 18130, "n11922755": 18131, "n11922839": 18132, "n11922926": 18133, "n11923174": 18134, "n11923397": 18135, "n11923637": 18136, "n11924014": 18137, "n11924445": 18138, "n11924849": 18139, "n11925303": 18140, "n11925450": 18141, "n11925898": 18142, "n11926365": 18143, "n11926833": 18144, "n11926976": 18145, "n11927215": 18146, "n11927740": 18147, "n11928352": 18148, "n11928858": 18149, "n11929743": 18150, "n11930038": 18151, "n11930203": 18152, "n11930353": 18153, "n11930571": 18154, "n11930788": 18155, "n11930994": 18156, "n11931135": 18157, "n11931540": 18158, "n11931918": 18159, "n11932745": 18160, "n11932927": 18161, "n11933099": 18162, "n11933257": 18163, "n11933387": 18164, "n11933546": 18165, "n11933728": 18166, "n11933903": 18167, "n11934041": 18168, "n11934239": 18169, "n11934463": 18170, "n11934616": 18171, "n11934807": 18172, "n11935027": 18173, "n11935187": 18174, "n11935330": 18175, "n11935469": 18176, "n11935627": 18177, "n11935715": 18178, "n11935794": 18179, "n11935877": 18180, "n11935953": 18181, "n11936027": 18182, "n11936113": 18183, "n11936199": 18184, "n11936287": 18185, "n11936369": 18186, "n11936448": 18187, "n11936539": 18188, "n11936624": 18189, "n11936707": 18190, "n11936782": 18191, "n11936864": 18192, "n11936946": 18193, "n11937023": 18194, "n11937102": 18195, "n11937195": 18196, "n11937278": 18197, "n11937360": 18198, "n11937446": 18199, "n11937692": 18200, "n11938556": 18201, "n11939180": 18202, "n11939491": 18203, "n11939699": 18204, "n11940006": 18205, "n11940349": 18206, "n11940599": 18207, "n11940750": 18208, "n11941094": 18209, "n11941478": 18210, "n11941924": 18211, "n11942659": 18212, "n11943133": 18213, "n11943407": 18214, "n11943660": 18215, "n11943992": 18216, "n11944196": 18217, "n11944751": 18218, "n11944954": 18219, "n11945367": 18220, "n11945514": 18221, "n11945783": 18222, "n11946051": 18223, "n11946313": 18224, "n11946727": 18225, "n11946918": 18226, "n11947251": 18227, "n11947629": 18228, "n11947802": 18229, "n11948044": 18230, "n11948264": 18231, "n11948469": 18232, "n11948864": 18233, "n11949015": 18234, "n11949402": 18235, "n11949857": 18236, "n11950345": 18237, "n11950686": 18238, "n11950877": 18239, "n11951052": 18240, "n11951511": 18241, "n11951820": 18242, "n11952346": 18243, "n11952541": 18244, "n11953038": 18245, "n11953339": 18246, "n11953610": 18247, "n11953884": 18248, "n11954161": 18249, "n11954345": 18250, "n11954484": 18251, "n11954642": 18252, "n11954798": 18253, "n11955040": 18254, "n11955153": 18255, "n11955532": 18256, "n11955896": 18257, "n11956348": 18258, "n11956850": 18259, "n11957317": 18260, "n11957514": 18261, "n11957678": 18262, "n11958080": 18263, "n11958499": 18264, "n11958888": 18265, "n11959259": 18266, "n11959632": 18267, "n11959862": 18268, "n11960245": 18269, "n11960673": 18270, "n11961100": 18271, "n11961446": 18272, "n11961871": 18273, "n11962272": 18274, "n11962667": 18275, "n11962994": 18276, "n11963572": 18277, "n11963932": 18278, "n11964446": 18279, "n11964848": 18280, "n11965218": 18281, "n11965627": 18282, "n11965962": 18283, "n11966083": 18284, "n11966215": 18285, "n11966385": 18286, "n11966617": 18287, "n11966896": 18288, "n11967142": 18289, "n11967315": 18290, "n11967744": 18291, "n11967878": 18292, "n11968519": 18293, "n11968704": 18294, "n11968931": 18295, "n11969166": 18296, "n11969607": 18297, "n11969806": 18298, "n11970101": 18299, "n11970298": 18300, "n11970586": 18301, "n11971248": 18302, "n11971406": 18303, "n11971783": 18304, "n11971927": 18305, "n11972291": 18306, "n11972759": 18307, "n11972959": 18308, "n11973341": 18309, "n11973634": 18310, "n11973749": 18311, "n11974373": 18312, "n11974557": 18313, "n11974888": 18314, "n11975254": 18315, "n11976170": 18316, "n11976314": 18317, "n11976511": 18318, "n11976933": 18319, "n11977303": 18320, "n11977660": 18321, "n11977887": 18322, "n11978233": 18323, "n11978551": 18324, "n11978713": 18325, "n11978961": 18326, "n11979187": 18327, "n11979354": 18328, "n11979527": 18329, "n11979715": 18330, "n11979964": 18331, "n11980318": 18332, "n11980682": 18333, "n11981192": 18334, "n11981475": 18335, "n11982115": 18336, "n11982545": 18337, "n11982939": 18338, "n11983375": 18339, "n11983606": 18340, "n11984144": 18341, "n11984542": 18342, "n11985053": 18343, "n11985321": 18344, "n11985739": 18345, "n11985903": 18346, "n11986511": 18347, "n11986729": 18348, "n11987126": 18349, "n11987349": 18350, "n11987511": 18351, "n11988132": 18352, "n11988596": 18353, "n11988893": 18354, "n11989087": 18355, "n11989393": 18356, "n11989869": 18357, "n11990167": 18358, "n11990313": 18359, "n11990627": 18360, "n11990920": 18361, "n11991263": 18362, "n11991549": 18363, "n11991777": 18364, "n11992479": 18365, "n11992806": 18366, "n11993203": 18367, "n11993444": 18368, "n11993675": 18369, "n11994150": 18370, "n11995092": 18371, "n11995396": 18372, "n11996251": 18373, "n11996677": 18374, "n11997032": 18375, "n11997160": 18376, "n11997969": 18377, "n11998492": 18378, "n11998888": 18379, "n11999278": 18380, "n11999656": 18381, "n12000191": 18382, "n12001294": 18383, "n12001707": 18384, "n12001924": 18385, "n12002428": 18386, "n12002651": 18387, "n12002826": 18388, "n12003167": 18389, "n12003696": 18390, "n12004120": 18391, "n12004547": 18392, "n12004987": 18393, "n12005656": 18394, "n12006306": 18395, "n12006766": 18396, "n12006930": 18397, "n12007196": 18398, "n12007406": 18399, "n12007766": 18400, "n12008252": 18401, "n12008487": 18402, "n12008749": 18403, "n12009047": 18404, "n12009420": 18405, "n12009792": 18406, "n12010628": 18407, "n12010815": 18408, "n12011370": 18409, "n12011620": 18410, "n12012111": 18411, "n12012253": 18412, "n12012510": 18413, "n12013035": 18414, "n12013511": 18415, "n12013701": 18416, "n12014085": 18417, "n12014355": 18418, "n12014923": 18419, "n12015221": 18420, "n12015525": 18421, "n12015959": 18422, "n12016434": 18423, "n12016567": 18424, "n12016777": 18425, "n12016914": 18426, "n12017127": 18427, "n12017326": 18428, "n12017511": 18429, "n12017664": 18430, "n12017853": 18431, "n12018014": 18432, "n12018100": 18433, "n12018188": 18434, "n12018271": 18435, "n12018363": 18436, "n12018447": 18437, "n12018530": 18438, "n12018760": 18439, "n12019035": 18440, "n12019827": 18441, "n12020184": 18442, "n12020507": 18443, "n12020736": 18444, "n12020941": 18445, "n12022054": 18446, "n12022382": 18447, "n12022821": 18448, "n12023108": 18449, "n12023407": 18450, "n12023726": 18451, "n12024176": 18452, "n12024445": 18453, "n12024690": 18454, "n12024805": 18455, "n12025220": 18456, "n12026018": 18457, "n12026476": 18458, "n12026981": 18459, "n12027222": 18460, "n12027658": 18461, "n12028424": 18462, "n12029039": 18463, "n12029635": 18464, "n12030092": 18465, "n12030654": 18466, "n12030908": 18467, "n12031139": 18468, "n12031388": 18469, "n12031547": 18470, "n12031927": 18471, "n12032429": 18472, "n12032686": 18473, "n12033139": 18474, "n12033504": 18475, "n12033709": 18476, "n12034141": 18477, "n12034384": 18478, "n12034594": 18479, "n12035631": 18480, "n12035907": 18481, "n12036067": 18482, "n12036226": 18483, "n12036939": 18484, "n12037499": 18485, "n12037691": 18486, "n12038038": 18487, "n12038208": 18488, "n12038406": 18489, "n12038585": 18490, "n12038760": 18491, "n12038898": 18492, "n12039317": 18493, "n12041446": 18494, "n12043444": 18495, "n12043673": 18496, "n12043836": 18497, "n12044041": 18498, "n12044467": 18499, "n12044784": 18500, "n12045157": 18501, "n12045514": 18502, "n12045860": 18503, "n12046028": 18504, "n12046428": 18505, "n12046815": 18506, "n12047345": 18507, "n12047884": 18508, "n12048056": 18509, "n12048399": 18510, "n12048928": 18511, "n12049282": 18512, "n12049562": 18513, "n12050533": 18514, "n12050959": 18515, "n12051103": 18516, "n12051514": 18517, "n12051792": 18518, "n12052267": 18519, "n12052447": 18520, "n12052787": 18521, "n12053405": 18522, "n12053690": 18523, "n12053962": 18524, "n12054195": 18525, "n12055073": 18526, "n12055516": 18527, "n12056099": 18528, "n12056217": 18529, "n12056601": 18530, "n12056758": 18531, "n12056990": 18532, "n12057211": 18533, "n12057447": 18534, "n12057660": 18535, "n12057895": 18536, "n12058192": 18537, "n12058630": 18538, "n12058822": 18539, "n12059314": 18540, "n12059625": 18541, "n12060546": 18542, "n12061104": 18543, "n12061380": 18544, "n12061614": 18545, "n12062105": 18546, "n12062468": 18547, "n12062626": 18548, "n12062781": 18549, "n12063211": 18550, "n12063639": 18551, "n12064389": 18552, "n12064591": 18553, "n12065316": 18554, "n12065649": 18555, "n12065777": 18556, "n12066018": 18557, "n12066261": 18558, "n12066451": 18559, "n12066630": 18560, "n12066821": 18561, "n12067029": 18562, "n12067193": 18563, "n12067433": 18564, "n12067672": 18565, "n12067817": 18566, "n12068138": 18567, "n12068432": 18568, "n12068615": 18569, "n12069009": 18570, "n12069217": 18571, "n12069679": 18572, "n12070016": 18573, "n12070381": 18574, "n12070583": 18575, "n12070712": 18576, "n12071259": 18577, "n12071477": 18578, "n12071744": 18579, "n12072210": 18580, "n12072722": 18581, "n12073217": 18582, "n12073554": 18583, "n12073991": 18584, "n12074408": 18585, "n12074867": 18586, "n12075010": 18587, "n12075151": 18588, "n12075299": 18589, "n12075830": 18590, "n12076223": 18591, "n12076577": 18592, "n12076852": 18593, "n12077244": 18594, "n12077944": 18595, "n12078172": 18596, "n12078451": 18597, "n12078747": 18598, "n12079120": 18599, "n12079523": 18600, "n12079963": 18601, "n12080395": 18602, "n12080588": 18603, "n12080820": 18604, "n12081215": 18605, "n12081649": 18606, "n12082131": 18607, "n12083113": 18608, "n12083591": 18609, "n12083847": 18610, "n12084158": 18611, "n12084400": 18612, "n12084555": 18613, "n12084890": 18614, "n12085267": 18615, "n12085664": 18616, "n12086012": 18617, "n12086192": 18618, "n12086539": 18619, "n12086778": 18620, "n12087961": 18621, "n12088223": 18622, "n12088327": 18623, "n12088495": 18624, "n12088909": 18625, "n12089320": 18626, "n12089496": 18627, "n12089846": 18628, "n12090890": 18629, "n12091213": 18630, "n12091377": 18631, "n12091550": 18632, "n12091697": 18633, "n12091953": 18634, "n12092262": 18635, "n12092417": 18636, "n12092629": 18637, "n12092930": 18638, "n12093329": 18639, "n12093600": 18640, "n12093885": 18641, "n12094244": 18642, "n12094401": 18643, "n12094612": 18644, "n12095020": 18645, "n12095281": 18646, "n12095412": 18647, "n12095543": 18648, "n12095647": 18649, "n12095934": 18650, "n12096089": 18651, "n12096395": 18652, "n12096563": 18653, "n12096674": 18654, "n12097396": 18655, "n12097556": 18656, "n12098403": 18657, "n12098524": 18658, "n12098827": 18659, "n12099342": 18660, "n12100187": 18661, "n12101870": 18662, "n12102133": 18663, "n12103680": 18664, "n12103894": 18665, "n12104104": 18666, "n12104238": 18667, "n12104501": 18668, "n12104734": 18669, "n12105125": 18670, "n12105353": 18671, "n12105828": 18672, "n12105981": 18673, "n12106134": 18674, "n12106323": 18675, "n12107002": 18676, "n12107191": 18677, "n12107710": 18678, "n12107970": 18679, "n12108432": 18680, "n12108613": 18681, "n12108871": 18682, "n12109365": 18683, "n12109827": 18684, "n12110085": 18685, "n12110236": 18686, "n12110352": 18687, "n12110475": 18688, "n12110778": 18689, "n12111238": 18690, "n12111627": 18691, "n12112008": 18692, "n12112337": 18693, "n12112609": 18694, "n12112918": 18695, "n12113195": 18696, "n12113323": 18697, "n12113657": 18698, "n12114010": 18699, "n12114590": 18700, "n12115180": 18701, "n12116058": 18702, "n12116429": 18703, "n12116734": 18704, "n12117017": 18705, "n12117235": 18706, "n12117326": 18707, "n12117695": 18708, "n12117912": 18709, "n12118414": 18710, "n12118661": 18711, "n12119099": 18712, "n12119238": 18713, "n12119390": 18714, "n12119539": 18715, "n12119717": 18716, "n12120347": 18717, "n12120578": 18718, "n12121033": 18719, "n12121187": 18720, "n12121610": 18721, "n12122442": 18722, "n12122725": 18723, "n12122918": 18724, "n12123648": 18725, "n12123741": 18726, "n12124172": 18727, "n12124627": 18728, "n12124818": 18729, "n12125001": 18730, "n12125183": 18731, "n12125584": 18732, "n12126084": 18733, "n12126360": 18734, "n12126736": 18735, "n12127460": 18736, "n12127575": 18737, "n12127768": 18738, "n12128071": 18739, "n12128306": 18740, "n12128490": 18741, "n12129134": 18742, "n12129738": 18743, "n12129986": 18744, "n12130549": 18745, "n12131405": 18746, "n12131550": 18747, "n12132092": 18748, "n12132956": 18749, "n12133151": 18750, "n12133462": 18751, "n12133682": 18752, "n12134025": 18753, "n12134486": 18754, "n12134695": 18755, "n12134836": 18756, "n12135049": 18757, "n12135576": 18758, "n12135729": 18759, "n12135898": 18760, "n12136392": 18761, "n12136581": 18762, "n12136720": 18763, "n12137120": 18764, "n12137569": 18765, "n12137791": 18766, "n12137954": 18767, "n12138110": 18768, "n12138248": 18769, "n12138444": 18770, "n12138578": 18771, "n12139196": 18772, "n12139575": 18773, "n12139793": 18774, "n12139921": 18775, "n12140511": 18776, "n12140759": 18777, "n12140903": 18778, "n12141167": 18779, "n12141385": 18780, "n12141495": 18781, "n12142085": 18782, "n12142357": 18783, "n12142450": 18784, "n12143065": 18785, "n12143215": 18786, "n12143405": 18787, "n12143676": 18788, "n12144313": 18789, "n12144580": 18790, "n12144987": 18791, "n12145148": 18792, "n12145477": 18793, "n12146311": 18794, "n12146488": 18795, "n12146654": 18796, "n12147226": 18797, "n12147835": 18798, "n12148757": 18799, "n12150722": 18800, "n12150969": 18801, "n12151170": 18802, "n12151615": 18803, "n12152031": 18804, "n12152251": 18805, "n12152532": 18806, "n12152722": 18807, "n12153033": 18808, "n12153224": 18809, "n12153580": 18810, "n12153741": 18811, "n12153914": 18812, "n12154114": 18813, "n12154773": 18814, "n12155009": 18815, "n12155583": 18816, "n12155773": 18817, "n12156679": 18818, "n12156819": 18819, "n12157056": 18820, "n12157179": 18821, "n12157769": 18822, "n12158031": 18823, "n12158443": 18824, "n12158798": 18825, "n12159055": 18826, "n12159388": 18827, "n12159555": 18828, "n12159804": 18829, "n12159942": 18830, "n12160125": 18831, "n12160303": 18832, "n12160490": 18833, "n12160857": 18834, "n12161056": 18835, "n12161285": 18836, "n12161577": 18837, "n12161744": 18838, "n12161969": 18839, "n12162181": 18840, "n12162425": 18841, "n12162758": 18842, "n12163035": 18843, "n12163279": 18844, "n12164363": 18845, "n12164656": 18846, "n12164881": 18847, "n12165170": 18848, "n12165384": 18849, "n12165758": 18850, "n12166128": 18851, "n12166424": 18852, "n12166793": 18853, "n12166929": 18854, "n12167075": 18855, "n12167436": 18856, "n12167602": 18857, "n12168565": 18858, "n12169099": 18859, "n12170585": 18860, "n12171098": 18861, "n12171316": 18862, "n12171966": 18863, "n12172364": 18864, "n12172481": 18865, "n12172906": 18866, "n12173069": 18867, "n12173664": 18868, "n12173912": 18869, "n12174311": 18870, "n12174521": 18871, "n12174926": 18872, "n12175181": 18873, "n12175370": 18874, "n12175598": 18875, "n12176453": 18876, "n12176709": 18877, "n12176953": 18878, "n12177129": 18879, "n12177455": 18880, "n12178129": 18881, "n12178780": 18882, "n12178896": 18883, "n12179122": 18884, "n12179632": 18885, "n12180168": 18886, "n12180456": 18887, "n12180885": 18888, "n12181352": 18889, "n12181612": 18890, "n12182049": 18891, "n12182276": 18892, "n12183026": 18893, "n12183452": 18894, "n12183816": 18895, "n12184095": 18896, "n12184468": 18897, "n12184912": 18898, "n12185254": 18899, "n12185859": 18900, "n12186352": 18901, "n12186554": 18902, "n12186839": 18903, "n12187247": 18904, "n12187663": 18905, "n12187891": 18906, "n12188289": 18907, "n12188635": 18908, "n12189429": 18909, "n12189779": 18910, "n12189987": 18911, "n12190410": 18912, "n12190869": 18913, "n12191240": 18914, "n12192132": 18915, "n12192877": 18916, "n12193334": 18917, "n12193665": 18918, "n12194147": 18919, "n12194613": 18920, "n12195391": 18921, "n12195533": 18922, "n12195734": 18923, "n12196129": 18924, "n12196336": 18925, "n12196527": 18926, "n12196694": 18927, "n12196954": 18928, "n12197359": 18929, "n12197601": 18930, "n12198286": 18931, "n12198793": 18932, "n12199266": 18933, "n12199399": 18934, "n12199790": 18935, "n12199982": 18936, "n12200143": 18937, "n12200504": 18938, "n12200905": 18939, "n12201331": 18940, "n12201580": 18941, "n12201938": 18942, "n12202936": 18943, "n12203529": 18944, "n12203699": 18945, "n12203896": 18946, "n12204032": 18947, "n12204175": 18948, "n12204730": 18949, "n12205460": 18950, "n12205694": 18951, "n12214789": 18952, "n12215022": 18953, "n12215210": 18954, "n12215579": 18955, "n12215824": 18956, "n12216215": 18957, "n12216628": 18958, "n12216968": 18959, "n12217453": 18960, "n12217851": 18961, "n12218274": 18962, "n12218490": 18963, "n12218868": 18964, "n12219668": 18965, "n12220019": 18966, "n12220496": 18967, "n12220829": 18968, "n12221191": 18969, "n12221368": 18970, "n12221522": 18971, "n12221801": 18972, "n12222090": 18973, "n12222493": 18974, "n12222900": 18975, "n12223160": 18976, "n12223569": 18977, "n12223764": 18978, "n12224978": 18979, "n12225222": 18980, "n12225349": 18981, "n12225563": 18982, "n12226932": 18983, "n12227658": 18984, "n12227909": 18985, "n12228229": 18986, "n12228387": 18987, "n12228689": 18988, "n12228886": 18989, "n12229111": 18990, "n12229651": 18991, "n12229887": 18992, "n12230540": 18993, "n12230794": 18994, "n12231192": 18995, "n12231709": 18996, "n12232114": 18997, "n12232280": 18998, "n12232851": 18999, "n12233249": 19000, "n12234318": 19001, "n12234669": 19002, "n12235051": 19003, "n12235479": 19004, "n12236160": 19005, "n12236546": 19006, "n12236768": 19007, "n12236977": 19008, "n12237152": 19009, "n12237486": 19010, "n12237641": 19011, "n12237855": 19012, "n12238756": 19013, "n12238913": 19014, "n12239240": 19015, "n12239647": 19016, "n12239880": 19017, "n12240150": 19018, "n12240477": 19019, "n12240965": 19020, "n12241192": 19021, "n12241426": 19022, "n12241880": 19023, "n12242123": 19024, "n12242409": 19025, "n12242850": 19026, "n12243109": 19027, "n12243693": 19028, "n12244153": 19029, "n12244458": 19030, "n12244650": 19031, "n12244819": 19032, "n12245319": 19033, "n12245695": 19034, "n12245885": 19035, "n12246037": 19036, "n12246232": 19037, "n12246773": 19038, "n12246941": 19039, "n12247202": 19040, "n12247407": 19041, "n12247963": 19042, "n12248141": 19043, "n12248359": 19044, "n12248574": 19045, "n12248780": 19046, "n12248941": 19047, "n12249122": 19048, "n12249294": 19049, "n12249542": 19050, "n12251001": 19051, "n12251278": 19052, "n12251740": 19053, "n12252168": 19054, "n12252383": 19055, "n12252866": 19056, "n12253229": 19057, "n12253487": 19058, "n12253664": 19059, "n12253835": 19060, "n12254168": 19061, "n12255225": 19062, "n12256112": 19063, "n12256325": 19064, "n12256522": 19065, "n12256708": 19066, "n12256920": 19067, "n12257570": 19068, "n12257725": 19069, "n12258101": 19070, "n12258885": 19071, "n12259316": 19072, "n12260799": 19073, "n12261359": 19074, "n12261571": 19075, "n12261808": 19076, "n12262018": 19077, "n12262185": 19078, "n12262553": 19079, "n12263038": 19080, "n12263204": 19081, "n12263410": 19082, "n12263588": 19083, "n12263738": 19084, "n12263987": 19085, "n12264512": 19086, "n12264786": 19087, "n12265083": 19088, "n12265394": 19089, "n12265600": 19090, "n12266217": 19091, "n12266528": 19092, "n12266644": 19093, "n12266796": 19094, "n12266984": 19095, "n12267133": 19096, "n12267265": 19097, "n12267411": 19098, "n12267534": 19099, "n12267677": 19100, "n12267931": 19101, "n12268246": 19102, "n12269241": 19103, "n12269406": 19104, "n12269652": 19105, "n12270027": 19106, "n12270278": 19107, "n12270460": 19108, "n12270741": 19109, "n12270946": 19110, "n12271187": 19111, "n12271451": 19112, "n12271643": 19113, "n12271933": 19114, "n12272239": 19115, "n12272432": 19116, "n12272735": 19117, "n12272883": 19118, "n12273114": 19119, "n12273344": 19120, "n12273515": 19121, "n12273768": 19122, "n12273939": 19123, "n12274151": 19124, "n12274358": 19125, "n12274630": 19126, "n12274863": 19127, "n12275131": 19128, "n12275317": 19129, "n12275489": 19130, "n12275675": 19131, "n12275888": 19132, "n12276110": 19133, "n12276314": 19134, "n12276477": 19135, "n12276628": 19136, "n12276872": 19137, "n12277150": 19138, "n12277334": 19139, "n12277578": 19140, "n12277800": 19141, "n12278107": 19142, "n12278371": 19143, "n12278650": 19144, "n12278865": 19145, "n12279060": 19146, "n12279293": 19147, "n12279458": 19148, "n12279772": 19149, "n12280060": 19150, "n12280364": 19151, "n12281241": 19152, "n12281788": 19153, "n12281974": 19154, "n12282235": 19155, "n12282527": 19156, "n12282737": 19157, "n12282933": 19158, "n12283147": 19159, "n12283395": 19160, "n12283542": 19161, "n12283790": 19162, "n12284262": 19163, "n12284821": 19164, "n12285049": 19165, "n12285195": 19166, "n12285369": 19167, "n12285512": 19168, "n12285705": 19169, "n12285900": 19170, "n12286068": 19171, "n12286197": 19172, "n12286826": 19173, "n12286988": 19174, "n12287195": 19175, "n12287642": 19176, "n12287836": 19177, "n12288005": 19178, "n12288823": 19179, "n12289310": 19180, "n12289433": 19181, "n12289585": 19182, "n12290748": 19183, "n12290975": 19184, "n12291143": 19185, "n12291459": 19186, "n12291671": 19187, "n12291959": 19188, "n12292463": 19189, "n12292877": 19190, "n12293723": 19191, "n12294124": 19192, "n12294331": 19193, "n12294542": 19194, "n12294723": 19195, "n12294871": 19196, "n12295033": 19197, "n12295237": 19198, "n12295429": 19199, "n12295796": 19200, "n12296045": 19201, "n12296432": 19202, "n12296735": 19203, "n12296929": 19204, "n12297110": 19205, "n12297280": 19206, "n12297507": 19207, "n12297846": 19208, "n12298165": 19209, "n12299640": 19210, "n12300840": 19211, "n12301180": 19212, "n12301445": 19213, "n12301613": 19214, "n12301766": 19215, "n12302071": 19216, "n12302248": 19217, "n12302565": 19218, "n12303083": 19219, "n12303462": 19220, "n12304115": 19221, "n12304286": 19222, "n12304420": 19223, "n12304703": 19224, "n12304899": 19225, "n12305089": 19226, "n12305293": 19227, "n12305475": 19228, "n12305654": 19229, "n12305819": 19230, "n12305986": 19231, "n12306089": 19232, "n12306270": 19233, "n12306717": 19234, "n12306938": 19235, "n12307076": 19236, "n12307240": 19237, "n12307756": 19238, "n12308112": 19239, "n12308447": 19240, "n12308907": 19241, "n12309277": 19242, "n12309630": 19243, "n12310021": 19244, "n12310349": 19245, "n12310638": 19246, "n12311045": 19247, "n12311224": 19248, "n12311413": 19249, "n12311579": 19250, "n12312110": 19251, "n12312728": 19252, "n12315060": 19253, "n12315245": 19254, "n12315598": 19255, "n12315999": 19256, "n12316444": 19257, "n12316572": 19258, "n12317296": 19259, "n12318378": 19260, "n12318782": 19261, "n12318965": 19262, "n12319204": 19263, "n12319414": 19264, "n12320010": 19265, "n12320414": 19266, "n12320627": 19267, "n12320806": 19268, "n12321077": 19269, "n12321395": 19270, "n12321669": 19271, "n12321873": 19272, "n12322099": 19273, "n12322501": 19274, "n12322699": 19275, "n12323665": 19276, "n12324056": 19277, "n12324222": 19278, "n12324388": 19279, "n12324558": 19280, "n12324906": 19281, "n12325234": 19282, "n12325787": 19283, "n12327022": 19284, "n12327528": 19285, "n12327846": 19286, "n12328398": 19287, "n12328567": 19288, "n12328801": 19289, "n12329260": 19290, "n12329473": 19291, "n12330239": 19292, "n12330469": 19293, "n12330587": 19294, "n12330891": 19295, "n12331066": 19296, "n12331263": 19297, "n12331655": 19298, "n12331788": 19299, "n12332030": 19300, "n12332218": 19301, "n12332555": 19302, "n12333053": 19303, "n12333530": 19304, "n12333771": 19305, "n12333961": 19306, "n12334153": 19307, "n12334293": 19308, "n12334891": 19309, "n12335483": 19310, "n12335664": 19311, "n12335800": 19312, "n12335937": 19313, "n12336092": 19314, "n12336224": 19315, "n12336333": 19316, "n12336586": 19317, "n12336727": 19318, "n12336973": 19319, "n12337131": 19320, "n12337246": 19321, "n12337391": 19322, "n12337617": 19323, "n12337800": 19324, "n12337922": 19325, "n12338034": 19326, "n12338146": 19327, "n12338258": 19328, "n12338454": 19329, "n12338655": 19330, "n12338796": 19331, "n12338979": 19332, "n12339526": 19333, "n12339831": 19334, "n12340383": 19335, "n12340581": 19336, "n12340755": 19337, "n12341542": 19338, "n12341931": 19339, "n12342299": 19340, "n12342498": 19341, "n12342852": 19342, "n12343480": 19343, "n12343753": 19344, "n12344283": 19345, "n12344483": 19346, "n12344700": 19347, "n12344837": 19348, "n12345280": 19349, "n12345899": 19350, "n12346578": 19351, "n12346813": 19352, "n12346986": 19353, "n12347158": 19354, "n12349315": 19355, "n12349711": 19356, "n12350032": 19357, "n12350758": 19358, "n12351091": 19359, "n12351790": 19360, "n12352287": 19361, "n12352639": 19362, "n12352844": 19363, "n12352990": 19364, "n12353203": 19365, "n12353431": 19366, "n12353754": 19367, "n12355760": 19368, "n12356023": 19369, "n12356395": 19370, "n12356960": 19371, "n12357485": 19372, "n12357968": 19373, "n12358293": 19374, "n12360108": 19375, "n12360534": 19376, "n12360684": 19377, "n12360817": 19378, "n12360958": 19379, "n12361135": 19380, "n12361560": 19381, "n12361754": 19382, "n12361946": 19383, "n12362274": 19384, "n12362514": 19385, "n12362668": 19386, "n12363301": 19387, "n12363768": 19388, "n12364604": 19389, "n12364940": 19390, "n12365158": 19391, "n12365285": 19392, "n12365462": 19393, "n12365900": 19394, "n12366053": 19395, "n12366186": 19396, "n12366313": 19397, "n12366675": 19398, "n12366870": 19399, "n12367611": 19400, "n12368028": 19401, "n12368257": 19402, "n12368451": 19403, "n12369066": 19404, "n12369309": 19405, "n12369476": 19406, "n12369665": 19407, "n12369845": 19408, "n12370174": 19409, "n12370549": 19410, "n12371202": 19411, "n12371439": 19412, "n12371704": 19413, "n12372233": 19414, "n12373100": 19415, "n12373739": 19416, "n12374418": 19417, "n12374705": 19418, "n12374862": 19419, "n12375769": 19420, "n12377198": 19421, "n12377494": 19422, "n12378249": 19423, "n12378753": 19424, "n12378963": 19425, "n12379531": 19426, "n12380761": 19427, "n12381511": 19428, "n12382233": 19429, "n12382875": 19430, "n12383737": 19431, "n12383894": 19432, "n12384037": 19433, "n12384227": 19434, "n12384375": 19435, "n12384569": 19436, "n12384680": 19437, "n12384839": 19438, "n12385429": 19439, "n12385566": 19440, "n12385830": 19441, "n12386945": 19442, "n12387103": 19443, "n12387633": 19444, "n12387839": 19445, "n12388143": 19446, "n12388293": 19447, "n12388858": 19448, "n12388989": 19449, "n12389130": 19450, "n12389501": 19451, "n12389727": 19452, "n12389932": 19453, "n12390099": 19454, "n12390314": 19455, "n12392070": 19456, "n12392549": 19457, "n12392765": 19458, "n12393269": 19459, "n12394118": 19460, "n12394328": 19461, "n12394638": 19462, "n12395068": 19463, "n12395289": 19464, "n12395463": 19465, "n12395906": 19466, "n12396091": 19467, "n12396924": 19468, "n12397431": 19469, "n12399132": 19470, "n12399384": 19471, "n12399534": 19472, "n12399656": 19473, "n12399899": 19474, "n12400489": 19475, "n12400720": 19476, "n12400924": 19477, "n12401335": 19478, "n12401684": 19479, "n12401893": 19480, "n12402051": 19481, "n12402348": 19482, "n12402596": 19483, "n12402840": 19484, "n12403075": 19485, "n12403276": 19486, "n12403513": 19487, "n12403994": 19488, "n12404729": 19489, "n12405714": 19490, "n12406304": 19491, "n12406488": 19492, "n12406715": 19493, "n12406902": 19494, "n12407079": 19495, "n12407222": 19496, "n12407396": 19497, "n12407545": 19498, "n12407715": 19499, "n12407890": 19500, "n12408077": 19501, "n12408280": 19502, "n12408466": 19503, "n12408717": 19504, "n12408873": 19505, "n12409231": 19506, "n12409470": 19507, "n12409651": 19508, "n12409840": 19509, "n12411461": 19510, "n12412355": 19511, "n12412606": 19512, "n12412987": 19513, "n12413165": 19514, "n12413301": 19515, "n12413419": 19516, "n12413642": 19517, "n12413880": 19518, "n12414035": 19519, "n12414159": 19520, "n12414329": 19521, "n12414449": 19522, "n12414818": 19523, "n12414932": 19524, "n12415595": 19525, "n12416073": 19526, "n12416423": 19527, "n12416703": 19528, "n12417836": 19529, "n12418221": 19530, "n12418507": 19531, "n12419037": 19532, "n12419878": 19533, "n12420124": 19534, "n12420535": 19535, "n12420722": 19536, "n12421137": 19537, "n12421467": 19538, "n12421683": 19539, "n12421917": 19540, "n12422129": 19541, "n12422559": 19542, "n12425281": 19543, "n12426623": 19544, "n12426749": 19545, "n12427184": 19546, "n12427391": 19547, "n12427566": 19548, "n12427757": 19549, "n12427946": 19550, "n12428076": 19551, "n12428242": 19552, "n12428412": 19553, "n12428747": 19554, "n12429352": 19555, "n12430198": 19556, "n12430471": 19557, "n12430675": 19558, "n12431434": 19559, "n12432069": 19560, "n12432356": 19561, "n12432574": 19562, "n12432707": 19563, "n12433081": 19564, "n12433178": 19565, "n12433769": 19566, "n12433952": 19567, "n12434106": 19568, "n12434483": 19569, "n12434634": 19570, "n12434775": 19571, "n12434985": 19572, "n12435152": 19573, "n12435486": 19574, "n12435649": 19575, "n12435777": 19576, "n12435965": 19577, "n12436090": 19578, "n12436907": 19579, "n12437513": 19580, "n12437769": 19581, "n12437930": 19582, "n12439154": 19583, "n12439830": 19584, "n12441183": 19585, "n12441390": 19586, "n12441552": 19587, "n12441958": 19588, "n12442548": 19589, "n12443323": 19590, "n12443736": 19591, "n12444095": 19592, "n12444898": 19593, "n12446200": 19594, "n12446519": 19595, "n12446737": 19596, "n12446908": 19597, "n12447121": 19598, "n12447346": 19599, "n12447581": 19600, "n12447891": 19601, "n12448136": 19602, "n12448361": 19603, "n12448700": 19604, "n12449296": 19605, "n12449526": 19606, "n12449784": 19607, "n12449934": 19608, "n12450344": 19609, "n12450607": 19610, "n12450840": 19611, "n12451070": 19612, "n12451240": 19613, "n12451399": 19614, "n12451566": 19615, "n12451915": 19616, "n12452256": 19617, "n12452480": 19618, "n12452673": 19619, "n12452836": 19620, "n12453018": 19621, "n12453186": 19622, "n12453714": 19623, "n12453857": 19624, "n12454159": 19625, "n12454436": 19626, "n12454556": 19627, "n12454705": 19628, "n12454793": 19629, "n12454949": 19630, "n12455950": 19631, "n12457091": 19632, "n12458550": 19633, "n12458713": 19634, "n12458874": 19635, "n12459629": 19636, "n12460146": 19637, "n12460697": 19638, "n12460957": 19639, "n12461109": 19640, "n12461466": 19641, "n12461673": 19642, "n12462032": 19643, "n12462221": 19644, "n12462582": 19645, "n12462805": 19646, "n12463134": 19647, "n12463743": 19648, "n12463975": 19649, "n12464128": 19650, "n12464476": 19651, "n12464649": 19652, "n12465557": 19653, "n12466727": 19654, "n12467018": 19655, "n12467197": 19656, "n12467433": 19657, "n12467592": 19658, "n12468545": 19659, "n12468719": 19660, "n12469517": 19661, "n12470092": 19662, "n12470512": 19663, "n12470907": 19664, "n12472024": 19665, "n12473608": 19666, "n12473840": 19667, "n12474167": 19668, "n12474418": 19669, "n12475035": 19670, "n12475242": 19671, "n12475774": 19672, "n12476510": 19673, "n12477163": 19674, "n12477401": 19675, "n12477583": 19676, "n12477747": 19677, "n12477983": 19678, "n12478768": 19679, "n12479537": 19680, "n12480456": 19681, "n12480895": 19682, "n12481150": 19683, "n12481289": 19684, "n12481458": 19685, "n12482437": 19686, "n12482668": 19687, "n12482893": 19688, "n12483282": 19689, "n12483427": 19690, "n12483625": 19691, "n12483841": 19692, "n12484244": 19693, "n12484784": 19694, "n12485653": 19695, "n12485981": 19696, "n12486574": 19697, "n12487058": 19698, "n12488454": 19699, "n12488709": 19700, "n12489046": 19701, "n12489676": 19702, "n12489815": 19703, "n12490490": 19704, "n12491017": 19705, "n12491435": 19706, "n12491826": 19707, "n12492106": 19708, "n12492460": 19709, "n12492682": 19710, "n12492900": 19711, "n12493208": 19712, "n12493426": 19713, "n12493868": 19714, "n12494794": 19715, "n12495146": 19716, "n12495670": 19717, "n12495895": 19718, "n12496427": 19719, "n12496949": 19720, "n12497669": 19721, "n12498055": 19722, "n12498457": 19723, "n12499163": 19724, "n12499757": 19725, "n12499979": 19726, "n12500309": 19727, "n12500518": 19728, "n12500751": 19729, "n12501202": 19730, "n12504570": 19731, "n12504783": 19732, "n12505253": 19733, "n12506181": 19734, "n12506341": 19735, "n12506991": 19736, "n12507379": 19737, "n12507823": 19738, "n12508309": 19739, "n12508618": 19740, "n12508762": 19741, "n12509109": 19742, "n12509476": 19743, "n12509665": 19744, "n12509821": 19745, "n12509993": 19746, "n12510343": 19747, "n12510774": 19748, "n12511488": 19749, "n12511856": 19750, "n12512095": 19751, "n12512294": 19752, "n12512674": 19753, "n12513172": 19754, "n12513613": 19755, "n12513933": 19756, "n12514138": 19757, "n12514592": 19758, "n12514992": 19759, "n12515393": 19760, "n12515711": 19761, "n12515925": 19762, "n12516165": 19763, "n12516584": 19764, "n12516828": 19765, "n12517077": 19766, "n12517445": 19767, "n12517642": 19768, "n12518013": 19769, "n12518481": 19770, "n12519089": 19771, "n12519563": 19772, "n12520406": 19773, "n12521186": 19774, "n12521394": 19775, "n12522188": 19776, "n12522678": 19777, "n12522894": 19778, "n12523141": 19779, "n12523475": 19780, "n12523850": 19781, "n12524188": 19782, "n12525168": 19783, "n12525513": 19784, "n12525753": 19785, "n12526178": 19786, "n12526516": 19787, "n12526754": 19788, "n12527081": 19789, "n12527738": 19790, "n12528109": 19791, "n12528382": 19792, "n12528549": 19793, "n12528768": 19794, "n12528974": 19795, "n12529220": 19796, "n12529500": 19797, "n12529905": 19798, "n12530629": 19799, "n12530818": 19800, "n12531328": 19801, "n12531727": 19802, "n12532564": 19803, "n12532886": 19804, "n12533190": 19805, "n12533437": 19806, "n12534208": 19807, "n12534625": 19808, "n12534862": 19809, "n12536291": 19810, "n12537253": 19811, "n12537569": 19812, "n12538209": 19813, "n12539074": 19814, "n12539306": 19815, "n12539832": 19816, "n12540250": 19817, "n12540647": 19818, "n12540966": 19819, "n12541157": 19820, "n12541403": 19821, "n12542043": 19822, "n12542240": 19823, "n12543186": 19824, "n12543455": 19825, "n12543639": 19826, "n12543826": 19827, "n12544240": 19828, "n12544539": 19829, "n12545232": 19830, "n12545635": 19831, "n12545865": 19832, "n12546183": 19833, "n12546420": 19834, "n12546617": 19835, "n12546962": 19836, "n12547215": 19837, "n12547503": 19838, "n12548280": 19839, "n12548564": 19840, "n12548804": 19841, "n12549005": 19842, "n12549192": 19843, "n12549420": 19844, "n12549799": 19845, "n12550210": 19846, "n12550408": 19847, "n12551173": 19848, "n12551457": 19849, "n12552309": 19850, "n12552893": 19851, "n12553742": 19852, "n12554029": 19853, "n12554526": 19854, "n12554729": 19855, "n12554911": 19856, "n12555255": 19857, "n12555859": 19858, "n12556656": 19859, "n12557064": 19860, "n12557438": 19861, "n12557556": 19862, "n12557681": 19863, "n12558230": 19864, "n12558425": 19865, "n12558680": 19866, "n12559044": 19867, "n12559518": 19868, "n12560282": 19869, "n12560621": 19870, "n12560775": 19871, "n12561169": 19872, "n12561309": 19873, "n12561594": 19874, "n12562141": 19875, "n12562577": 19876, "n12562785": 19877, "n12563045": 19878, "n12563702": 19879, "n12564083": 19880, "n12564613": 19881, "n12565102": 19882, "n12565912": 19883, "n12566331": 19884, "n12566954": 19885, "n12567950": 19886, "n12568186": 19887, "n12568649": 19888, "n12569037": 19889, "n12569616": 19890, "n12569851": 19891, "n12570394": 19892, "n12570703": 19893, "n12570972": 19894, "n12571781": 19895, "n12572546": 19896, "n12572759": 19897, "n12572858": 19898, "n12573256": 19899, "n12573474": 19900, "n12573647": 19901, "n12573911": 19902, "n12574320": 19903, "n12574470": 19904, "n12574866": 19905, "n12575322": 19906, "n12575812": 19907, "n12576323": 19908, "n12576451": 19909, "n12576695": 19910, "n12577362": 19911, "n12577895": 19912, "n12578255": 19913, "n12578626": 19914, "n12578916": 19915, "n12579038": 19916, "n12579404": 19917, "n12579822": 19918, "n12580012": 19919, "n12580654": 19920, "n12580786": 19921, "n12580896": 19922, "n12581110": 19923, "n12582231": 19924, "n12582665": 19925, "n12582846": 19926, "n12583126": 19927, "n12583401": 19928, "n12583681": 19929, "n12583855": 19930, "n12584191": 19931, "n12584365": 19932, "n12584715": 19933, "n12585137": 19934, "n12585373": 19935, "n12585629": 19936, "n12586298": 19937, "n12586499": 19938, "n12586725": 19939, "n12586989": 19940, "n12587132": 19941, "n12587487": 19942, "n12587803": 19943, "n12588320": 19944, "n12588780": 19945, "n12589142": 19946, "n12589458": 19947, "n12589687": 19948, "n12589841": 19949, "n12590232": 19950, "n12590499": 19951, "n12590600": 19952, "n12590715": 19953, "n12591017": 19954, "n12591351": 19955, "n12591702": 19956, "n12592058": 19957, "n12592544": 19958, "n12592839": 19959, "n12593122": 19960, "n12593341": 19961, "n12593994": 19962, "n12594324": 19963, "n12594989": 19964, "n12595699": 19965, "n12595964": 19966, "n12596148": 19967, "n12596345": 19968, "n12596709": 19969, "n12596849": 19970, "n12597134": 19971, "n12597466": 19972, "n12597798": 19973, "n12598027": 19974, "n12599185": 19975, "n12599435": 19976, "n12599661": 19977, "n12599874": 19978, "n12600095": 19979, "n12600267": 19980, "n12601494": 19981, "n12601805": 19982, "n12602262": 19983, "n12602434": 19984, "n12602612": 19985, "n12602980": 19986, "n12603273": 19987, "n12603449": 19988, "n12603672": 19989, "n12604228": 19990, "n12604460": 19991, "n12604639": 19992, "n12604845": 19993, "n12605683": 19994, "n12606438": 19995, "n12606545": 19996, "n12607456": 19997, "n12609379": 19998, "n12610328": 19999, "n12610740": 20000, "n12611640": 20001, "n12612170": 20002, "n12612811": 20003, "n12613706": 20004, "n12614096": 20005, "n12614477": 20006, "n12614625": 20007, "n12615232": 20008, "n12615710": 20009, "n12616248": 20010, "n12616630": 20011, "n12616996": 20012, "n12617559": 20013, "n12618146": 20014, "n12618727": 20015, "n12620196": 20016, "n12620546": 20017, "n12620969": 20018, "n12621410": 20019, "n12621619": 20020, "n12621945": 20021, "n12622297": 20022, "n12622875": 20023, "n12623077": 20024, "n12623211": 20025, "n12623818": 20026, "n12624381": 20027, "n12624568": 20028, "n12625003": 20029, "n12625383": 20030, "n12625670": 20031, "n12625823": 20032, "n12626674": 20033, "n12626878": 20034, "n12627119": 20035, "n12627347": 20036, "n12627526": 20037, "n12628356": 20038, "n12628705": 20039, "n12628986": 20040, "n12629305": 20041, "n12629666": 20042, "n12630763": 20043, "n12630999": 20044, "n12631331": 20045, "n12631637": 20046, "n12631932": 20047, "n12632335": 20048, "n12632733": 20049, "n12633061": 20050, "n12633638": 20051, "n12633994": 20052, "n12634211": 20053, "n12634429": 20054, "n12634734": 20055, "n12634986": 20056, "n12635151": 20057, "n12635359": 20058, "n12635532": 20059, "n12635744": 20060, "n12635955": 20061, "n12636224": 20062, "n12636885": 20063, "n12637123": 20064, "n12637485": 20065, "n12638218": 20066, "n12638556": 20067, "n12638753": 20068, "n12638964": 20069, "n12639168": 20070, "n12639376": 20071, "n12639584": 20072, "n12639736": 20073, "n12639910": 20074, "n12640081": 20075, "n12640284": 20076, "n12640435": 20077, "n12640607": 20078, "n12640839": 20079, "n12641007": 20080, "n12641180": 20081, "n12641413": 20082, "n12641931": 20083, "n12642090": 20084, "n12642200": 20085, "n12642435": 20086, "n12642600": 20087, "n12642964": 20088, "n12643113": 20089, "n12643313": 20090, "n12643473": 20091, "n12643688": 20092, "n12643877": 20093, "n12644283": 20094, "n12644902": 20095, "n12645174": 20096, "n12645530": 20097, "n12646072": 20098, "n12646197": 20099, "n12646397": 20100, "n12646605": 20101, "n12646740": 20102, "n12646950": 20103, "n12647231": 20104, "n12647376": 20105, "n12647560": 20106, "n12647787": 20107, "n12647893": 20108, "n12648045": 20109, "n12648196": 20110, "n12648424": 20111, "n12648693": 20112, "n12648888": 20113, "n12649065": 20114, "n12649317": 20115, "n12649539": 20116, "n12649866": 20117, "n12650038": 20118, "n12650229": 20119, "n12650379": 20120, "n12650556": 20121, "n12650805": 20122, "n12650915": 20123, "n12651229": 20124, "n12651611": 20125, "n12651821": 20126, "n12653218": 20127, "n12653436": 20128, "n12653633": 20129, "n12654227": 20130, "n12654857": 20131, "n12655062": 20132, "n12655245": 20133, "n12655351": 20134, "n12655498": 20135, "n12655605": 20136, "n12655726": 20137, "n12655869": 20138, "n12656369": 20139, "n12656528": 20140, "n12656685": 20141, "n12656909": 20142, "n12657082": 20143, "n12657755": 20144, "n12658118": 20145, "n12658308": 20146, "n12658481": 20147, "n12658603": 20148, "n12658715": 20149, "n12658846": 20150, "n12659064": 20151, "n12659356": 20152, "n12659539": 20153, "n12660601": 20154, "n12661045": 20155, "n12661227": 20156, "n12661538": 20157, "n12662074": 20158, "n12662379": 20159, "n12662772": 20160, "n12663023": 20161, "n12663254": 20162, "n12663359": 20163, "n12663804": 20164, "n12664005": 20165, "n12664187": 20166, "n12664469": 20167, "n12664710": 20168, "n12665048": 20169, "n12665271": 20170, "n12665659": 20171, "n12665857": 20172, "n12666050": 20173, "n12666159": 20174, "n12666369": 20175, "n12666965": 20176, "n12667406": 20177, "n12667582": 20178, "n12667964": 20179, "n12668131": 20180, "n12669803": 20181, "n12670334": 20182, "n12670758": 20183, "n12670962": 20184, "n12671651": 20185, "n12672289": 20186, "n12673588": 20187, "n12674120": 20188, "n12674685": 20189, "n12674895": 20190, "n12675299": 20191, "n12675515": 20192, "n12675876": 20193, "n12676134": 20194, "n12676370": 20195, "n12676534": 20196, "n12676703": 20197, "n12677120": 20198, "n12677331": 20199, "n12677612": 20200, "n12677841": 20201, "n12678794": 20202, "n12679023": 20203, "n12679432": 20204, "n12679593": 20205, "n12679876": 20206, "n12680402": 20207, "n12680652": 20208, "n12680864": 20209, "n12681376": 20210, "n12681579": 20211, "n12681893": 20212, "n12682411": 20213, "n12682668": 20214, "n12682882": 20215, "n12683096": 20216, "n12683407": 20217, "n12683571": 20218, "n12683791": 20219, "n12684379": 20220, "n12685431": 20221, "n12685831": 20222, "n12686077": 20223, "n12686274": 20224, "n12686496": 20225, "n12686676": 20226, "n12686877": 20227, "n12687044": 20228, "n12687462": 20229, "n12687698": 20230, "n12687957": 20231, "n12688187": 20232, "n12688372": 20233, "n12688716": 20234, "n12689305": 20235, "n12690653": 20236, "n12691428": 20237, "n12691661": 20238, "n12692024": 20239, "n12692160": 20240, "n12692521": 20241, "n12692714": 20242, "n12693244": 20243, "n12693352": 20244, "n12693865": 20245, "n12694486": 20246, "n12695144": 20247, "n12695975": 20248, "n12696492": 20249, "n12696830": 20250, "n12697152": 20251, "n12697514": 20252, "n12698027": 20253, "n12698435": 20254, "n12698598": 20255, "n12698774": 20256, "n12699031": 20257, "n12699301": 20258, "n12699922": 20259, "n12700088": 20260, "n12700357": 20261, "n12702124": 20262, "n12703190": 20263, "n12703383": 20264, "n12703557": 20265, "n12703716": 20266, "n12703856": 20267, "n12704041": 20268, "n12704343": 20269, "n12704513": 20270, "n12705013": 20271, "n12705220": 20272, "n12705458": 20273, "n12705698": 20274, "n12705978": 20275, "n12706410": 20276, "n12707199": 20277, "n12707781": 20278, "n12708293": 20279, "n12708654": 20280, "n12708941": 20281, "n12709103": 20282, "n12709349": 20283, "n12709688": 20284, "n12709901": 20285, "n12710295": 20286, "n12710415": 20287, "n12710577": 20288, "n12710693": 20289, "n12710917": 20290, "n12711182": 20291, "n12711398": 20292, "n12711596": 20293, "n12711817": 20294, "n12711984": 20295, "n12712320": 20296, "n12712626": 20297, "n12713063": 20298, "n12713358": 20299, "n12713521": 20300, "n12713866": 20301, "n12714254": 20302, "n12714755": 20303, "n12714949": 20304, "n12715195": 20305, "n12715914": 20306, "n12716400": 20307, "n12716594": 20308, "n12717072": 20309, "n12717224": 20310, "n12717644": 20311, "n12718074": 20312, "n12718483": 20313, "n12718995": 20314, "n12719684": 20315, "n12719944": 20316, "n12720200": 20317, "n12720354": 20318, "n12721122": 20319, "n12721477": 20320, "n12722071": 20321, "n12723062": 20322, "n12723610": 20323, "n12724942": 20324, "n12725521": 20325, "n12725738": 20326, "n12725940": 20327, "n12726159": 20328, "n12726357": 20329, "n12726528": 20330, "n12726670": 20331, "n12726902": 20332, "n12727101": 20333, "n12727301": 20334, "n12727518": 20335, "n12727729": 20336, "n12727960": 20337, "n12728164": 20338, "n12728322": 20339, "n12728508": 20340, "n12728656": 20341, "n12728864": 20342, "n12729023": 20343, "n12729164": 20344, "n12729315": 20345, "n12729521": 20346, "n12729729": 20347, "n12729950": 20348, "n12730143": 20349, "n12730370": 20350, "n12730544": 20351, "n12730776": 20352, "n12731029": 20353, "n12731401": 20354, "n12731835": 20355, "n12732009": 20356, "n12732252": 20357, "n12732491": 20358, "n12732605": 20359, "n12732756": 20360, "n12732966": 20361, "n12733218": 20362, "n12733428": 20363, "n12733647": 20364, "n12733870": 20365, "n12734070": 20366, "n12734215": 20367, "n12735160": 20368, "n12736603": 20369, "n12736999": 20370, "n12737383": 20371, "n12737898": 20372, "n12738259": 20373, "n12739332": 20374, "n12739966": 20375, "n12740967": 20376, "n12741222": 20377, "n12741586": 20378, "n12741792": 20379, "n12742290": 20380, "n12742741": 20381, "n12742878": 20382, "n12743009": 20383, "n12743352": 20384, "n12743823": 20385, "n12743976": 20386, "n12744142": 20387, "n12744387": 20388, "n12744850": 20389, "n12745386": 20390, "n12745564": 20391, "n12746884": 20392, "n12747120": 20393, "n12748248": 20394, "n12749049": 20395, "n12749456": 20396, "n12749679": 20397, "n12749852": 20398, "n12750076": 20399, "n12750767": 20400, "n12751172": 20401, "n12751675": 20402, "n12752205": 20403, "n12753007": 20404, "n12753245": 20405, "n12753573": 20406, "n12753762": 20407, "n12754003": 20408, "n12754174": 20409, "n12754311": 20410, "n12754468": 20411, "n12754648": 20412, "n12754781": 20413, "n12754981": 20414, "n12755225": 20415, "n12755387": 20416, "n12755559": 20417, "n12755727": 20418, "n12755876": 20419, "n12756457": 20420, "n12757115": 20421, "n12757303": 20422, "n12757458": 20423, "n12757668": 20424, "n12757816": 20425, "n12757930": 20426, "n12758014": 20427, "n12758099": 20428, "n12758176": 20429, "n12758250": 20430, "n12758325": 20431, "n12758399": 20432, "n12758471": 20433, "n12758555": 20434, "n12759273": 20435, "n12759668": 20436, "n12760539": 20437, "n12760875": 20438, "n12761284": 20439, "n12761702": 20440, "n12761905": 20441, "n12762049": 20442, "n12762405": 20443, "n12762896": 20444, "n12763529": 20445, "n12764008": 20446, "n12764202": 20447, "n12764507": 20448, "n12764978": 20449, "n12765115": 20450, "n12765402": 20451, "n12765846": 20452, "n12766043": 20453, "n12766595": 20454, "n12766869": 20455, "n12767208": 20456, "n12767423": 20457, "n12767648": 20458, "n12768369": 20459, "n12768682": 20460, "n12768809": 20461, "n12768933": 20462, "n12769065": 20463, "n12769219": 20464, "n12769318": 20465, "n12770529": 20466, "n12770892": 20467, "n12771085": 20468, "n12771192": 20469, "n12771390": 20470, "n12771597": 20471, "n12771890": 20472, "n12772753": 20473, "n12772908": 20474, "n12773142": 20475, "n12773651": 20476, "n12773917": 20477, "n12774299": 20478, "n12774641": 20479, "n12775070": 20480, "n12775393": 20481, "n12775717": 20482, "n12775919": 20483, "n12776558": 20484, "n12776774": 20485, "n12777436": 20486, "n12777680": 20487, "n12777778": 20488, "n12777892": 20489, "n12778398": 20490, "n12778605": 20491, "n12779603": 20492, "n12779851": 20493, "n12780325": 20494, "n12780563": 20495, "n12781940": 20496, "n12782530": 20497, "n12782915": 20498, "n12783316": 20499, "n12783730": 20500, "n12784371": 20501, "n12784889": 20502, "n12785724": 20503, "n12785889": 20504, "n12786273": 20505, "n12786464": 20506, "n12786836": 20507, "n12787364": 20508, "n12788854": 20509, "n12789054": 20510, "n12789554": 20511, "n12789977": 20512, "n12790430": 20513, "n12791064": 20514, "n12791329": 20515, "n12793015": 20516, "n12793284": 20517, "n12793494": 20518, "n12793695": 20519, "n12793886": 20520, "n12794135": 20521, "n12794367": 20522, "n12794568": 20523, "n12794985": 20524, "n12795209": 20525, "n12795352": 20526, "n12795555": 20527, "n12796022": 20528, "n12796385": 20529, "n12796849": 20530, "n12797368": 20531, "n12797860": 20532, "n12798284": 20533, "n12798910": 20534, "n12799269": 20535, "n12799776": 20536, "n12800049": 20537, "n12800586": 20538, "n12801072": 20539, "n12801520": 20540, "n12801781": 20541, "n12801966": 20542, "n12803226": 20543, "n12803754": 20544, "n12803958": 20545, "n12804352": 20546, "n12805146": 20547, "n12805561": 20548, "n12805762": 20549, "n12806015": 20550, "n12806732": 20551, "n12807251": 20552, "n12807409": 20553, "n12807624": 20554, "n12807773": 20555, "n12808007": 20556, "n12809868": 20557, "n12810007": 20558, "n12810151": 20559, "n12810595": 20560, "n12811027": 20561, "n12811713": 20562, "n12812235": 20563, "n12812478": 20564, "n12812801": 20565, "n12813189": 20566, "n12814643": 20567, "n12814857": 20568, "n12814960": 20569, "n12815198": 20570, "n12815668": 20571, "n12815838": 20572, "n12816508": 20573, "n12816942": 20574, "n12817464": 20575, "n12817694": 20576, "n12817855": 20577, "n12818004": 20578, "n12818346": 20579, "n12818601": 20580, "n12818966": 20581, "n12819141": 20582, "n12819354": 20583, "n12819728": 20584, "n12820113": 20585, "n12820669": 20586, "n12820853": 20587, "n12821505": 20588, "n12821895": 20589, "n12822115": 20590, "n12822466": 20591, "n12822769": 20592, "n12822955": 20593, "n12823717": 20594, "n12823859": 20595, "n12824053": 20596, "n12824289": 20597, "n12824735": 20598, "n12825497": 20599, "n12826143": 20600, "n12827270": 20601, "n12827537": 20602, "n12827907": 20603, "n12828220": 20604, "n12828379": 20605, "n12828520": 20606, "n12828791": 20607, "n12828977": 20608, "n12829582": 20609, "n12829975": 20610, "n12830222": 20611, "n12830568": 20612, "n12831141": 20613, "n12831535": 20614, "n12831932": 20615, "n12832315": 20616, "n12832538": 20617, "n12832822": 20618, "n12833149": 20619, "n12833985": 20620, "n12834190": 20621, "n12834798": 20622, "n12834938": 20623, "n12835331": 20624, "n12835766": 20625, "n12836212": 20626, "n12836337": 20627, "n12836508": 20628, "n12836862": 20629, "n12837052": 20630, "n12837259": 20631, "n12837466": 20632, "n12837803": 20633, "n12839574": 20634, "n12839979": 20635, "n12840168": 20636, "n12840362": 20637, "n12840502": 20638, "n12840749": 20639, "n12841007": 20640, "n12841193": 20641, "n12841354": 20642, "n12842302": 20643, "n12842519": 20644, "n12842642": 20645, "n12842887": 20646, "n12843144": 20647, "n12843316": 20648, "n12843557": 20649, "n12843970": 20650, "n12844409": 20651, "n12844939": 20652, "n12845187": 20653, "n12845413": 20654, "n12845908": 20655, "n12846335": 20656, "n12846690": 20657, "n12847008": 20658, "n12847374": 20659, "n12847927": 20660, "n12848499": 20661, "n12849061": 20662, "n12849279": 20663, "n12849416": 20664, "n12849952": 20665, "n12850168": 20666, "n12850336": 20667, "n12850906": 20668, "n12851094": 20669, "n12851469": 20670, "n12851860": 20671, "n12852234": 20672, "n12852428": 20673, "n12852570": 20674, "n12853080": 20675, "n12853287": 20676, "n12853482": 20677, "n12854048": 20678, "n12854193": 20679, "n12854600": 20680, "n12855365": 20681, "n12855494": 20682, "n12855710": 20683, "n12855886": 20684, "n12856091": 20685, "n12856287": 20686, "n12856479": 20687, "n12856680": 20688, "n12857204": 20689, "n12857779": 20690, "n12858150": 20691, "n12858397": 20692, "n12858618": 20693, "n12858871": 20694, "n12858987": 20695, "n12859153": 20696, "n12859272": 20697, "n12859679": 20698, "n12859986": 20699, "n12860365": 20700, "n12860978": 20701, "n12861345": 20702, "n12861541": 20703, "n12861892": 20704, "n12862512": 20705, "n12862828": 20706, "n12863234": 20707, "n12863624": 20708, "n12864160": 20709, "n12865037": 20710, "n12865562": 20711, "n12865708": 20712, "n12865824": 20713, "n12866002": 20714, "n12866162": 20715, "n12866333": 20716, "n12866459": 20717, "n12866635": 20718, "n12866968": 20719, "n12867184": 20720, "n12867449": 20721, "n12867826": 20722, "n12868019": 20723, "n12868880": 20724, "n12869061": 20725, "n12869478": 20726, "n12869668": 20727, "n12870048": 20728, "n12870225": 20729, "n12870535": 20730, "n12870682": 20731, "n12870891": 20732, "n12871272": 20733, "n12871696": 20734, "n12871859": 20735, "n12872458": 20736, "n12872914": 20737, "n12873341": 20738, "n12873984": 20739, "n12875269": 20740, "n12875697": 20741, "n12875861": 20742, "n12876899": 20743, "n12877244": 20744, "n12877493": 20745, "n12877637": 20746, "n12877838": 20747, "n12878169": 20748, "n12878325": 20749, "n12878784": 20750, "n12879068": 20751, "n12879527": 20752, "n12879963": 20753, "n12880244": 20754, "n12880462": 20755, "n12880638": 20756, "n12880799": 20757, "n12881105": 20758, "n12881913": 20759, "n12882158": 20760, "n12882779": 20761, "n12882945": 20762, "n12883265": 20763, "n12883628": 20764, "n12884100": 20765, "n12884260": 20766, "n12885045": 20767, "n12885265": 20768, "n12885510": 20769, "n12885754": 20770, "n12886185": 20771, "n12886402": 20772, "n12886600": 20773, "n12886831": 20774, "n12887293": 20775, "n12887532": 20776, "n12887713": 20777, "n12888016": 20778, "n12888234": 20779, "n12888457": 20780, "n12889219": 20781, "n12889412": 20782, "n12889579": 20783, "n12889713": 20784, "n12890265": 20785, "n12890490": 20786, "n12890685": 20787, "n12890928": 20788, "n12891093": 20789, "n12891305": 20790, "n12891469": 20791, "n12891643": 20792, "n12891824": 20793, "n12892013": 20794, "n12893463": 20795, "n12893993": 20796, "n12895298": 20797, "n12895811": 20798, "n12896615": 20799, "n12897118": 20800, "n12897788": 20801, "n12897999": 20802, "n12898342": 20803, "n12898774": 20804, "n12899166": 20805, "n12899537": 20806, "n12899752": 20807, "n12899971": 20808, "n12900783": 20809, "n12901724": 20810, "n12902466": 20811, "n12902662": 20812, "n12903014": 20813, "n12903367": 20814, "n12903503": 20815, "n12903964": 20816, "n12904314": 20817, "n12904562": 20818, "n12904938": 20819, "n12905135": 20820, "n12905412": 20821, "n12906214": 20822, "n12906498": 20823, "n12906771": 20824, "n12907057": 20825, "n12907671": 20826, "n12907857": 20827, "n12908093": 20828, "n12908645": 20829, "n12908854": 20830, "n12909421": 20831, "n12909614": 20832, "n12909759": 20833, "n12909917": 20834, "n12911079": 20835, "n12911264": 20836, "n12911440": 20837, "n12911673": 20838, "n12911914": 20839, "n12912274": 20840, "n12912670": 20841, "n12912801": 20842, "n12913144": 20843, "n12913524": 20844, "n12913791": 20845, "n12914923": 20846, "n12915140": 20847, "n12915568": 20848, "n12915811": 20849, "n12916179": 20850, "n12916511": 20851, "n12917901": 20852, "n12918609": 20853, "n12918810": 20854, "n12918991": 20855, "n12919195": 20856, "n12919403": 20857, "n12919646": 20858, "n12919847": 20859, "n12920043": 20860, "n12920204": 20861, "n12920521": 20862, "n12920719": 20863, "n12920955": 20864, "n12921315": 20865, "n12921499": 20866, "n12921660": 20867, "n12921868": 20868, "n12922119": 20869, "n12922458": 20870, "n12922763": 20871, "n12923108": 20872, "n12923257": 20873, "n12924623": 20874, "n12925179": 20875, "n12925583": 20876, "n12926039": 20877, "n12926480": 20878, "n12926689": 20879, "n12927013": 20880, "n12927194": 20881, "n12927494": 20882, "n12927758": 20883, "n12928071": 20884, "n12928307": 20885, "n12928491": 20886, "n12928819": 20887, "n12929403": 20888, "n12929600": 20889, "n12930778": 20890, "n12930951": 20891, "n12931231": 20892, "n12931542": 20893, "n12931906": 20894, "n12932173": 20895, "n12932365": 20896, "n12932706": 20897, "n12932966": 20898, "n12933274": 20899, "n12934036": 20900, "n12934174": 20901, "n12934479": 20902, "n12934685": 20903, "n12934985": 20904, "n12935166": 20905, "n12935609": 20906, "n12936155": 20907, "n12936826": 20908, "n12937130": 20909, "n12938081": 20910, "n12938193": 20911, "n12938445": 20912, "n12938667": 20913, "n12939104": 20914, "n12939282": 20915, "n12939479": 20916, "n12939874": 20917, "n12940226": 20918, "n12940609": 20919, "n12941220": 20920, "n12941536": 20921, "n12941717": 20922, "n12942025": 20923, "n12942395": 20924, "n12942572": 20925, "n12942729": 20926, "n12943049": 20927, "n12943443": 20928, "n12943912": 20929, "n12944095": 20930, "n12945177": 20931, "n12945366": 20932, "n12945549": 20933, "n12946849": 20934, "n12947313": 20935, "n12947544": 20936, "n12947756": 20937, "n12947895": 20938, "n12948053": 20939, "n12948251": 20940, "n12948495": 20941, "n12949160": 20942, "n12949361": 20943, "n12950126": 20944, "n12950314": 20945, "n12950796": 20946, "n12951146": 20947, "n12951835": 20948, "n12952165": 20949, "n12952469": 20950, "n12952590": 20951, "n12952717": 20952, "n12953206": 20953, "n12953484": 20954, "n12953712": 20955, "n12954353": 20956, "n12954799": 20957, "n12955414": 20958, "n12955840": 20959, "n12956170": 20960, "n12956367": 20961, "n12956588": 20962, "n12956922": 20963, "n12957608": 20964, "n12957803": 20965, "n12957924": 20966, "n12958261": 20967, "n12958615": 20968, "n12959074": 20969, "n12959538": 20970, "n12960378": 20971, "n12960552": 20972, "n12960863": 20973, "n12961242": 20974, "n12961393": 20975, "n12961536": 20976, "n12961879": 20977, "n12963628": 20978, "n12964920": 20979, "n12965626": 20980, "n12965951": 20981, "n12966804": 20982, "n12966945": 20983, "n12968136": 20984, "n12968309": 20985, "n12969131": 20986, "n12969425": 20987, "n12969670": 20988, "n12969927": 20989, "n12970193": 20990, "n12970293": 20991, "n12970733": 20992, "n12971400": 20993, "n12971804": 20994, "n12972136": 20995, "n12973443": 20996, "n12973791": 20997, "n12973937": 20998, "n12974987": 20999, "n12975804": 21000, "n12976198": 21001, "n12976554": 21002, "n12978076": 21003, "n12979316": 21004, "n12979829": 21005, "n12980080": 21006, "n12980840": 21007, "n12981086": 21008, "n12981301": 21009, "n12981443": 21010, "n12981954": 21011, "n12982468": 21012, "n12982590": 21013, "n12982915": 21014, "n12983048": 21015, "n12983654": 21016, "n12983873": 21017, "n12983961": 21018, "n12984267": 21019, "n12984489": 21020, "n12984595": 21021, "n12985420": 21022, "n12985773": 21023, "n12985857": 21024, "n12986227": 21025, "n12987056": 21026, "n12987423": 21027, "n12987535": 21028, "n12988158": 21029, "n12988341": 21030, "n12988572": 21031, "n12989007": 21032, "n12989938": 21033, "n12990597": 21034, "n12991184": 21035, "n12991837": 21036, "n12992177": 21037, "n12992868": 21038, "n12994892": 21039, "n12995601": 21040, "n12997654": 21041, "n12997919": 21042, "n12998815": 21043, "n13000891": 21044, "n13001041": 21045, "n13001206": 21046, "n13001366": 21047, "n13001529": 21048, "n13001930": 21049, "n13002209": 21050, "n13002750": 21051, "n13002925": 21052, "n13003061": 21053, "n13003254": 21054, "n13003522": 21055, "n13003712": 21056, "n13004423": 21057, "n13004640": 21058, "n13004826": 21059, "n13004992": 21060, "n13005329": 21061, "n13005984": 21062, "n13006171": 21063, "n13006631": 21064, "n13006894": 21065, "n13007034": 21066, "n13007417": 21067, "n13007629": 21068, "n13008157": 21069, "n13008315": 21070, "n13008485": 21071, "n13008689": 21072, "n13008839": 21073, "n13009085": 21074, "n13009244": 21075, "n13009429": 21076, "n13009656": 21077, "n13010694": 21078, "n13010951": 21079, "n13011221": 21080, "n13011595": 21081, "n13012253": 21082, "n13012469": 21083, "n13012973": 21084, "n13013534": 21085, "n13013764": 21086, "n13013965": 21087, "n13014097": 21088, "n13014265": 21089, "n13014409": 21090, "n13014581": 21091, "n13014741": 21092, "n13014879": 21093, "n13015509": 21094, "n13015688": 21095, "n13016076": 21096, "n13016289": 21097, "n13017102": 21098, "n13017240": 21099, "n13017439": 21100, "n13017610": 21101, "n13017789": 21102, "n13017979": 21103, "n13018088": 21104, "n13018232": 21105, "n13018407": 21106, "n13018906": 21107, "n13019496": 21108, "n13019643": 21109, "n13019835": 21110, "n13020191": 21111, "n13020481": 21112, "n13020964": 21113, "n13021166": 21114, "n13021332": 21115, "n13021543": 21116, "n13021689": 21117, "n13021867": 21118, "n13022210": 21119, "n13022709": 21120, "n13022903": 21121, "n13023134": 21122, "n13024012": 21123, "n13024500": 21124, "n13024653": 21125, "n13025647": 21126, "n13025854": 21127, "n13026015": 21128, "n13027557": 21129, "n13027879": 21130, "n13028611": 21131, "n13028937": 21132, "n13029122": 21133, "n13029326": 21134, "n13029610": 21135, "n13029760": 21136, "n13030337": 21137, "n13030616": 21138, "n13030852": 21139, "n13031193": 21140, "n13031323": 21141, "n13031474": 21142, "n13032115": 21143, "n13032381": 21144, "n13032618": 21145, "n13032923": 21146, "n13033134": 21147, "n13033396": 21148, "n13033577": 21149, "n13033879": 21150, "n13034062": 21151, "n13034555": 21152, "n13034788": 21153, "n13035241": 21154, "n13035389": 21155, "n13035707": 21156, "n13035925": 21157, "n13036116": 21158, "n13036312": 21159, "n13036804": 21160, "n13037406": 21161, "n13037585": 21162, "n13037805": 21163, "n13038068": 21164, "n13038376": 21165, "n13038577": 21166, "n13038744": 21167, "n13039349": 21168, "n13040303": 21169, "n13040629": 21170, "n13040796": 21171, "n13041312": 21172, "n13041943": 21173, "n13042134": 21174, "n13042316": 21175, "n13042982": 21176, "n13043926": 21177, "n13044375": 21178, "n13044778": 21179, "n13045210": 21180, "n13045594": 21181, "n13045975": 21182, "n13046130": 21183, "n13046669": 21184, "n13047862": 21185, "n13048447": 21186, "n13049953": 21187, "n13050397": 21188, "n13050705": 21189, "n13050940": 21190, "n13051346": 21191, "n13052014": 21192, "n13052248": 21193, "n13052670": 21194, "n13052931": 21195, "n13053608": 21196, "n13054073": 21197, "n13054560": 21198, "n13055423": 21199, "n13055577": 21200, "n13055792": 21201, "n13055949": 21202, "n13056135": 21203, "n13056349": 21204, "n13056607": 21205, "n13056799": 21206, "n13057054": 21207, "n13057242": 21208, "n13057422": 21209, "n13057639": 21210, "n13058037": 21211, "n13058272": 21212, "n13058608": 21213, "n13059298": 21214, "n13059657": 21215, "n13060017": 21216, "n13060190": 21217, "n13061172": 21218, "n13061348": 21219, "n13061471": 21220, "n13061704": 21221, "n13062421": 21222, "n13063269": 21223, "n13063514": 21224, "n13064111": 21225, "n13064457": 21226, "n13065089": 21227, "n13065514": 21228, "n13066129": 21229, "n13066448": 21230, "n13066979": 21231, "n13067191": 21232, "n13067330": 21233, "n13067532": 21234, "n13067672": 21235, "n13068255": 21236, "n13068434": 21237, "n13068735": 21238, "n13068917": 21239, "n13069224": 21240, "n13069773": 21241, "n13070308": 21242, "n13070875": 21243, "n13071371": 21244, "n13071553": 21245, "n13071815": 21246, "n13072031": 21247, "n13072209": 21248, "n13072350": 21249, "n13072528": 21250, "n13072706": 21251, "n13072863": 21252, "n13073055": 21253, "n13073703": 21254, "n13074619": 21255, "n13074814": 21256, "n13075020": 21257, "n13075272": 21258, "n13075441": 21259, "n13075684": 21260, "n13075847": 21261, "n13076041": 21262, "n13076405": 21263, "n13076643": 21264, "n13076831": 21265, "n13077033": 21266, "n13077295": 21267, "n13078021": 21268, "n13079073": 21269, "n13079419": 21270, "n13079567": 21271, "n13080306": 21272, "n13080866": 21273, "n13081229": 21274, "n13081999": 21275, "n13082568": 21276, "n13083023": 21277, "n13083461": 21278, "n13084184": 21279, "n13084834": 21280, "n13085113": 21281, "n13085747": 21282, "n13090018": 21283, "n13090871": 21284, "n13091620": 21285, "n13091774": 21286, "n13091982": 21287, "n13092078": 21288, "n13092240": 21289, "n13092385": 21290, "n13092987": 21291, "n13093275": 21292, "n13093629": 21293, "n13094145": 21294, "n13094273": 21295, "n13095013": 21296, "n13096779": 21297, "n13098515": 21298, "n13098962": 21299, "n13099833": 21300, "n13099999": 21301, "n13100156": 21302, "n13100677": 21303, "n13102648": 21304, "n13102775": 21305, "n13103023": 21306, "n13103660": 21307, "n13103750": 21308, "n13103877": 21309, "n13104059": 21310, "n13107694": 21311, "n13107807": 21312, "n13107891": 21313, "n13108131": 21314, "n13108323": 21315, "n13108481": 21316, "n13108545": 21317, "n13108662": 21318, "n13108841": 21319, "n13109733": 21320, "n13110915": 21321, "n13111174": 21322, "n13111340": 21323, "n13111504": 21324, "n13111881": 21325, "n13112035": 21326, "n13112201": 21327, "n13118330": 21328, "n13118707": 21329, "n13119870": 21330, "n13120211": 21331, "n13120958": 21332, "n13121104": 21333, "n13121349": 21334, "n13122364": 21335, "n13123309": 21336, "n13123431": 21337, "n13123841": 21338, "n13124358": 21339, "n13124654": 21340, "n13125117": 21341, "n13126050": 21342, "n13126856": 21343, "n13127001": 21344, "n13127303": 21345, "n13127666": 21346, "n13127843": 21347, "n13128278": 21348, "n13128582": 21349, "n13128976": 21350, "n13129078": 21351, "n13130014": 21352, "n13130161": 21353, "n13130726": 21354, "n13131028": 21355, "n13131618": 21356, "n13132034": 21357, "n13132156": 21358, "n13132338": 21359, "n13132486": 21360, "n13132656": 21361, "n13132756": 21362, "n13132940": 21363, "n13133140": 21364, "n13133233": 21365, "n13133316": 21366, "n13133613": 21367, "n13133932": 21368, "n13134302": 21369, "n13134531": 21370, "n13134844": 21371, "n13134947": 21372, "n13135692": 21373, "n13135832": 21374, "n13136316": 21375, "n13136556": 21376, "n13136781": 21377, "n13137010": 21378, "n13137225": 21379, "n13137409": 21380, "n13137672": 21381, "n13137951": 21382, "n13138155": 21383, "n13138308": 21384, "n13138658": 21385, "n13138842": 21386, "n13139055": 21387, "n13139321": 21388, "n13139482": 21389, "n13139647": 21390, "n13139837": 21391, "n13140049": 21392, "n13140367": 21393, "n13141141": 21394, "n13141415": 21395, "n13141564": 21396, "n13141797": 21397, "n13141972": 21398, "n13142182": 21399, "n13142504": 21400, "n13142907": 21401, "n13143285": 21402, "n13143758": 21403, "n13144084": 21404, "n13145040": 21405, "n13145250": 21406, "n13145444": 21407, "n13146403": 21408, "n13146583": 21409, "n13146928": 21410, "n13147153": 21411, "n13147270": 21412, "n13147386": 21413, "n13147532": 21414, "n13147689": 21415, "n13147918": 21416, "n13148208": 21417, "n13148384": 21418, "n13149296": 21419, "n13149970": 21420, "n13150378": 21421, "n13150592": 21422, "n13150894": 21423, "n13151082": 21424, "n13152339": 21425, "n13154388": 21426, "n13154494": 21427, "n13154841": 21428, "n13155095": 21429, "n13155305": 21430, "n13155611": 21431, "n13156986": 21432, "n13157137": 21433, "n13157346": 21434, "n13157481": 21435, "n13157684": 21436, "n13157971": 21437, "n13158167": 21438, "n13158512": 21439, "n13158605": 21440, "n13158714": 21441, "n13158815": 21442, "n13159357": 21443, "n13159691": 21444, "n13159890": 21445, "n13160116": 21446, "n13160254": 21447, "n13160365": 21448, "n13160604": 21449, "n13160831": 21450, "n13160938": 21451, "n13161151": 21452, "n13161254": 21453, "n13161904": 21454, "n13163553": 21455, "n13163649": 21456, "n13163991": 21457, "n13164501": 21458, "n13170840": 21459, "n13171210": 21460, "n13171797": 21461, "n13172923": 21462, "n13173132": 21463, "n13173259": 21464, "n13173488": 21465, "n13173697": 21466, "n13173882": 21467, "n13174354": 21468, "n13174670": 21469, "n13174823": 21470, "n13175682": 21471, "n13176363": 21472, "n13176714": 21473, "n13177048": 21474, "n13177529": 21475, "n13177768": 21476, "n13177884": 21477, "n13178284": 21478, "n13178707": 21479, "n13179056": 21480, "n13179804": 21481, "n13180534": 21482, "n13180875": 21483, "n13181055": 21484, "n13181244": 21485, "n13181406": 21486, "n13181811": 21487, "n13182164": 21488, "n13182338": 21489, "n13182799": 21490, "n13182937": 21491, "n13183056": 21492, "n13183489": 21493, "n13184394": 21494, "n13185269": 21495, "n13185658": 21496, "n13186388": 21497, "n13186546": 21498, "n13187367": 21499, "n13188096": 21500, "n13188268": 21501, "n13188462": 21502, "n13188767": 21503, "n13190060": 21504, "n13190747": 21505, "n13191148": 21506, "n13191620": 21507, "n13191884": 21508, "n13192625": 21509, "n13193143": 21510, "n13193269": 21511, "n13193466": 21512, "n13193642": 21513, "n13193856": 21514, "n13194036": 21515, "n13194212": 21516, "n13194572": 21517, "n13194758": 21518, "n13194918": 21519, "n13195341": 21520, "n13195761": 21521, "n13196003": 21522, "n13196234": 21523, "n13196369": 21524, "n13196738": 21525, "n13197274": 21526, "n13197507": 21527, "n13198054": 21528, "n13198482": 21529, "n13198914": 21530, "n13199717": 21531, "n13199970": 21532, "n13200193": 21533, "n13200542": 21534, "n13200651": 21535, "n13200986": 21536, "n13201423": 21537, "n13201566": 21538, "n13201969": 21539, "n13202125": 21540, "n13202355": 21541, "n13202602": 21542, "n13205058": 21543, "n13205249": 21544, "n13206178": 21545, "n13206817": 21546, "n13207094": 21547, "n13207335": 21548, "n13207572": 21549, "n13207736": 21550, "n13207923": 21551, "n13208302": 21552, "n13208705": 21553, "n13208965": 21554, "n13209129": 21555, "n13209270": 21556, "n13209460": 21557, "n13209808": 21558, "n13210350": 21559, "n13210597": 21560, "n13211020": 21561, "n13211790": 21562, "n13212025": 21563, "n13212175": 21564, "n13212379": 21565, "n13212559": 21566, "n13213066": 21567, "n13213397": 21568, "n13213577": 21569, "n13214217": 21570, "n13214340": 21571, "n13214485": 21572, "n13215258": 21573, "n13215586": 21574, "n13217005": 21575, "n13219422": 21576, "n13219833": 21577, "n13219976": 21578, "n13220122": 21579, "n13220355": 21580, "n13220525": 21581, "n13220663": 21582, "n13221529": 21583, "n13222877": 21584, "n13222985": 21585, "n13223090": 21586, "n13223588": 21587, "n13223710": 21588, "n13223843": 21589, "n13224673": 21590, "n13224922": 21591, "n13225244": 21592, "n13225365": 21593, "n13225617": 21594, "n13226320": 21595, "n13226871": 21596, "n13228017": 21597, "n13228536": 21598, "n13229543": 21599, "n13229951": 21600, "n13230190": 21601, "n13230662": 21602, "n13230843": 21603, "n13231078": 21604, "n13231678": 21605, "n13231919": 21606, "n13232106": 21607, "n13232363": 21608, "n13232779": 21609, "n13233727": 21610, "n13234114": 21611, "n13234519": 21612, "n13234678": 21613, "n13234857": 21614, "n13235011": 21615, "n13235159": 21616, "n13235319": 21617, "n13235503": 21618, "n13235766": 21619, "n13236100": 21620, "n13237188": 21621, "n13237508": 21622, "n13238375": 21623, "n13238654": 21624, "n13238988": 21625, "n13239177": 21626, "n13239736": 21627, "n13239921": 21628, "n13240362": 21629, "n13252672": 21630, "n13354021": 21631, "n13555775": 21632, "n13579829": 21633, "n13650447": 21634, "n13653902": 21635, "n13862407": 21636, "n13862552": 21637, "n13862780": 21638, "n13863020": 21639, "n13863186": 21640, "n13863473": 21641, "n13863771": 21642, "n13864035": 21643, "n13864153": 21644, "n13864965": 21645, "n13865298": 21646, "n13865483": 21647, "n13865904": 21648, "n13866144": 21649, "n13866626": 21650, "n13866827": 21651, "n13867005": 21652, "n13867492": 21653, "n13868248": 21654, "n13868371": 21655, "n13868515": 21656, "n13868944": 21657, "n13869045": 21658, "n13869547": 21659, "n13869788": 21660, "n13869896": 21661, "n13871717": 21662, "n13872592": 21663, "n13872822": 21664, "n13873361": 21665, "n13873502": 21666, "n13873917": 21667, "n13874073": 21668, "n13874558": 21669, "n13875392": 21670, "n13875571": 21671, "n13875884": 21672, "n13876561": 21673, "n13877547": 21674, "n13877667": 21675, "n13878306": 21676, "n13879049": 21677, "n13879320": 21678, "n13879816": 21679, "n13880199": 21680, "n13880415": 21681, "n13880551": 21682, "n13880704": 21683, "n13880994": 21684, "n13881512": 21685, "n13881644": 21686, "n13882201": 21687, "n13882276": 21688, "n13882487": 21689, "n13882563": 21690, "n13882639": 21691, "n13882713": 21692, "n13882961": 21693, "n13883603": 21694, "n13883763": 21695, "n13884261": 21696, "n13884384": 21697, "n13884930": 21698, "n13885011": 21699, "n13886260": 21700, "n13888491": 21701, "n13889066": 21702, "n13889331": 21703, "n13891547": 21704, "n13891937": 21705, "n13893786": 21706, "n13894154": 21707, "n13894434": 21708, "n13895262": 21709, "n13896100": 21710, "n13896217": 21711, "n13897198": 21712, "n13897528": 21713, "n13897996": 21714, "n13898207": 21715, "n13898315": 21716, "n13898645": 21717, "n13899735": 21718, "n13900287": 21719, "n13900422": 21720, "n13901211": 21721, "n13901321": 21722, "n13901423": 21723, "n13901490": 21724, "n13901858": 21725, "n13902048": 21726, "n13902336": 21727, "n13902793": 21728, "n13903079": 21729, "n13905121": 21730, "n13905275": 21731, "n13905792": 21732, "n13906484": 21733, "n13906669": 21734, "n13906767": 21735, "n13906936": 21736, "n13907272": 21737, "n13908201": 21738, "n13908580": 21739, "n13911045": 21740, "n13912260": 21741, "n13912540": 21742, "n13914141": 21743, "n13914265": 21744, "n13914608": 21745, "n13915023": 21746, "n13915113": 21747, "n13915209": 21748, "n13915305": 21749, "n13915999": 21750, "n13916363": 21751, "n13916721": 21752, "n13917690": 21753, "n13917785": 21754, "n13918274": 21755, "n13918387": 21756, "n13918717": 21757, "n13919547": 21758, "n13919919": 21759, "n13926786": 21760, "n14131950": 21761, "n14175579": 21762, "n14564779": 21763, "n14582716": 21764, "n14583400": 21765, "n14585392": 21766, "n14592309": 21767, "n14603798": 21768, "n14633206": 21769, "n14685296": 21770, "n14696793": 21771, "n14698884": 21772, "n14714645": 21773, "n14720833": 21774, "n14765422": 21775, "n14785065": 21776, "n14786943": 21777, "n14804958": 21778, "n14810561": 21779, "n14820180": 21780, "n14821852": 21781, "n14844693": 21782, "n14853210": 21783, "n14858292": 21784, "n14867545": 21785, "n14891255": 21786, "n14899328": 21787, "n14900184": 21788, "n14900342": 21789, "n14908027": 21790, "n14909584": 21791, "n14914945": 21792, "n14915184": 21793, "n14919819": 21794, "n14938389": 21795, "n14941787": 21796, "n14942411": 21797, "n14973585": 21798, "n14974264": 21799, "n14975598": 21800, "n14976759": 21801, "n14976871": 21802, "n14977188": 21803, "n14977504": 21804, "n14992287": 21805, "n14993378": 21806, "n15005577": 21807, "n15006012": 21808, "n15019030": 21809, "n15048888": 21810, "n15060326": 21811, "n15060688": 21812, "n15062057": 21813, "n15067877": 21814, "n15075141": 21815, "n15086247": 21816, "n15089258": 21817, "n15089472": 21818, "n15089645": 21819, "n15089803": 21820, "n15090065": 21821, "n15090238": 21822, "n15090742": 21823, "n15091129": 21824, "n15091304": 21825, "n15091473": 21826, "n15091669": 21827, "n15091846": 21828, "n15092059": 21829, "n15092227": 21830, "n15092409": 21831, "n15092650": 21832, "n15092751": 21833, "n15092942": 21834, "n15093049": 21835, "n15093137": 21836, "n15093298": 21837, "n15102359": 21838, "n15102455": 21839, "n15102894": 21840} diff --git a/classification/meta_data/map22kto1k.txt b/classification/meta_data/map22kto1k.txt new file mode 100644 index 0000000..a543079 --- /dev/null +++ b/classification/meta_data/map22kto1k.txt @@ -0,0 +1,1000 @@ +359 +368 +460 +475 +486 +492 +496 +514 +516 +525 +547 +548 +556 +563 +575 +641 +648 +723 +733 +765 +801 +826 +852 +858 +878 +896 +900 +905 +908 +910 +935 +946 +947 +994 +999 +1003 +1005 +1010 +1027 +1029 +1048 +1055 +1064 +1065 +1069 +1075 +1079 +1081 +1085 +1088 +1093 +1106 +1143 +1144 +1145 +1147 +1168 +1171 +1178 +1187 +1190 +1197 +1205 +1216 +1223 +1230 +1236 +1241 +1245 +1257 +1259 +1260 +1267 +1268 +1269 +1271 +1272 +1273 +1277 +1303 +1344 +1349 +1355 +1357 +1384 +1388 +1391 +1427 +1429 +1432 +1437 +1450 +1461 +1462 +1474 +1502 +1503 +1512 +1552 +1555 +1577 +1584 +1587 +1589 +1599 +1615 +1616 +1681 +1692 +1701 +1716 +1729 +1757 +1759 +1764 +1777 +1786 +1822 +1841 +1842 +1848 +1850 +1856 +1860 +1861 +1864 +1876 +1897 +1898 +1910 +1913 +1918 +1922 +1928 +1932 +1935 +1947 +1951 +1953 +1970 +1977 +1979 +2001 +2017 +2067 +2081 +2087 +2112 +2128 +2135 +2147 +2174 +2175 +2176 +2177 +2178 +2181 +2183 +2184 +2187 +2189 +2190 +2191 +2192 +2193 +2197 +2202 +2203 +2206 +2208 +2209 +2211 +2212 +2213 +2214 +2215 +2216 +2217 +2219 +2222 +2223 +2224 +2225 +2226 +2227 +2228 +2229 +2230 +2236 +2238 +2240 +2241 +2242 +2243 +2244 +2245 +2247 +2248 +2249 +2250 +2251 +2252 +2255 +2256 +2257 +2262 +2263 +2264 +2265 +2266 +2268 +2270 +2271 +2272 +2273 +2275 +2276 +2279 +2280 +2281 +2282 +2285 +2289 +2292 +2295 +2296 +2297 +2298 +2299 +2300 +2301 +2302 +2303 +2304 +2305 +2306 +2309 +2310 +2312 +2313 +2314 +2315 +2316 +2318 +2319 +2321 +2322 +2326 +2329 +2330 +2331 +2332 +2334 +2335 +2336 +2337 +2338 +2339 +2341 +2342 +2343 +2344 +2346 +2348 +2349 +2351 +2352 +2353 +2355 +2357 +2358 +2359 +2360 +2364 +2365 +2368 +2369 +2377 +2382 +2383 +2385 +2397 +2398 +2400 +2402 +2405 +2412 +2421 +2428 +2431 +2432 +2433 +2436 +2441 +2445 +2450 +2453 +2454 +2465 +2469 +2532 +2533 +2538 +2544 +2547 +2557 +2565 +2578 +2612 +2658 +2702 +2722 +2731 +2738 +2741 +2747 +2810 +2818 +2833 +2844 +2845 +2867 +2874 +2882 +2884 +2888 +2889 +3008 +3012 +3019 +3029 +3033 +3042 +3091 +3106 +3138 +3159 +3164 +3169 +3280 +3296 +3311 +3318 +3320 +3324 +3330 +3366 +3375 +3381 +3406 +3419 +3432 +3434 +3435 +3493 +3495 +3503 +3509 +3511 +3513 +3517 +3521 +3526 +3546 +3554 +3600 +3601 +3606 +3612 +3613 +3616 +3622 +3623 +3627 +3632 +3634 +3636 +3638 +3644 +3646 +3649 +3650 +3651 +3656 +3663 +3673 +3674 +3689 +3690 +3702 +3733 +3769 +3971 +3974 +4065 +4068 +4073 +4102 +4136 +4140 +4151 +4159 +4165 +4207 +4219 +4226 +4249 +4256 +4263 +4270 +4313 +4321 +4378 +4386 +4478 +4508 +4512 +4536 +4542 +4550 +4560 +4562 +4570 +4571 +4572 +4583 +4588 +4594 +4604 +4608 +4623 +4634 +4636 +4646 +4651 +4652 +4686 +4688 +4691 +4699 +4724 +4727 +4737 +4770 +4774 +4789 +4802 +4807 +4819 +4880 +4886 +4908 +4927 +4931 +4936 +4964 +4976 +4993 +5028 +5033 +5043 +5046 +5096 +5111 +5114 +5131 +5132 +5183 +5199 +5235 +5275 +5291 +5293 +5294 +5343 +5360 +5362 +5364 +5390 +5402 +5418 +5428 +5430 +5437 +5443 +5473 +5484 +5486 +5505 +5507 +5508 +5510 +5567 +5578 +5580 +5584 +5606 +5613 +5629 +5672 +5676 +5692 +5701 +5760 +5769 +5770 +5779 +5814 +5850 +5871 +5893 +5911 +5949 +5954 +6005 +6006 +6012 +6017 +6023 +6024 +6040 +6050 +6054 +6087 +6105 +6157 +6235 +6237 +6256 +6259 +6286 +6291 +6306 +6339 +6341 +6343 +6379 +6383 +6393 +6405 +6479 +6511 +6517 +6541 +6561 +6608 +6611 +6615 +6678 +6682 +6707 +6752 +6798 +6850 +6880 +6885 +6890 +6920 +6981 +7000 +7009 +7038 +7049 +7050 +7052 +7073 +7078 +7098 +7111 +7165 +7198 +7204 +7280 +7283 +7286 +7287 +7293 +7294 +7305 +7318 +7341 +7346 +7354 +7382 +7427 +7428 +7435 +7445 +7450 +7455 +7467 +7469 +7497 +7502 +7506 +7514 +7523 +7651 +7661 +7664 +7672 +7679 +7685 +7696 +7730 +7871 +7873 +7895 +7914 +7915 +7920 +7934 +7935 +7949 +8009 +8036 +8051 +8065 +8074 +8090 +8112 +8140 +8164 +8168 +8178 +8182 +8198 +8212 +8216 +8230 +8242 +8288 +8289 +8295 +8318 +8352 +8368 +8371 +8375 +8376 +8401 +8416 +8419 +8436 +8460 +8477 +8478 +8482 +8498 +8500 +8539 +8543 +8552 +8555 +8580 +8584 +8586 +8594 +8598 +8601 +8606 +8610 +8611 +8622 +8627 +8639 +8649 +8650 +8653 +8654 +8667 +8672 +8673 +8674 +8676 +8684 +8720 +8723 +8750 +8753 +8801 +8815 +8831 +8835 +8842 +8845 +8858 +8897 +8916 +8951 +8954 +8959 +8970 +8976 +8981 +8983 +8989 +8991 +8993 +9019 +9039 +9042 +9043 +9056 +9057 +9070 +9087 +9098 +9106 +9130 +9131 +9155 +9171 +9183 +9198 +9199 +9201 +9204 +9211 +9220 +9224 +9228 +9249 +9259 +9270 +9278 +9294 +9299 +9309 +9321 +9344 +9351 +9375 +9376 +9381 +9391 +9400 +9404 +9440 +9448 +9463 +9474 +9501 +9504 +9513 +9514 +9544 +9566 +9575 +9607 +9608 +9623 +9632 +9638 +9642 +9655 +9673 +9739 +9751 +9759 +9766 +9777 +9801 +9819 +9838 +9878 +9923 +9955 +9960 +9962 +9969 +9996 +10009 +10030 +10039 +10051 +10072 +10074 +10077 +10093 +10096 +10108 +10117 +10120 +10123 +10157 +10225 +10275 +10303 +10306 +10313 +10314 +10331 +10336 +10337 +10412 +10422 +10450 +10462 +10464 +10486 +10518 +10521 +10522 +10531 +10533 +10534 +10550 +10558 +10573 +10582 +10585 +10588 +10611 +10625 +10634 +10637 +10676 +10682 +10725 +10775 +10781 +10782 +10806 +10836 +10839 +10847 +10858 +10870 +10880 +10883 +10907 +10913 +10920 +10935 +10946 +10950 +10951 +10956 +10998 +11002 +11017 +11022 +11024 +11026 +11044 +11054 +11094 +11109 +11136 +11136 +11167 +11185 +11220 +11222 +11241 +11254 +11258 +11278 +11305 +11310 +11330 +11366 +11376 +11388 +11391 +11400 +11406 +11436 +11448 +11465 +11468 +11472 +11477 +11482 +11483 +11506 +11535 +11557 +11565 +11574 +11583 +11593 +11610 +11611 +11618 +11620 +11639 +11642 +11663 +11673 +11688 +11708 +11709 +11715 +11720 +11725 +11728 +11742 +11759 +11770 +11836 +11838 +11855 +11875 +11877 +11883 +11888 +11895 +11916 +11922 +11929 +11943 +11951 +11979 +11983 +12213 +12228 +12238 +12240 +12241 +12246 +12282 +12348 +12368 +12372 +12421 +12559 +12565 +12574 +12687 +12754 +12767 +12777 +12779 +12811 +12831 +12834 +12835 +12842 +12846 +12848 +12849 +12855 +12857 +12872 +12937 +12970 +13016 +13037 +13045 +13058 +13084 +13085 +13087 +13093 +13133 +13181 +13229 +13405 +13443 +13613 +13689 +13697 +13708 +13748 +13803 +13981 +14050 +14058 +14218 +14245 +14255 +14263 +14293 +14323 +14366 +14388 +14393 +14437 +14441 +14964 +15730 +16742 +18035 +18203 +18533 +18790 +19100 +20017 +20460 +21024 +21043 +21161 +21169 +21179 +21194 +21198 +21367 +21815 \ No newline at end of file diff --git a/classification/meta_data/meta b/classification/meta_data/meta new file mode 120000 index 0000000..20a6977 --- /dev/null +++ b/classification/meta_data/meta @@ -0,0 +1 @@ +/mnt/petrelfs/share/images/meta/ \ No newline at end of file diff --git a/classification/models/__init__.py b/classification/models/__init__.py new file mode 100644 index 0000000..732efbc --- /dev/null +++ b/classification/models/__init__.py @@ -0,0 +1,7 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + +from .build import build_model \ No newline at end of file diff --git a/classification/models/build.py b/classification/models/build.py new file mode 100644 index 0000000..5d22332 --- /dev/null +++ b/classification/models/build.py @@ -0,0 +1,58 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + + +from .intern_image import InternImage +from .flash_intern_image import FlashInternImage + +def build_model(config): + model_type = config.MODEL.TYPE + if model_type == 'intern_image': + model = InternImage( + core_op=config.MODEL.INTERN_IMAGE.CORE_OP, + num_classes=config.MODEL.NUM_CLASSES, + channels=config.MODEL.INTERN_IMAGE.CHANNELS, + depths=config.MODEL.INTERN_IMAGE.DEPTHS, + groups=config.MODEL.INTERN_IMAGE.GROUPS, + layer_scale=config.MODEL.INTERN_IMAGE.LAYER_SCALE, + offset_scale=config.MODEL.INTERN_IMAGE.OFFSET_SCALE, + post_norm=config.MODEL.INTERN_IMAGE.POST_NORM, + mlp_ratio=config.MODEL.INTERN_IMAGE.MLP_RATIO, + with_cp=config.TRAIN.USE_CHECKPOINT, + drop_path_rate=config.MODEL.DROP_PATH_RATE, + res_post_norm=config.MODEL.INTERN_IMAGE.RES_POST_NORM, # for InternImage-H/G + dw_kernel_size=config.MODEL.INTERN_IMAGE.DW_KERNEL_SIZE, # for InternImage-H/G + use_clip_projector=config.MODEL.INTERN_IMAGE.USE_CLIP_PROJECTOR, # for InternImage-H/G + level2_post_norm=config.MODEL.INTERN_IMAGE.LEVEL2_POST_NORM, # for InternImage-H/G + level2_post_norm_block_ids=config.MODEL.INTERN_IMAGE.LEVEL2_POST_NORM_BLOCK_IDS, # for InternImage-H/G + center_feature_scale=config.MODEL.INTERN_IMAGE.CENTER_FEATURE_SCALE # for InternImage-H/G + ) + elif model_type == 'flash_intern_image': + model = FlashInternImage( + core_op=config.MODEL.FLASH_INTERN_IMAGE.CORE_OP, + num_classes=config.MODEL.NUM_CLASSES, + channels=config.MODEL.FLASH_INTERN_IMAGE.CHANNELS, + depths=config.MODEL.FLASH_INTERN_IMAGE.DEPTHS, + groups=config.MODEL.FLASH_INTERN_IMAGE.GROUPS, + layer_scale=config.MODEL.FLASH_INTERN_IMAGE.LAYER_SCALE, + offset_scale=config.MODEL.FLASH_INTERN_IMAGE.OFFSET_SCALE, + post_norm=config.MODEL.FLASH_INTERN_IMAGE.POST_NORM, + mlp_ratio=config.MODEL.FLASH_INTERN_IMAGE.MLP_RATIO, + with_cp=config.TRAIN.USE_CHECKPOINT, + drop_path_rate=config.MODEL.DROP_PATH_RATE, + mlp_fc2_bias=config.MODEL.FLASH_INTERN_IMAGE.MLP_FC2_BIAS, + dcn_output_bias=config.MODEL.FLASH_INTERN_IMAGE.DCN_OUTPUT_BIAS, + res_post_norm=config.MODEL.FLASH_INTERN_IMAGE.RES_POST_NORM, # for InternImage-H/G + dw_kernel_size=config.MODEL.FLASH_INTERN_IMAGE.DW_KERNEL_SIZE, + use_clip_projector=config.MODEL.FLASH_INTERN_IMAGE.USE_CLIP_PROJECTOR, # for InternImage-H/G + level2_post_norm=config.MODEL.FLASH_INTERN_IMAGE.LEVEL2_POST_NORM, # for InternImage-H/G + level2_post_norm_block_ids=config.MODEL.FLASH_INTERN_IMAGE.LEVEL2_POST_NORM_BLOCK_IDS, # for InternImage-H/G + center_feature_scale=config.MODEL.FLASH_INTERN_IMAGE.CENTER_FEATURE_SCALE # for InternImage-H/G + ) + else: + raise NotImplementedError(f"Unkown model: {model_type}") + + return model diff --git a/classification/models/flash_intern_image.py b/classification/models/flash_intern_image.py new file mode 100644 index 0000000..0e07f19 --- /dev/null +++ b/classification/models/flash_intern_image.py @@ -0,0 +1,795 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + + +import torch +import torch.nn as nn +from collections import OrderedDict +import torch.utils.checkpoint as checkpoint +from timm.models.layers import trunc_normal_, DropPath +import torch.nn.functional as F +import DCNv4 + + +class to_channels_first(nn.Module): + + def __init__(self): + super().__init__() + + def forward(self, x): + return x.permute(0, 3, 1, 2) + + +class to_channels_last(nn.Module): + + def __init__(self): + super().__init__() + + def forward(self, x): + return x.permute(0, 2, 3, 1) + + +def build_norm_layer(dim, + norm_layer, + in_format='channels_last', + out_format='channels_last', + eps=1e-6): + layers = [] + if norm_layer == 'BN': + if in_format == 'channels_last': + layers.append(to_channels_first()) + layers.append(nn.BatchNorm2d(dim)) + if out_format == 'channels_last': + layers.append(to_channels_last()) + elif norm_layer == 'LN': + if in_format == 'channels_first': + layers.append(to_channels_last()) + layers.append(nn.LayerNorm(dim, eps=eps)) + if out_format == 'channels_first': + layers.append(to_channels_first()) + else: + raise NotImplementedError( + f'build_norm_layer does not support {norm_layer}') + return nn.Sequential(*layers) + + +def build_act_layer(act_layer): + if act_layer == 'ReLU': + return nn.ReLU(inplace=True) + elif act_layer == 'SiLU': + return nn.SiLU(inplace=True) + elif act_layer == 'GELU': + return nn.GELU() + + raise NotImplementedError(f'build_act_layer does not support {act_layer}') + + +class CrossAttention(nn.Module): + r""" Cross Attention Module + Args: + dim (int): Number of input channels. + num_heads (int): Number of attention heads. Default: 8 + qkv_bias (bool, optional): If True, add a learnable bias to q, k, v. + Default: False. + qk_scale (float | None, optional): Override default qk scale of + head_dim ** -0.5 if set. Default: None. + attn_drop (float, optional): Dropout ratio of attention weight. + Default: 0.0 + proj_drop (float, optional): Dropout ratio of output. Default: 0.0 + attn_head_dim (int, optional): Dimension of attention head. + out_dim (int, optional): Dimension of output. + """ + + def __init__(self, + dim, + num_heads=8, + qkv_bias=False, + qk_scale=None, + attn_drop=0., + proj_drop=0., + attn_head_dim=None, + out_dim=None): + super().__init__() + if out_dim is None: + out_dim = dim + self.num_heads = num_heads + head_dim = dim // num_heads + if attn_head_dim is not None: + head_dim = attn_head_dim + all_head_dim = head_dim * self.num_heads + self.scale = qk_scale or head_dim ** -0.5 + assert all_head_dim == dim + + self.q = nn.Linear(dim, all_head_dim, bias=False) + self.k = nn.Linear(dim, all_head_dim, bias=False) + self.v = nn.Linear(dim, all_head_dim, bias=False) + + if qkv_bias: + self.q_bias = nn.Parameter(torch.zeros(all_head_dim)) + self.k_bias = nn.Parameter(torch.zeros(all_head_dim)) + self.v_bias = nn.Parameter(torch.zeros(all_head_dim)) + else: + self.q_bias = None + self.k_bias = None + self.v_bias = None + + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(all_head_dim, out_dim) + self.proj_drop = nn.Dropout(proj_drop) + + def forward(self, x, k=None, v=None): + B, N, C = x.shape + N_k = k.shape[1] + N_v = v.shape[1] + + q_bias, k_bias, v_bias = None, None, None + if self.q_bias is not None: + q_bias = self.q_bias + k_bias = self.k_bias + v_bias = self.v_bias + + q = F.linear(input=x, weight=self.q.weight, bias=q_bias) + q = q.reshape(B, N, 1, self.num_heads, + -1).permute(2, 0, 3, 1, + 4).squeeze(0) # (B, N_head, N_q, dim) + + k = F.linear(input=k, weight=self.k.weight, bias=k_bias) + k = k.reshape(B, N_k, 1, self.num_heads, -1).permute(2, 0, 3, 1, + 4).squeeze(0) + + v = F.linear(input=v, weight=self.v.weight, bias=v_bias) + v = v.reshape(B, N_v, 1, self.num_heads, -1).permute(2, 0, 3, 1, + 4).squeeze(0) + + q = q * self.scale + attn = (q @ k.transpose(-2, -1)) # (B, N_head, N_q, N_k) + + attn = attn.softmax(dim=-1) + attn = self.attn_drop(attn) + + x = (attn @ v).transpose(1, 2).reshape(B, N, -1) + x = self.proj(x) + x = self.proj_drop(x) + + return x + + +class AttentiveBlock(nn.Module): + r"""Attentive Block + Args: + dim (int): Number of input channels. + num_heads (int): Number of attention heads. Default: 8 + qkv_bias (bool, optional): If True, add a learnable bias to q, k, v. + Default: False. + qk_scale (float | None, optional): Override default qk scale of + head_dim ** -0.5 if set. Default: None. + drop (float, optional): Dropout rate. Default: 0.0. + attn_drop (float, optional): Attention dropout rate. Default: 0.0. + drop_path (float | tuple[float], optional): Stochastic depth rate. + Default: 0.0. + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm. + attn_head_dim (int, optional): Dimension of attention head. Default: None. + out_dim (int, optional): Dimension of output. Default: None. + """ + + def __init__(self, + dim, + num_heads, + qkv_bias=False, + qk_scale=None, + drop=0., + attn_drop=0., + drop_path=0., + norm_layer="LN", + attn_head_dim=None, + out_dim=None): + super().__init__() + + self.norm1_q = build_norm_layer(dim, norm_layer, eps=1e-6) + self.norm1_k = build_norm_layer(dim, norm_layer, eps=1e-6) + self.norm1_v = build_norm_layer(dim, norm_layer, eps=1e-6) + self.cross_dcn = CrossAttention(dim, + num_heads=num_heads, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + attn_drop=attn_drop, + proj_drop=drop, + attn_head_dim=attn_head_dim, + out_dim=out_dim) + + self.drop_path = DropPath( + drop_path) if drop_path > 0. else nn.Identity() + + def forward(self, + x_q, + x_kv, + pos_q, + pos_k, + bool_masked_pos, + rel_pos_bias=None): + x_q = self.norm1_q(x_q + pos_q) + x_k = self.norm1_k(x_kv + pos_k) + x_v = self.norm1_v(x_kv) + + x = self.cross_dcn(x_q, k=x_k, v=x_v) + + return x + + +class AttentionPoolingBlock(AttentiveBlock): + + def forward(self, x): + x_q = x.mean(1, keepdim=True) + x_kv = x + pos_q, pos_k = 0, 0 + x = super().forward(x_q, x_kv, pos_q, pos_k, + bool_masked_pos=None, + rel_pos_bias=None) + x = x.squeeze(1) + return x + + +class StemLayer(nn.Module): + r""" Stem layer of InternImage + Args: + in_chans (int): number of input channels + out_chans (int): number of output channels + act_layer (str): activation layer + norm_layer (str): normalization layer + """ + + def __init__(self, + in_chans=3, + out_chans=96, + act_layer='GELU', + norm_layer='BN'): + super().__init__() + self.conv1 = nn.Conv2d(in_chans, + out_chans // 2, + kernel_size=3, + stride=2, + padding=1) + self.norm1 = build_norm_layer(out_chans // 2, norm_layer, + 'channels_first', 'channels_first') + self.act = build_act_layer(act_layer) + self.conv2 = nn.Conv2d(out_chans // 2, + out_chans, + kernel_size=3, + stride=2, + padding=1) + self.norm2 = build_norm_layer(out_chans, norm_layer, 'channels_first', + 'channels_last') + + def forward(self, x): + x = self.conv1(x) + x = self.norm1(x) + x = self.act(x) + x = self.conv2(x) + x = self.norm2(x) + return x + + + +class DownsampleLayer(nn.Module): + r""" Downsample layer of InternImage + Args: + channels (int): number of input channels + norm_layer (str): normalization layer + """ + + def __init__(self, channels, norm_layer='LN'): + super().__init__() + self.conv = nn.Conv2d(channels, + 2 * channels, + kernel_size=3, + stride=2, + padding=1, + bias=False) + self.norm = build_norm_layer(2 * channels, norm_layer, + 'channels_first', 'channels_first') + + + def forward(self, x, shape=None): + H, W = shape + N, HW, C = x.shape + x = x.view(N, H, W, C) + x = self.conv(x.permute(0, 3, 1, 2)) + x = self.norm(x) # B C H W + H, W = x.size(2), x.size(3) + x = x.flatten(2).permute(0, 2, 1) + + return x, (H, W) + + + +class MLPLayer(nn.Module): + r""" MLP layer of InternImage + Args: + in_features (int): number of input features + hidden_features (int): number of hidden features + out_features (int): number of output features + act_layer (str): activation layer + drop (float): dropout rate + """ + + def __init__(self, + in_features, + hidden_features=None, + out_features=None, + act_layer='GELU', + mlp_fc2_bias=False, + drop=0.): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = nn.Linear(in_features, hidden_features, bias=True) + self.act = build_act_layer(act_layer) + self.fc2 = nn.Linear(hidden_features, out_features, bias=mlp_fc2_bias) + self.drop = nn.Dropout(drop) + + + def forward(self, x, shape): + x = self.fc1(x) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + return x + + +class InternImageLayer(nn.Module): + r""" Basic layer of InternImage + Args: + core_op (nn.Module): core operation of InternImage + channels (int): number of input channels + groups (list): Groups of each block. + mlp_ratio (float): ratio of mlp hidden features to input channels + drop (float): dropout rate + drop_path (float): drop path rate + act_layer (str): activation layer + norm_layer (str): normalization layer + post_norm (bool): whether to use post normalization + layer_scale (float): layer scale + offset_scale (float): offset scale + with_cp (bool): whether to use checkpoint + """ + + def __init__(self, + core_op, + channels, + groups, + mlp_ratio=4., + drop=0., + drop_path=0., + act_layer='GELU', + norm_layer='LN', + post_norm=False, + layer_scale=None, + offset_scale=1.0, + with_cp=False, + dcn_output_bias=False, + mlp_fc2_bias=False, + dw_kernel_size=None, # for InternImage-H/G + res_post_norm=False, # for InternImage-H/G + center_feature_scale=False): # for InternImage-H/G + super().__init__() + self.channels = channels + self.groups = groups + self.mlp_ratio = mlp_ratio + self.with_cp = with_cp + + self.norm1 = build_norm_layer(channels, 'LN') + self.post_norm = post_norm + self.dcn = core_op( + channels=channels, + group=groups, + offset_scale=offset_scale, + dw_kernel_size=dw_kernel_size, + output_bias=dcn_output_bias, + ) + self.drop_path = DropPath(drop_path) if drop_path > 0. \ + else nn.Identity() + self.norm2 = build_norm_layer(channels, 'LN') + self.mlp = MLPLayer(in_features=channels, + hidden_features=int(channels * mlp_ratio), + act_layer=act_layer, + drop=drop, + mlp_fc2_bias=mlp_fc2_bias + ) + self.layer_scale = layer_scale is not None + if self.layer_scale: + self.gamma1 = nn.Parameter(layer_scale * torch.ones(channels), + requires_grad=True) + self.gamma2 = nn.Parameter(layer_scale * torch.ones(channels), + requires_grad=True) + self.res_post_norm = res_post_norm + if res_post_norm: + self.res_post_norm1 = build_norm_layer(channels, 'LN') + self.res_post_norm2 = build_norm_layer(channels, 'LN') + def forward(self, x, shape): + + def _inner_forward(x, shape): + if not self.layer_scale: + if self.post_norm: + x = x + self.drop_path(self.norm1(self.dcn(x, shape))) + x = x + self.drop_path(self.norm2(self.mlp(x, shape))) + elif self.res_post_norm: # for InternImage-H/G + x = x + self.drop_path(self.res_post_norm1(self.dcn(self.norm1(x), shape))) + x = x + self.drop_path(self.res_post_norm2(self.mlp(self.norm2(x), shape))) + + else: + x = x + self.drop_path(self.dcn(self.norm1(x), shape,)) + x = x + self.drop_path(self.mlp(self.norm2(x), shape)) + return x + if self.post_norm: + x = x + self.drop_path(self.gamma1 * self.norm1(self.dcn(x, shape))) + x = x + self.drop_path(self.gamma2 * self.norm2(self.mlp(x, shape))) + else: + x = x + self.drop_path(self.gamma1 * self.dcn(self.norm1(x), shape)) + x = x + self.drop_path(self.gamma2 * self.mlp(self.norm2(x), shape)) + return x + + if self.with_cp and x.requires_grad: + x = checkpoint.checkpoint(_inner_forward, x, shape) + else: + x = _inner_forward(x, shape) + + return x + + +class InternImageBlock(nn.Module): + r""" Block of InternImage + Args: + core_op (nn.Module): core operation of InternImage + channels (int): number of input channels + depths (list): Depth of each block. + groups (list): Groups of each block. + mlp_ratio (float): ratio of mlp hidden features to input channels + drop (float): dropout rate + drop_path (float): drop path rate + act_layer (str): activation layer + norm_layer (str): normalization layer + post_norm (bool): whether to use post normalization + layer_scale (float): layer scale + offset_scale (float): offset scale + with_cp (bool): whether to use checkpoint + """ + + def __init__(self, + core_op, + channels, + depth, + groups, + downsample=True, + downsample_layer=DownsampleLayer, + mlp_ratio=4., + drop=0., + drop_path=0., + act_layer='GELU', + norm_layer='LN', + post_norm=False, + offset_scale=0.5, + layer_scale=None, + with_cp=False, + dcn_output_bias=False, + mlp_fc2_bias=False, + dw_kernel_size=None, # for InternImage-H/G + post_norm_block_ids=None, # for InternImage-H/G + res_post_norm=False, # for InternImage-H/G + center_feature_scale=False): # for InternImage-H/G + super().__init__() + self.channels = channels + self.depth = depth + self.post_norm = post_norm + self.center_feature_scale = center_feature_scale + + self.blocks = nn.ModuleList([ + InternImageLayer( + core_op=core_op, + channels=channels, + groups=groups, + mlp_ratio=mlp_ratio, + drop=drop, + drop_path=drop_path[i] if isinstance( + drop_path, list) else drop_path, + act_layer=act_layer, + norm_layer=norm_layer, + post_norm=post_norm, + layer_scale=layer_scale, + offset_scale=offset_scale, + with_cp=with_cp, + dcn_output_bias=dcn_output_bias, + mlp_fc2_bias=mlp_fc2_bias, + dw_kernel_size=dw_kernel_size, # for InternImage-H/G + res_post_norm=res_post_norm, # for InternImage-H/G + center_feature_scale=center_feature_scale # for InternImage-H/G + ) for i in range(depth) + ]) + if not self.post_norm or center_feature_scale: + self.norm = build_norm_layer(channels, 'LN') + self.post_norm_block_ids = post_norm_block_ids + if post_norm_block_ids is not None: # for InternImage-H/G + self.post_norms = nn.ModuleList( + [build_norm_layer(channels, 'LN', eps=1e-6) for _ in post_norm_block_ids] + ) + self.downsample = downsample_layer( + channels=channels, norm_layer=norm_layer) if downsample else None + + + def forward(self, x, return_wo_downsample=False, shape=None): + for i, blk in enumerate(self.blocks): + x = blk(x, shape=shape) + if (self.post_norm_block_ids is not None) and (i in self.post_norm_block_ids): + index = self.post_norm_block_ids.index(i) + x = self.post_norms[index](x) # for InternImage-H/G + if not self.post_norm or self.center_feature_scale: + x = self.norm(x) + if return_wo_downsample: + x_ = x.clone() + if self.downsample is not None: + x, shape = self.downsample(x, shape=shape) + + if return_wo_downsample: + return x, x_, shape + return x, shape + + +class FlashInternImage(nn.Module): + r""" FlashInternImage + A PyTorch impl based on : + `InternImage: Exploring Large-Scale Vision Foundation Models with Deformable Convolutions` - + https://arxiv.org/pdf/2103.14030 + 'DCNv4': TODO: add arxiv + Args: + core_op (str): Core operator. Default: 'DCNv4' + channels (int): Number of the first stage. Default: 64 + depths (list): Depth of each block. Default: [3, 4, 18, 5] + groups (list): Groups of each block. Default: [3, 6, 12, 24] + num_classes (int): Number of classes. Default: 1000 + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. Default: 4. + drop_rate (float): Probability of an element to be zeroed. Default: 0. + drop_path_rate (float): Stochastic depth rate. Default: 0.2 + act_layer (str): Activation layer. Default: 'GELU' + norm_layer (str): Normalization layer. Default: 'LN' + layer_scale (bool): Whether to use layer scale. Default: False + cls_scale (bool): Whether to use class scale. Default: False + with_cp (bool): Use checkpoint or not. Using checkpoint will save some + dw_kernel_size (int): Size of the dwconv. Default: None + use_clip_projector (bool): Whether to use clip projector. Default: False + level2_post_norm (bool): Whether to use level2 post norm. Default: False + level2_post_norm_block_ids (list): Indexes of post norm blocks. Default: None + res_post_norm (bool): Whether to use res post norm. Default: False + center_feature_scale (bool): Whether to use center feature scale. Default: False + """ + + def __init__(self, + core_op='DCNv4', + channels=64, + depths=[3, 4, 18, 5], + groups=[3, 6, 12, 24], + num_classes=1000, + mlp_ratio=4., + drop_rate=0., + drop_path_rate=0.2, + drop_path_type='linear', + act_layer='GELU', + norm_layer='LN', + layer_scale=None, + offset_scale=0.5, + post_norm=False, + cls_scale=1.5, + with_cp=False, + mlp_fc2_bias=False, + dcn_output_bias=False, + dw_kernel_size=None, + use_clip_projector=False, # for InternImage-H/G + level2_post_norm=False, # for InternImage-H/G + level2_post_norm_block_ids=None, # for InternImage-H/G + res_post_norm=False, # for InternImage-H/G + center_feature_scale=False, # for InternImage-H/G + **kwargs): + super().__init__() + self.core_op = core_op + self.num_classes = num_classes + self.num_levels = len(depths) + self.depths = depths + self.channels = channels + self.num_features = int(channels * 2**(self.num_levels - 1)) + self.post_norm = post_norm + self.mlp_ratio = mlp_ratio + self.use_clip_projector = use_clip_projector + + self.level2_post_norm_block_ids = level2_post_norm_block_ids + print(f'using core type: {core_op}') + print(f'using activation layer: {act_layer}') + print(f'using main norm layer: {norm_layer}') + print(f'using dpr: {drop_path_type}, {drop_path_rate}') + print(f"level2_post_norm: {level2_post_norm}") + print(f"level2_post_norm_block_ids: {level2_post_norm_block_ids}") + print(f"res_post_norm: {res_post_norm}") + + in_chans = 3 + self.patch_embed = StemLayer(in_chans=in_chans, + out_chans=channels, + act_layer=act_layer, + norm_layer=norm_layer) + self.pos_drop = nn.Dropout(p=drop_rate) + + dpr = [ + x.item() for x in torch.linspace(0, drop_path_rate, sum(depths)) + ] + if drop_path_type == 'uniform': + for i in range(len(dpr)): + dpr[i] = drop_path_rate + + self.levels = nn.ModuleList() + for i in range(self.num_levels): + post_norm_block_ids = level2_post_norm_block_ids if level2_post_norm and ( + i == 2) else None # for InternImage-H/G + + level = InternImageBlock( + core_op=getattr(DCNv4, core_op), + channels=int(channels * 2**i), + depth=depths[i], + groups=groups[i], + mlp_ratio=self.mlp_ratio, + drop=drop_rate, + drop_path=dpr[sum(depths[:i]):sum(depths[:i + 1])], + act_layer=act_layer, + norm_layer=norm_layer, + post_norm=post_norm, + downsample=(i < self.num_levels - 1), + downsample_layer = DownsampleLayer, + layer_scale=layer_scale, + offset_scale=offset_scale, + with_cp=with_cp, + mlp_fc2_bias=mlp_fc2_bias, + dcn_output_bias=dcn_output_bias, + dw_kernel_size=dw_kernel_size, # for InternImage-H/G + post_norm_block_ids=post_norm_block_ids, # for InternImage-H/G + res_post_norm=res_post_norm, # for InternImage-H/G + center_feature_scale=center_feature_scale # for InternImage-H/G + ) + self.levels.append(level) + + if not use_clip_projector: # for InternImage-T/S/B/L/XL + self.conv_head = nn.Sequential( + nn.Conv2d(self.num_features, + int(self.num_features * cls_scale), + kernel_size=1, + bias=False), + build_norm_layer(int(self.num_features * cls_scale), 'BN', + 'channels_first', 'channels_first'), + build_act_layer(act_layer)) + self.head = nn.Linear(int(self.num_features * cls_scale), num_classes) \ + if num_classes > 0 else nn.Identity() + else: # for InternImage-H/G + pretrain_embed_dim, _stride, attnpool_num_heads, clip_embed_dim = 1024, 2, 16, 768 + self.dcnv3_head_x4 = nn.Sequential( + nn.Conv2d(in_channels=self.num_features, + out_channels=pretrain_embed_dim * (_stride ** 2), + kernel_size=1), nn.PixelShuffle(_stride)) + self.dcnv3_head_x3 = nn.Conv2d(in_channels=self.num_features // 2, + out_channels=pretrain_embed_dim, + kernel_size=1) + self.clip_projector = AttentionPoolingBlock( + dim=pretrain_embed_dim, + num_heads=attnpool_num_heads, + qkv_bias=True, + qk_scale=None, + drop=0., + attn_drop=0., + norm_layer=norm_layer, + out_dim=clip_embed_dim) + self.fc_norm = build_norm_layer(clip_embed_dim, norm_layer, eps=1e-6) + self.head = nn.Linear( + clip_embed_dim, num_classes) if num_classes > 0 else nn.Identity() + + self.avgpool = nn.AdaptiveAvgPool2d((1, 1)) + self.num_layers = len(depths) + self.apply(self._init_weights) + self.apply(self._init_deform_weights) + + def _init_weights(self, m): + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + + def _init_deform_weights(self, m): + if isinstance(m, getattr(DCNv4, self.core_op)): + m._reset_parameters() + + @torch.jit.ignore + def lr_decay_keywards(self, decay_ratio=0.87): + lr_ratios = {} + + # blocks + idx = 0 + for i in range(4): + layer_num = 3 - i # 3 2 1 0 + for j in range(self.depths[layer_num]): + block_num = self.depths[layer_num] - j - 1 + tag = 'levels.{}.blocks.{}.'.format(layer_num, block_num) + decay = 1.0 * (decay_ratio**idx) + lr_ratios[tag] = decay + idx += 1 + # patch_embed (before stage-1) + lr_ratios["patch_embed"] = lr_ratios['levels.0.blocks.0.'] + # levels.0.downsample (between stage-1 and stage-2) + lr_ratios["levels.0.downsample"] = lr_ratios['levels.1.blocks.0.'] + lr_ratios["levels.0.norm"] = lr_ratios['levels.1.blocks.0.'] + # levels.1.downsample (between stage-2 and stage-3) + lr_ratios["levels.1.downsample"] = lr_ratios['levels.2.blocks.0.'] + lr_ratios["levels.1.norm"] = lr_ratios['levels.2.blocks.0.'] + # levels.2.downsample (between stage-3 and stage-4) + lr_ratios["levels.2.downsample"] = lr_ratios['levels.3.blocks.0.'] + lr_ratios["levels.2.norm"] = lr_ratios['levels.3.blocks.0.'] + return lr_ratios + + def forward_features(self, x): + x = self.patch_embed(x) + N, H, W, C = x.shape + x = x.view(N, H*W, C) + + shape=(H, W) + seq_out = [] + for level_idx, level in enumerate(self.levels): + old_shape = shape + x, shape = level(x, shape=shape) + h, w = shape + x = x.view(N, h, w, -1) + x = self.conv_head(x.permute(0, 3, 1, 2)) + x = self.avgpool(x) + x = torch.flatten(x, 1) + return x + + def forward_features_seq_out(self, x): + x = self.patch_embed(x) + N, H, W, C = x.shape + x = x.view(N, H*W, C) + shape=(H, W) + seq_out = [] + for level_idx, level in enumerate(self.levels): + old_shape = shape + x, x_ , shape = level(x, return_wo_downsample=True, shape=shape) + h, w= old_shape + seq_out.append(x_.reshape(N, h, w, -1).permute(0, 3, 1, 2)) + return seq_out + + def forward_clip_projector(self, x): # for InternImage-H/G + xs = self.forward_features_seq_out(x) + x1, x2, x3, x4 = xs + + x1 = x1.permute(0, 3, 1, 2) # NHWC -> NCHW + x2 = x2.permute(0, 3, 1, 2) # NHWC -> NCHW + x3 = x3.permute(0, 3, 1, 2) # NHWC -> NCHW + x4 = x4.permute(0, 3, 1, 2) # NHWC -> NCHW + + x4 = self.dcnv3_head_x4(x4) + x = x4 + x3 = self.dcnv3_head_x3(x3) + x = x + x3 + + x = x.flatten(-2).transpose(1, 2).contiguous() + x = self.clip_projector(x) + x = self.fc_norm(x) + + return x + + def forward(self, x): + if self.use_clip_projector: # for InternImage-H/G + x = self.forward_clip_projector(x) + else: # for InternImage-T/S/B/L/XL + x = self.forward_features(x) + x = self.head(x) + return x + + + diff --git a/classification/models/intern_image.py b/classification/models/intern_image.py new file mode 100644 index 0000000..2a741dc --- /dev/null +++ b/classification/models/intern_image.py @@ -0,0 +1,759 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + + +import torch +import torch.nn as nn +import torch.utils.checkpoint as checkpoint +from timm.models.layers import trunc_normal_, DropPath + +import torch.nn.functional as F + + +class to_channels_first(nn.Module): + + def __init__(self): + super().__init__() + + def forward(self, x): + return x.permute(0, 3, 1, 2) + + +class to_channels_last(nn.Module): + + def __init__(self): + super().__init__() + + def forward(self, x): + return x.permute(0, 2, 3, 1) + + +def build_norm_layer(dim, + norm_layer, + in_format='channels_last', + out_format='channels_last', + eps=1e-6): + layers = [] + if norm_layer == 'BN': + if in_format == 'channels_last': + layers.append(to_channels_first()) + layers.append(nn.BatchNorm2d(dim)) + if out_format == 'channels_last': + layers.append(to_channels_last()) + elif norm_layer == 'LN': + if in_format == 'channels_first': + layers.append(to_channels_last()) + layers.append(nn.LayerNorm(dim, eps=eps)) + if out_format == 'channels_first': + layers.append(to_channels_first()) + else: + raise NotImplementedError( + f'build_norm_layer does not support {norm_layer}') + return nn.Sequential(*layers) + + +def build_act_layer(act_layer): + if act_layer == 'ReLU': + return nn.ReLU(inplace=True) + elif act_layer == 'SiLU': + return nn.SiLU(inplace=True) + elif act_layer == 'GELU': + return nn.GELU() + + raise NotImplementedError(f'build_act_layer does not support {act_layer}') + + +class CrossAttention(nn.Module): + r""" Cross Attention Module + Args: + dim (int): Number of input channels. + num_heads (int): Number of attention heads. Default: 8 + qkv_bias (bool, optional): If True, add a learnable bias to q, k, v. + Default: False. + qk_scale (float | None, optional): Override default qk scale of + head_dim ** -0.5 if set. Default: None. + attn_drop (float, optional): Dropout ratio of attention weight. + Default: 0.0 + proj_drop (float, optional): Dropout ratio of output. Default: 0.0 + attn_head_dim (int, optional): Dimension of attention head. + out_dim (int, optional): Dimension of output. + """ + + def __init__(self, + dim, + num_heads=8, + qkv_bias=False, + qk_scale=None, + attn_drop=0., + proj_drop=0., + attn_head_dim=None, + out_dim=None): + super().__init__() + if out_dim is None: + out_dim = dim + self.num_heads = num_heads + head_dim = dim // num_heads + if attn_head_dim is not None: + head_dim = attn_head_dim + all_head_dim = head_dim * self.num_heads + self.scale = qk_scale or head_dim ** -0.5 + assert all_head_dim == dim + + self.q = nn.Linear(dim, all_head_dim, bias=False) + self.k = nn.Linear(dim, all_head_dim, bias=False) + self.v = nn.Linear(dim, all_head_dim, bias=False) + + if qkv_bias: + self.q_bias = nn.Parameter(torch.zeros(all_head_dim)) + self.k_bias = nn.Parameter(torch.zeros(all_head_dim)) + self.v_bias = nn.Parameter(torch.zeros(all_head_dim)) + else: + self.q_bias = None + self.k_bias = None + self.v_bias = None + + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(all_head_dim, out_dim) + self.proj_drop = nn.Dropout(proj_drop) + + def forward(self, x, k=None, v=None): + B, N, C = x.shape + N_k = k.shape[1] + N_v = v.shape[1] + + q_bias, k_bias, v_bias = None, None, None + if self.q_bias is not None: + q_bias = self.q_bias + k_bias = self.k_bias + v_bias = self.v_bias + + q = F.linear(input=x, weight=self.q.weight, bias=q_bias) + q = q.reshape(B, N, 1, self.num_heads, + -1).permute(2, 0, 3, 1, + 4).squeeze(0) # (B, N_head, N_q, dim) + + k = F.linear(input=k, weight=self.k.weight, bias=k_bias) + k = k.reshape(B, N_k, 1, self.num_heads, -1).permute(2, 0, 3, 1, + 4).squeeze(0) + + v = F.linear(input=v, weight=self.v.weight, bias=v_bias) + v = v.reshape(B, N_v, 1, self.num_heads, -1).permute(2, 0, 3, 1, + 4).squeeze(0) + + q = q * self.scale + attn = (q @ k.transpose(-2, -1)) # (B, N_head, N_q, N_k) + + attn = attn.softmax(dim=-1) + attn = self.attn_drop(attn) + + x = (attn @ v).transpose(1, 2).reshape(B, N, -1) + x = self.proj(x) + x = self.proj_drop(x) + + return x + + +class AttentiveBlock(nn.Module): + r"""Attentive Block + Args: + dim (int): Number of input channels. + num_heads (int): Number of attention heads. Default: 8 + qkv_bias (bool, optional): If True, add a learnable bias to q, k, v. + Default: False. + qk_scale (float | None, optional): Override default qk scale of + head_dim ** -0.5 if set. Default: None. + drop (float, optional): Dropout rate. Default: 0.0. + attn_drop (float, optional): Attention dropout rate. Default: 0.0. + drop_path (float | tuple[float], optional): Stochastic depth rate. + Default: 0.0. + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm. + attn_head_dim (int, optional): Dimension of attention head. Default: None. + out_dim (int, optional): Dimension of output. Default: None. + """ + + def __init__(self, + dim, + num_heads, + qkv_bias=False, + qk_scale=None, + drop=0., + attn_drop=0., + drop_path=0., + norm_layer="LN", + attn_head_dim=None, + out_dim=None): + super().__init__() + + self.norm1_q = build_norm_layer(dim, norm_layer, eps=1e-6) + self.norm1_k = build_norm_layer(dim, norm_layer, eps=1e-6) + self.norm1_v = build_norm_layer(dim, norm_layer, eps=1e-6) + self.cross_dcn = CrossAttention(dim, + num_heads=num_heads, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + attn_drop=attn_drop, + proj_drop=drop, + attn_head_dim=attn_head_dim, + out_dim=out_dim) + + self.drop_path = DropPath( + drop_path) if drop_path > 0. else nn.Identity() + + def forward(self, + x_q, + x_kv, + pos_q, + pos_k, + bool_masked_pos, + rel_pos_bias=None): + x_q = self.norm1_q(x_q + pos_q) + x_k = self.norm1_k(x_kv + pos_k) + x_v = self.norm1_v(x_kv) + + x = self.cross_dcn(x_q, k=x_k, v=x_v) + + return x + + +class AttentionPoolingBlock(AttentiveBlock): + + def forward(self, x): + x_q = x.mean(1, keepdim=True) + x_kv = x + pos_q, pos_k = 0, 0 + x = super().forward(x_q, x_kv, pos_q, pos_k, + bool_masked_pos=None, + rel_pos_bias=None) + x = x.squeeze(1) + return x + + +class StemLayer(nn.Module): + r""" Stem layer of InternImage + Args: + in_chans (int): number of input channels + out_chans (int): number of output channels + act_layer (str): activation layer + norm_layer (str): normalization layer + """ + + def __init__(self, + in_chans=3, + out_chans=96, + act_layer='GELU', + norm_layer='BN'): + super().__init__() + self.conv1 = nn.Conv2d(in_chans, + out_chans // 2, + kernel_size=3, + stride=2, + padding=1) + self.norm1 = build_norm_layer(out_chans // 2, norm_layer, + 'channels_first', 'channels_first') + self.act = build_act_layer(act_layer) + self.conv2 = nn.Conv2d(out_chans // 2, + out_chans, + kernel_size=3, + stride=2, + padding=1) + self.norm2 = build_norm_layer(out_chans, norm_layer, 'channels_first', + 'channels_last') + + def forward(self, x): + x = self.conv1(x) + x = self.norm1(x) + x = self.act(x) + x = self.conv2(x) + x = self.norm2(x) + return x + + +class DownsampleLayer(nn.Module): + r""" Downsample layer of InternImage + Args: + channels (int): number of input channels + norm_layer (str): normalization layer + """ + + def __init__(self, channels, norm_layer='LN'): + super().__init__() + self.conv = nn.Conv2d(channels, + 2 * channels, + kernel_size=3, + stride=2, + padding=1, + bias=False) + self.norm = build_norm_layer(2 * channels, norm_layer, + 'channels_first', 'channels_last') + + def forward(self, x): + x = self.conv(x.permute(0, 3, 1, 2)) + x = self.norm(x) + return x + + +class MLPLayer(nn.Module): + r""" MLP layer of InternImage + Args: + in_features (int): number of input features + hidden_features (int): number of hidden features + out_features (int): number of output features + act_layer (str): activation layer + drop (float): dropout rate + """ + + def __init__(self, + in_features, + hidden_features=None, + out_features=None, + act_layer='GELU', + drop=0.): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = nn.Linear(in_features, hidden_features) + self.act = build_act_layer(act_layer) + self.fc2 = nn.Linear(hidden_features, out_features) + self.drop = nn.Dropout(drop) + + def forward(self, x): + x = self.fc1(x) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + return x + + +class InternImageLayer(nn.Module): + r""" Basic layer of InternImage + Args: + core_op (nn.Module): core operation of InternImage + channels (int): number of input channels + groups (list): Groups of each block. + mlp_ratio (float): ratio of mlp hidden features to input channels + drop (float): dropout rate + drop_path (float): drop path rate + act_layer (str): activation layer + norm_layer (str): normalization layer + post_norm (bool): whether to use post normalization + layer_scale (float): layer scale + offset_scale (float): offset scale + with_cp (bool): whether to use checkpoint + """ + + def __init__(self, + core_op, + channels, + groups, + mlp_ratio=4., + drop=0., + drop_path=0., + act_layer='GELU', + norm_layer='LN', + post_norm=False, + layer_scale=None, + offset_scale=1.0, + with_cp=False, + dw_kernel_size=None, # for InternImage-H/G + res_post_norm=False, # for InternImage-H/G + center_feature_scale=False): # for InternImage-H/G + super().__init__() + self.channels = channels + self.groups = groups + self.mlp_ratio = mlp_ratio + self.with_cp = with_cp + + self.norm1 = build_norm_layer(channels, 'LN') + self.post_norm = post_norm + self.dcn = core_op( + channels=channels, + kernel_size=3, + stride=1, + pad=1, + dilation=1, + group=groups, + offset_scale=offset_scale, + act_layer=act_layer, + norm_layer=norm_layer, + dw_kernel_size=dw_kernel_size, # for InternImage-H/G + center_feature_scale=center_feature_scale) # for InternImage-H/G + self.drop_path = DropPath(drop_path) if drop_path > 0. \ + else nn.Identity() + self.norm2 = build_norm_layer(channels, 'LN') + self.mlp = MLPLayer(in_features=channels, + hidden_features=int(channels * mlp_ratio), + act_layer=act_layer, + drop=drop) + self.layer_scale = layer_scale is not None + if self.layer_scale: + self.gamma1 = nn.Parameter(layer_scale * torch.ones(channels), + requires_grad=True) + self.gamma2 = nn.Parameter(layer_scale * torch.ones(channels), + requires_grad=True) + self.res_post_norm = res_post_norm + if res_post_norm: + self.res_post_norm1 = build_norm_layer(channels, 'LN') + self.res_post_norm2 = build_norm_layer(channels, 'LN') + + def forward(self, x): + + def _inner_forward(x): + if not self.layer_scale: + if self.post_norm: + x = x + self.drop_path(self.norm1(self.dcn(x))) + x = x + self.drop_path(self.norm2(self.mlp(x))) + elif self.res_post_norm: # for InternImage-H/G + x = x + self.drop_path(self.res_post_norm1(self.dcn(self.norm1(x)))) + x = x + self.drop_path(self.res_post_norm2(self.mlp(self.norm2(x)))) + else: + x = x + self.drop_path(self.dcn(self.norm1(x))) + x = x + self.drop_path(self.mlp(self.norm2(x))) + return x + if self.post_norm: + x = x + self.drop_path(self.gamma1 * self.norm1(self.dcn(x))) + x = x + self.drop_path(self.gamma2 * self.norm2(self.mlp(x))) + else: + x = x + self.drop_path(self.gamma1 * self.dcn(self.norm1(x))) + x = x + self.drop_path(self.gamma2 * self.mlp(self.norm2(x))) + return x + + if self.with_cp and x.requires_grad: + x = checkpoint.checkpoint(_inner_forward, x) + else: + x = _inner_forward(x) + return x + + +class InternImageBlock(nn.Module): + r""" Block of InternImage + Args: + core_op (nn.Module): core operation of InternImage + channels (int): number of input channels + depths (list): Depth of each block. + groups (list): Groups of each block. + mlp_ratio (float): ratio of mlp hidden features to input channels + drop (float): dropout rate + drop_path (float): drop path rate + act_layer (str): activation layer + norm_layer (str): normalization layer + post_norm (bool): whether to use post normalization + layer_scale (float): layer scale + offset_scale (float): offset scale + with_cp (bool): whether to use checkpoint + """ + + def __init__(self, + core_op, + channels, + depth, + groups, + downsample=True, + mlp_ratio=4., + drop=0., + drop_path=0., + act_layer='GELU', + norm_layer='LN', + post_norm=False, + offset_scale=1.0, + layer_scale=None, + with_cp=False, + dw_kernel_size=None, # for InternImage-H/G + post_norm_block_ids=None, # for InternImage-H/G + res_post_norm=False, # for InternImage-H/G + center_feature_scale=False): # for InternImage-H/G + super().__init__() + self.channels = channels + self.depth = depth + self.post_norm = post_norm + self.center_feature_scale = center_feature_scale + + self.blocks = nn.ModuleList([ + InternImageLayer( + core_op=core_op, + channels=channels, + groups=groups, + mlp_ratio=mlp_ratio, + drop=drop, + drop_path=drop_path[i] if isinstance( + drop_path, list) else drop_path, + act_layer=act_layer, + norm_layer=norm_layer, + post_norm=post_norm, + layer_scale=layer_scale, + offset_scale=offset_scale, + with_cp=with_cp, + dw_kernel_size=dw_kernel_size, # for InternImage-H/G + res_post_norm=res_post_norm, # for InternImage-H/G + center_feature_scale=center_feature_scale # for InternImage-H/G + ) for i in range(depth) + ]) + if not self.post_norm or center_feature_scale: + self.norm = build_norm_layer(channels, 'LN') + self.post_norm_block_ids = post_norm_block_ids + if post_norm_block_ids is not None: # for InternImage-H/G + self.post_norms = nn.ModuleList( + [build_norm_layer(channels, 'LN', eps=1e-6) for _ in post_norm_block_ids] + ) + self.downsample = DownsampleLayer( + channels=channels, norm_layer=norm_layer) if downsample else None + + def forward(self, x, return_wo_downsample=False): + for i, blk in enumerate(self.blocks): + x = blk(x) + if (self.post_norm_block_ids is not None) and (i in self.post_norm_block_ids): + index = self.post_norm_block_ids.index(i) + x = self.post_norms[index](x) # for InternImage-H/G + if not self.post_norm or self.center_feature_scale: + x = self.norm(x) + if return_wo_downsample: + x_ = x + if self.downsample is not None: + x = self.downsample(x) + + if return_wo_downsample: + return x, x_ + return x + + +class InternImage(nn.Module): + r""" InternImage + A PyTorch impl of : `InternImage: Exploring Large-Scale Vision Foundation Models with Deformable Convolutions` - + https://arxiv.org/pdf/2103.14030 + Args: + core_op (str): Core operator. Default: 'DCNv3' + channels (int): Number of the first stage. Default: 64 + depths (list): Depth of each block. Default: [3, 4, 18, 5] + groups (list): Groups of each block. Default: [3, 6, 12, 24] + num_classes (int): Number of classes. Default: 1000 + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. Default: 4. + drop_rate (float): Probability of an element to be zeroed. Default: 0. + drop_path_rate (float): Stochastic depth rate. Default: 0. + act_layer (str): Activation layer. Default: 'GELU' + norm_layer (str): Normalization layer. Default: 'LN' + layer_scale (bool): Whether to use layer scale. Default: False + cls_scale (bool): Whether to use class scale. Default: False + with_cp (bool): Use checkpoint or not. Using checkpoint will save some + dw_kernel_size (int): Size of the dwconv. Default: None + use_clip_projector (bool): Whether to use clip projector. Default: False + level2_post_norm (bool): Whether to use level2 post norm. Default: False + level2_post_norm_block_ids (list): Indexes of post norm blocks. Default: None + res_post_norm (bool): Whether to use res post norm. Default: False + center_feature_scale (bool): Whether to use center feature scale. Default: False + """ + + def __init__(self, + core_op='DCNv3', + channels=64, + depths=[3, 4, 18, 5], + groups=[3, 6, 12, 24], + num_classes=1000, + mlp_ratio=4., + drop_rate=0., + drop_path_rate=0.2, + drop_path_type='linear', + act_layer='GELU', + norm_layer='LN', + layer_scale=None, + offset_scale=1.0, + post_norm=False, + cls_scale=1.5, + with_cp=False, + dw_kernel_size=None, # for InternImage-H/G + use_clip_projector=False, # for InternImage-H/G + level2_post_norm=False, # for InternImage-H/G + level2_post_norm_block_ids=None, # for InternImage-H/G + res_post_norm=False, # for InternImage-H/G + center_feature_scale=False, # for InternImage-H/G + **kwargs): + super().__init__() + self.core_op = core_op + self.num_classes = num_classes + self.num_levels = len(depths) + self.depths = depths + self.channels = channels + self.num_features = int(channels * 2**(self.num_levels - 1)) + self.post_norm = post_norm + self.mlp_ratio = mlp_ratio + self.use_clip_projector = use_clip_projector + self.level2_post_norm_block_ids = level2_post_norm_block_ids + print(f'using core type: {core_op}') + print(f'using activation layer: {act_layer}') + print(f'using main norm layer: {norm_layer}') + print(f'using dpr: {drop_path_type}, {drop_path_rate}') + print(f"level2_post_norm: {level2_post_norm}") + print(f"level2_post_norm_block_ids: {level2_post_norm_block_ids}") + print(f"res_post_norm: {res_post_norm}") + + in_chans = 3 + self.patch_embed = StemLayer(in_chans=in_chans, + out_chans=channels, + act_layer=act_layer, + norm_layer=norm_layer) + self.pos_drop = nn.Dropout(p=drop_rate) + + dpr = [ + x.item() for x in torch.linspace(0, drop_path_rate, sum(depths)) + ] + if drop_path_type == 'uniform': + for i in range(len(dpr)): + dpr[i] = drop_path_rate + + self.levels = nn.ModuleList() + for i in range(self.num_levels): + post_norm_block_ids = level2_post_norm_block_ids if level2_post_norm and ( + i == 2) else None # for InternImage-H/G + from ops_offset import modules as opsm + level = InternImageBlock( + core_op=getattr(opsm, core_op), + channels=int(channels * 2**i), + depth=depths[i], + groups=groups[i], + mlp_ratio=self.mlp_ratio, + drop=drop_rate, + drop_path=dpr[sum(depths[:i]):sum(depths[:i + 1])], + act_layer=act_layer, + norm_layer=norm_layer, + post_norm=post_norm, + downsample=(i < self.num_levels - 1), + layer_scale=layer_scale, + offset_scale=offset_scale, + with_cp=with_cp, + dw_kernel_size=dw_kernel_size, # for InternImage-H/G + post_norm_block_ids=post_norm_block_ids, # for InternImage-H/G + res_post_norm=res_post_norm, # for InternImage-H/G + center_feature_scale=center_feature_scale # for InternImage-H/G + ) + self.levels.append(level) + + if not use_clip_projector: # for InternImage-T/S/B/L/XL + self.conv_head = nn.Sequential( + nn.Conv2d(self.num_features, + int(self.num_features * cls_scale), + kernel_size=1, + bias=False), + build_norm_layer(int(self.num_features * cls_scale), 'BN', + 'channels_first', 'channels_first'), + build_act_layer(act_layer)) + self.head = nn.Linear(int(self.num_features * cls_scale), num_classes) \ + if num_classes > 0 else nn.Identity() + else: # for InternImage-H/G + pretrain_embed_dim, _stride, attnpool_num_heads, clip_embed_dim = 1024, 2, 16, 768 + self.dcnv3_head_x4 = nn.Sequential( + nn.Conv2d(in_channels=self.num_features, + out_channels=pretrain_embed_dim * (_stride ** 2), + kernel_size=1), nn.PixelShuffle(_stride)) + self.dcnv3_head_x3 = nn.Conv2d(in_channels=self.num_features // 2, + out_channels=pretrain_embed_dim, + kernel_size=1) + self.clip_projector = AttentionPoolingBlock( + dim=pretrain_embed_dim, + num_heads=attnpool_num_heads, + qkv_bias=True, + qk_scale=None, + drop=0., + attn_drop=0., + norm_layer=norm_layer, + out_dim=clip_embed_dim) + self.fc_norm = build_norm_layer(clip_embed_dim, norm_layer, eps=1e-6) + self.head = nn.Linear( + clip_embed_dim, num_classes) if num_classes > 0 else nn.Identity() + + self.avgpool = nn.AdaptiveAvgPool2d((1, 1)) + self.num_layers = len(depths) + self.apply(self._init_weights) + self.apply(self._init_deform_weights) + + def _init_weights(self, m): + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + + def _init_deform_weights(self, m): + from ops_offset import modules as opsm + if isinstance(m, getattr(opsm, self.core_op)): + m._reset_parameters() + + @torch.jit.ignore + def lr_decay_keywards(self, decay_ratio=0.87): + lr_ratios = {} + + # blocks + idx = 0 + for i in range(4): + layer_num = 3 - i # 3 2 1 0 + for j in range(self.depths[layer_num]): + block_num = self.depths[layer_num] - j - 1 + tag = 'levels.{}.blocks.{}.'.format(layer_num, block_num) + decay = 1.0 * (decay_ratio**idx) + lr_ratios[tag] = decay + idx += 1 + # patch_embed (before stage-1) + lr_ratios["patch_embed"] = lr_ratios['levels.0.blocks.0.'] + # levels.0.downsample (between stage-1 and stage-2) + lr_ratios["levels.0.downsample"] = lr_ratios['levels.1.blocks.0.'] + lr_ratios["levels.0.norm"] = lr_ratios['levels.1.blocks.0.'] + # levels.1.downsample (between stage-2 and stage-3) + lr_ratios["levels.1.downsample"] = lr_ratios['levels.2.blocks.0.'] + lr_ratios["levels.1.norm"] = lr_ratios['levels.2.blocks.0.'] + # levels.2.downsample (between stage-3 and stage-4) + lr_ratios["levels.2.downsample"] = lr_ratios['levels.3.blocks.0.'] + lr_ratios["levels.2.norm"] = lr_ratios['levels.3.blocks.0.'] + return lr_ratios + + def forward_features(self, x): + x = self.patch_embed(x) + x = self.pos_drop(x) + + for level in self.levels: + x = level(x) + + x = self.conv_head(x.permute(0, 3, 1, 2)) + x = self.avgpool(x) + x = torch.flatten(x, 1) + return x + + def forward_features_seq_out(self, x): + x = self.patch_embed(x) + x = self.pos_drop(x) + + seq_out = [] + for level in self.levels: + x, x_ = level(x, return_wo_downsample=True) + seq_out.append(x_) + return seq_out + + def forward_clip_projector(self, x): # for InternImage-H/G + xs = self.forward_features_seq_out(x) + x1, x2, x3, x4 = xs + + x1 = x1.permute(0, 3, 1, 2) # NHWC -> NCHW + x2 = x2.permute(0, 3, 1, 2) # NHWC -> NCHW + x3 = x3.permute(0, 3, 1, 2) # NHWC -> NCHW + x4 = x4.permute(0, 3, 1, 2) # NHWC -> NCHW + + x4 = self.dcnv3_head_x4(x4) + x = x4 + x3 = self.dcnv3_head_x3(x3) + x = x + x3 + + x = x.flatten(-2).transpose(1, 2).contiguous() + x = self.clip_projector(x) + x = self.fc_norm(x) + + return x + + def forward(self, x): + if self.use_clip_projector: # for InternImage-H/G + x = self.forward_clip_projector(x) + else: # for InternImage-T/S/B/L/XL + x = self.forward_features(x) + x = self.head(x) + return x diff --git a/classification/ops_dcnv3/functions/__init__.py b/classification/ops_dcnv3/functions/__init__.py new file mode 100644 index 0000000..4ff82f3 --- /dev/null +++ b/classification/ops_dcnv3/functions/__init__.py @@ -0,0 +1,7 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + +from .dcnv3_func import DCNv3Function, dcnv3_core_pytorch diff --git a/classification/ops_dcnv3/functions/dcnv3_func.py b/classification/ops_dcnv3/functions/dcnv3_func.py new file mode 100644 index 0000000..1a9b3c0 --- /dev/null +++ b/classification/ops_dcnv3/functions/dcnv3_func.py @@ -0,0 +1,220 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + +from __future__ import absolute_import +from __future__ import print_function +from __future__ import division + +import torch +import torch.nn.functional as F +from torch.autograd import Function +from torch.autograd.function import once_differentiable +from torch.cuda.amp import custom_bwd, custom_fwd +import DCNv3 + +import pkg_resources +dcn_version = float(pkg_resources.get_distribution('DCNv3').version) + + +class DCNv3Function(Function): + @staticmethod + @custom_fwd + def forward( + ctx, input, offset, mask, + kernel_h, kernel_w, stride_h, stride_w, + pad_h, pad_w, dilation_h, dilation_w, + group, group_channels, offset_scale, im2col_step, remove_center): + ctx.kernel_h = kernel_h + ctx.kernel_w = kernel_w + ctx.stride_h = stride_h + ctx.stride_w = stride_w + ctx.pad_h = pad_h + ctx.pad_w = pad_w + ctx.dilation_h = dilation_h + ctx.dilation_w = dilation_w + ctx.group = group + ctx.group_channels = group_channels + ctx.offset_scale = offset_scale + ctx.im2col_step = im2col_step + ctx.remove_center = remove_center + + args = [ + input, offset, mask, kernel_h, + kernel_w, stride_h, stride_w, pad_h, + pad_w, dilation_h, dilation_w, group, + group_channels, offset_scale, ctx.im2col_step + ] + if remove_center or dcn_version > 1.0: + args.append(remove_center) + + output = DCNv3.dcnv3_forward(*args) + ctx.save_for_backward(input, offset, mask) + + return output + + @staticmethod + @once_differentiable + @custom_bwd + def backward(ctx, grad_output): + input, offset, mask = ctx.saved_tensors + + args = [ + input, offset, mask, ctx.kernel_h, + ctx.kernel_w, ctx.stride_h, ctx.stride_w, ctx.pad_h, + ctx.pad_w, ctx.dilation_h, ctx.dilation_w, ctx.group, + ctx.group_channels, ctx.offset_scale, grad_output.contiguous(), ctx.im2col_step + ] + if ctx.remove_center or dcn_version > 1.0: + args.append(ctx.remove_center) + + grad_input, grad_offset, grad_mask = \ + DCNv3.dcnv3_backward(*args) + + return grad_input, grad_offset, grad_mask, \ + None, None, None, None, None, None, None, None, None, None, None, None, None + + @staticmethod + def symbolic(g, input, offset, mask, kernel_h, kernel_w, stride_h, + stride_w, pad_h, pad_w, dilation_h, dilation_w, group, + group_channels, offset_scale, im2col_step, remove_center): + """Symbolic function for mmdeploy::DCNv3. + + Returns: + DCNv3 op for onnx. + """ + return g.op( + 'mmdeploy::TRTDCNv3', + input, + offset, + mask, + kernel_h_i=int(kernel_h), + kernel_w_i=int(kernel_w), + stride_h_i=int(stride_h), + stride_w_i=int(stride_w), + pad_h_i=int(pad_h), + pad_w_i=int(pad_w), + dilation_h_i=int(dilation_h), + dilation_w_i=int(dilation_w), + group_i=int(group), + group_channels_i=int(group_channels), + offset_scale_f=float(offset_scale), + im2col_step_i=int(im2col_step), + remove_center=int(remove_center), + ) + + +def _get_reference_points(spatial_shapes, device, kernel_h, kernel_w, dilation_h, dilation_w, pad_h=0, pad_w=0, stride_h=1, stride_w=1): + _, H_, W_, _ = spatial_shapes + H_out = (H_ - (dilation_h * (kernel_h - 1) + 1)) // stride_h + 1 + W_out = (W_ - (dilation_w * (kernel_w - 1) + 1)) // stride_w + 1 + + ref_y, ref_x = torch.meshgrid( + torch.linspace( + # pad_h + 0.5, + # H_ - pad_h - 0.5, + (dilation_h * (kernel_h - 1)) // 2 + 0.5, + (dilation_h * (kernel_h - 1)) // 2 + 0.5 + (H_out - 1) * stride_h, + H_out, + dtype=torch.float32, + device=device), + torch.linspace( + # pad_w + 0.5, + # W_ - pad_w - 0.5, + (dilation_w * (kernel_w - 1)) // 2 + 0.5, + (dilation_w * (kernel_w - 1)) // 2 + 0.5 + (W_out - 1) * stride_w, + W_out, + dtype=torch.float32, + device=device)) + ref_y = ref_y.reshape(-1)[None] / H_ + ref_x = ref_x.reshape(-1)[None] / W_ + + ref = torch.stack((ref_x, ref_y), -1).reshape( + 1, H_out, W_out, 1, 2) + + return ref + + +def _generate_dilation_grids(spatial_shapes, kernel_h, kernel_w, dilation_h, dilation_w, group, device): + _, H_, W_, _ = spatial_shapes + points_list = [] + x, y = torch.meshgrid( + torch.linspace( + -((dilation_w * (kernel_w - 1)) // 2), + -((dilation_w * (kernel_w - 1)) // 2) + (kernel_w - 1) * dilation_w, + kernel_w, + dtype=torch.float32, + device=device), + torch.linspace( + -((dilation_h * (kernel_h - 1)) // 2), + -((dilation_h * (kernel_h - 1)) // 2) + (kernel_h - 1) * dilation_h, + kernel_h, + dtype=torch.float32, + device=device)) + + points_list.extend([x / W_, y / H_]) + grid = torch.stack(points_list, -1).reshape(-1, 1, 2).\ + repeat(1, group, 1).permute(1, 0, 2) + grid = grid.reshape(1, 1, 1, group * kernel_h * kernel_w, 2) + + return grid + + +def remove_center_sampling_locations(sampling_locations, kernel_w, kernel_h): + idx = list(range(sampling_locations.shape[-2])) + C = (kernel_w * kernel_h - 1)//2 + idx = [i for i in idx if i != C and (i-C) % (C*2+1) != 0] + sampling_locations = sampling_locations[:,:,:,idx, :] + return sampling_locations + +def dcnv3_core_pytorch( + input, offset, mask, kernel_h, + kernel_w, stride_h, stride_w, pad_h, + pad_w, dilation_h, dilation_w, group, + group_channels, offset_scale, remove_center): + # for debug and test only, + # need to use cuda version instead + + if remove_center and (kernel_h % 2 == 0 or kernel_w % 2 == 0 or kernel_w != kernel_h): + raise ValueError('remove_center is only compatible with square odd kernel size.') + + input = F.pad( + input, + [0, 0, pad_h, pad_h, pad_w, pad_w]) + N_, H_in, W_in, _ = input.shape + _, H_out, W_out, _ = offset.shape + + ref = _get_reference_points( + input.shape, input.device, kernel_h, kernel_w, dilation_h, dilation_w, pad_h, pad_w, stride_h, stride_w) + grid = _generate_dilation_grids( + input.shape, kernel_h, kernel_w, dilation_h, dilation_w, group, input.device) + spatial_norm = torch.tensor([W_in, H_in]).reshape(1, 1, 1, 2).\ + repeat(1, 1, 1, group*(kernel_h*kernel_w-remove_center)).to(input.device) + + sampling_locations = (ref + grid * offset_scale).repeat(N_, 1, 1, 1, 1) + if remove_center: + sampling_locations = remove_center_sampling_locations(sampling_locations, kernel_w=kernel_w, kernel_h=kernel_h) + sampling_locations = sampling_locations.flatten(3, 4) + sampling_locations = sampling_locations + offset * offset_scale / spatial_norm + + P_ = kernel_h * kernel_w - remove_center + sampling_grids = 2 * sampling_locations - 1 + # N_, H_in, W_in, group*group_channels -> N_, H_in*W_in, group*group_channels -> N_, group*group_channels, H_in*W_in -> N_*group, group_channels, H_in, W_in + input_ = input.view(N_, H_in*W_in, group*group_channels).transpose(1, 2).\ + reshape(N_*group, group_channels, H_in, W_in) + # N_, H_out, W_out, group*P_*2 -> N_, H_out*W_out, group, P_, 2 -> N_, group, H_out*W_out, P_, 2 -> N_*group, H_out*W_out, P_, 2 + sampling_grid_ = sampling_grids.view(N_, H_out*W_out, group, P_, 2).transpose(1, 2).\ + flatten(0, 1) + # N_*group, group_channels, H_out*W_out, P_ + sampling_input_ = F.grid_sample( + input_, sampling_grid_, mode='bilinear', padding_mode='zeros', align_corners=False) + + # (N_, H_out, W_out, group*P_) -> N_, H_out*W_out, group, P_ -> (N_, group, H_out*W_out, P_) -> (N_*group, 1, H_out*W_out, P_) + mask = mask.view(N_, H_out*W_out, group, P_).transpose(1, 2).\ + reshape(N_*group, 1, H_out*W_out, P_) + output = (sampling_input_ * mask).sum(-1).view(N_, + group*group_channels, H_out*W_out) + + return output.transpose(1, 2).reshape(N_, H_out, W_out, -1).contiguous() diff --git a/classification/ops_dcnv3/make.sh b/classification/ops_dcnv3/make.sh new file mode 100755 index 0000000..0240533 --- /dev/null +++ b/classification/ops_dcnv3/make.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + +python setup.py build install diff --git a/classification/ops_dcnv3/modules/__init__.py b/classification/ops_dcnv3/modules/__init__.py new file mode 100644 index 0000000..4bba45c --- /dev/null +++ b/classification/ops_dcnv3/modules/__init__.py @@ -0,0 +1,7 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + +from .dcnv3 import DCNv3, DCNv3_pytorch \ No newline at end of file diff --git a/classification/ops_dcnv3/modules/dcnv3.py b/classification/ops_dcnv3/modules/dcnv3.py new file mode 100644 index 0000000..3341db7 --- /dev/null +++ b/classification/ops_dcnv3/modules/dcnv3.py @@ -0,0 +1,357 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + + +from __future__ import absolute_import +from __future__ import print_function +from __future__ import division + +import warnings +import torch +from torch import nn +import torch.nn.functional as F +from torch.nn.init import xavier_uniform_, constant_ +from ..functions import DCNv3Function, dcnv3_core_pytorch + + +class to_channels_first(nn.Module): + + def __init__(self): + super().__init__() + + def forward(self, x): + return x.permute(0, 3, 1, 2) + + +class to_channels_last(nn.Module): + + def __init__(self): + super().__init__() + + def forward(self, x): + return x.permute(0, 2, 3, 1) + + +def build_norm_layer(dim, + norm_layer, + in_format='channels_last', + out_format='channels_last', + eps=1e-6): + layers = [] + if norm_layer == 'BN': + if in_format == 'channels_last': + layers.append(to_channels_first()) + layers.append(nn.BatchNorm2d(dim)) + if out_format == 'channels_last': + layers.append(to_channels_last()) + elif norm_layer == 'LN': + if in_format == 'channels_first': + layers.append(to_channels_last()) + layers.append(nn.LayerNorm(dim, eps=eps)) + if out_format == 'channels_first': + layers.append(to_channels_first()) + else: + raise NotImplementedError( + f'build_norm_layer does not support {norm_layer}') + return nn.Sequential(*layers) + + +def build_act_layer(act_layer): + if act_layer == 'ReLU': + return nn.ReLU(inplace=True) + elif act_layer == 'SiLU': + return nn.SiLU(inplace=True) + elif act_layer == 'GELU': + return nn.GELU() + + raise NotImplementedError(f'build_act_layer does not support {act_layer}') + + +def _is_power_of_2(n): + if (not isinstance(n, int)) or (n < 0): + raise ValueError( + "invalid input for _is_power_of_2: {} (type: {})".format(n, type(n))) + + return (n & (n - 1) == 0) and n != 0 + + +class CenterFeatureScaleModule(nn.Module): + def forward(self, + query, + center_feature_scale_proj_weight, + center_feature_scale_proj_bias): + center_feature_scale = F.linear(query, + weight=center_feature_scale_proj_weight, + bias=center_feature_scale_proj_bias).sigmoid() + return center_feature_scale + + +class DCNv3_pytorch(nn.Module): + def __init__( + self, + channels=64, + kernel_size=3, + dw_kernel_size=None, + stride=1, + pad=1, + dilation=1, + group=4, + offset_scale=1.0, + act_layer='GELU', + norm_layer='LN', + center_feature_scale=False, + remove_center=False, + ): + """ + DCNv3 Module + :param channels + :param kernel_size + :param stride + :param pad + :param dilation + :param group + :param offset_scale + :param act_layer + :param norm_layer + """ + super().__init__() + if channels % group != 0: + raise ValueError( + f'channels must be divisible by group, but got {channels} and {group}') + _d_per_group = channels // group + dw_kernel_size = dw_kernel_size if dw_kernel_size is not None else kernel_size + # you'd better set _d_per_group to a power of 2 which is more efficient in our CUDA implementation + if not _is_power_of_2(_d_per_group): + warnings.warn( + "You'd better set channels in DCNv3 to make the dimension of each attention head a power of 2 " + "which is more efficient in our CUDA implementation.") + + self.offset_scale = offset_scale + self.channels = channels + self.kernel_size = kernel_size + self.dw_kernel_size = dw_kernel_size + self.stride = stride + self.dilation = dilation + self.pad = pad + self.group = group + self.group_channels = channels // group + self.offset_scale = offset_scale + self.center_feature_scale = center_feature_scale + self.remove_center = int(remove_center) + + self.dw_conv = nn.Sequential( + nn.Conv2d( + channels, + channels, + kernel_size=dw_kernel_size, + stride=1, + padding=(dw_kernel_size - 1) // 2, + groups=channels), + build_norm_layer( + channels, + norm_layer, + 'channels_first', + 'channels_last'), + build_act_layer(act_layer)) + self.offset = nn.Linear( + channels, + group * (kernel_size * kernel_size - remove_center) * 2) + self.mask = nn.Linear( + channels, + group * (kernel_size * kernel_size - remove_center)) + self.input_proj = nn.Linear(channels, channels) + self.output_proj = nn.Linear(channels, channels) + self._reset_parameters() + + if center_feature_scale: + self.center_feature_scale_proj_weight = nn.Parameter( + torch.zeros((group, channels), dtype=torch.float)) + self.center_feature_scale_proj_bias = nn.Parameter( + torch.tensor(0.0, dtype=torch.float).view((1,)).repeat(group, )) + self.center_feature_scale_module = CenterFeatureScaleModule() + + def _reset_parameters(self): + constant_(self.offset.weight.data, 0.) + constant_(self.offset.bias.data, 0.) + constant_(self.mask.weight.data, 0.) + constant_(self.mask.bias.data, 0.) + xavier_uniform_(self.input_proj.weight.data) + constant_(self.input_proj.bias.data, 0.) + xavier_uniform_(self.output_proj.weight.data) + constant_(self.output_proj.bias.data, 0.) + + def forward(self, input): + """ + :param query (N, H, W, C) + :return output (N, H, W, C) + """ + N, H, W, _ = input.shape + + x = self.input_proj(input) + x_proj = x + + x1 = input.permute(0, 3, 1, 2) + x1 = self.dw_conv(x1) + offset = self.offset(x1) + mask = self.mask(x1).reshape(N, H, W, self.group, -1) + mask = F.softmax(mask, -1).reshape(N, H, W, -1) + + x = dcnv3_core_pytorch( + x, offset, mask, + self.kernel_size, self.kernel_size, + self.stride, self.stride, + self.pad, self.pad, + self.dilation, self.dilation, + self.group, self.group_channels, + self.offset_scale, self.remove_center) + if self.center_feature_scale: + center_feature_scale = self.center_feature_scale_module( + x1, self.center_feature_scale_proj_weight, self.center_feature_scale_proj_bias) + # N, H, W, groups -> N, H, W, groups, 1 -> N, H, W, groups, _d_per_group -> N, H, W, channels + center_feature_scale = center_feature_scale[..., None].repeat( + 1, 1, 1, 1, self.channels // self.group).flatten(-2) + x = x * (1 - center_feature_scale) + x_proj * center_feature_scale + x = self.output_proj(x) + + return x + + +class DCNv3(nn.Module): + def __init__( + self, + channels=64, + kernel_size=3, + dw_kernel_size=None, + stride=1, + pad=1, + dilation=1, + group=4, + offset_scale=1.0, + act_layer='GELU', + norm_layer='LN', + center_feature_scale=False, + remove_center=False, + ): + """ + DCNv3 Module + :param channels + :param kernel_size + :param stride + :param pad + :param dilation + :param group + :param offset_scale + :param act_layer + :param norm_layer + """ + super().__init__() + if channels % group != 0: + raise ValueError( + f'channels must be divisible by group, but got {channels} and {group}') + _d_per_group = channels // group + dw_kernel_size = dw_kernel_size if dw_kernel_size is not None else kernel_size + # you'd better set _d_per_group to a power of 2 which is more efficient in our CUDA implementation + if not _is_power_of_2(_d_per_group): + warnings.warn( + "You'd better set channels in DCNv3 to make the dimension of each attention head a power of 2 " + "which is more efficient in our CUDA implementation.") + + self.offset_scale = offset_scale + self.channels = channels + self.kernel_size = kernel_size + self.dw_kernel_size = dw_kernel_size + self.stride = stride + self.dilation = dilation + self.pad = pad + self.group = group + self.group_channels = channels // group + self.offset_scale = offset_scale + self.center_feature_scale = center_feature_scale + self.remove_center = int(remove_center) + + if self.remove_center and self.kernel_size % 2 == 0: + raise ValueError('remove_center is only compatible with odd kernel size.') + + self.dw_conv = nn.Sequential( + nn.Conv2d( + channels, + channels, + kernel_size=dw_kernel_size, + stride=1, + padding=(dw_kernel_size - 1) // 2, + groups=channels), + build_norm_layer( + channels, + norm_layer, + 'channels_first', + 'channels_last'), + build_act_layer(act_layer)) + self.offset = nn.Linear( + channels, + group * (kernel_size * kernel_size - remove_center) * 2) + self.mask = nn.Linear( + channels, + group * (kernel_size * kernel_size - remove_center)) + self.input_proj = nn.Linear(channels, channels) + self.output_proj = nn.Linear(channels, channels) + self._reset_parameters() + + if center_feature_scale: + self.center_feature_scale_proj_weight = nn.Parameter( + torch.zeros((group, channels), dtype=torch.float)) + self.center_feature_scale_proj_bias = nn.Parameter( + torch.tensor(0.0, dtype=torch.float).view((1,)).repeat(group, )) + self.center_feature_scale_module = CenterFeatureScaleModule() + + def _reset_parameters(self): + constant_(self.offset.weight.data, 0.) + constant_(self.offset.bias.data, 0.) + constant_(self.mask.weight.data, 0.) + constant_(self.mask.bias.data, 0.) + xavier_uniform_(self.input_proj.weight.data) + constant_(self.input_proj.bias.data, 0.) + xavier_uniform_(self.output_proj.weight.data) + constant_(self.output_proj.bias.data, 0.) + + def forward(self, input): + """ + :param query (N, H, W, C) + :return output (N, H, W, C) + """ + N, H, W, _ = input.shape + + x = self.input_proj(input) + x_proj = x + dtype = x.dtype + + x1 = input.permute(0, 3, 1, 2) + x1 = self.dw_conv(x1) + offset = self.offset(x1) + mask = self.mask(x1).reshape(N, H, W, self.group, -1) + mask = F.softmax(mask, -1) + mask = mask.reshape(N, H, W, -1).type(dtype) + + x = DCNv3Function.apply( + x, offset, mask, + self.kernel_size, self.kernel_size, + self.stride, self.stride, + self.pad, self.pad, + self.dilation, self.dilation, + self.group, self.group_channels, + self.offset_scale, + 256, + self.remove_center) + + if self.center_feature_scale: + center_feature_scale = self.center_feature_scale_module( + x1, self.center_feature_scale_proj_weight, self.center_feature_scale_proj_bias) + # N, H, W, groups -> N, H, W, groups, 1 -> N, H, W, groups, _d_per_group -> N, H, W, channels + center_feature_scale = center_feature_scale[..., None].repeat( + 1, 1, 1, 1, self.channels // self.group).flatten(-2) + x = x * (1 - center_feature_scale) + x_proj * center_feature_scale + x = self.output_proj(x) + + return x diff --git a/classification/ops_dcnv3/setup.py b/classification/ops_dcnv3/setup.py new file mode 100644 index 0000000..d0c220d --- /dev/null +++ b/classification/ops_dcnv3/setup.py @@ -0,0 +1,75 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + +import os +import glob + +import torch + +from torch.utils.cpp_extension import CUDA_HOME +from torch.utils.cpp_extension import CppExtension +from torch.utils.cpp_extension import CUDAExtension + +from setuptools import find_packages +from setuptools import setup + +requirements = ["torch", "torchvision"] + + +def get_extensions(): + this_dir = os.path.dirname(os.path.abspath(__file__)) + extensions_dir = os.path.join(this_dir, "src") + + main_file = glob.glob(os.path.join(extensions_dir, "*.cpp")) + source_cpu = glob.glob(os.path.join(extensions_dir, "cpu", "*.cpp")) + source_cuda = glob.glob(os.path.join(extensions_dir, "cuda", "*.cu")) + + sources = main_file + source_cpu + extension = CppExtension + extra_compile_args = {"cxx": []} + define_macros = [] + + if torch.cuda.is_available() and CUDA_HOME is not None: + extension = CUDAExtension + sources += source_cuda + define_macros += [("WITH_CUDA", None)] + extra_compile_args["nvcc"] = [ + # "-DCUDA_HAS_FP16=1", + # "-D__CUDA_NO_HALF_OPERATORS__", + # "-D__CUDA_NO_HALF_CONVERSIONS__", + # "-D__CUDA_NO_HALF2_OPERATORS__", + ] + else: + raise NotImplementedError('Cuda is not availabel') + + sources = [os.path.join(extensions_dir, s) for s in sources] + include_dirs = [extensions_dir] + ext_modules = [ + extension( + "DCNv3", + sources, + include_dirs=include_dirs, + define_macros=define_macros, + extra_compile_args=extra_compile_args, + ) + ] + return ext_modules + + +setup( + name="DCNv3", + version="1.1", + author="InternImage", + url="https://github.com/OpenGVLab/InternImage", + description= + "PyTorch Wrapper for CUDA Functions of DCNv3", + packages=find_packages(exclude=( + "configs", + "tests", + )), + ext_modules=get_extensions(), + cmdclass={"build_ext": torch.utils.cpp_extension.BuildExtension}, +) diff --git a/classification/ops_dcnv3/src/cpu/dcnv3_cpu.cpp b/classification/ops_dcnv3/src/cpu/dcnv3_cpu.cpp new file mode 100644 index 0000000..a3bddc1 --- /dev/null +++ b/classification/ops_dcnv3/src/cpu/dcnv3_cpu.cpp @@ -0,0 +1,37 @@ +/*! +************************************************************************************************** +* InternImage +* Copyright (c) 2022 OpenGVLab +* Licensed under The MIT License [see LICENSE for details] +************************************************************************************************** +* Modified from +*https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +************************************************************************************************** +*/ + +#include + +#include +#include + +at::Tensor dcnv3_cpu_forward(const at::Tensor &input, const at::Tensor &offset, + const at::Tensor &mask, const int kernel_h, + const int kernel_w, const int stride_h, + const int stride_w, const int pad_h, + const int pad_w, const int dilation_h, + const int dilation_w, const int group, + const int group_channels, const float offset_scale, + const int im2col_step) { + AT_ERROR("Not implement on cpu"); +} + +std::vector +dcnv3_cpu_backward(const at::Tensor &input, const at::Tensor &offset, + const at::Tensor &mask, const int kernel_h, + const int kernel_w, const int stride_h, const int stride_w, + const int pad_h, const int pad_w, const int dilation_h, + const int dilation_w, const int group, + const int group_channels, const float offset_scale, + const at::Tensor &grad_output, const int im2col_step) { + AT_ERROR("Not implement on cpu"); +} diff --git a/classification/ops_dcnv3/src/cpu/dcnv3_cpu.h b/classification/ops_dcnv3/src/cpu/dcnv3_cpu.h new file mode 100644 index 0000000..d457bcb --- /dev/null +++ b/classification/ops_dcnv3/src/cpu/dcnv3_cpu.h @@ -0,0 +1,31 @@ +/*! +************************************************************************************************** +* InternImage +* Copyright (c) 2022 OpenGVLab +* Licensed under The MIT License [see LICENSE for details] +************************************************************************************************** +* Modified from +*https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +************************************************************************************************** +*/ + +#pragma once +#include + +at::Tensor dcnv3_cpu_forward(const at::Tensor &input, const at::Tensor &offset, + const at::Tensor &mask, const int kernel_h, + const int kernel_w, const int stride_h, + const int stride_w, const int pad_h, + const int pad_w, const int dilation_h, + const int dilation_w, const int group, + const int group_channels, const float offset_scale, + const int im2col_step); + +std::vector +dcnv3_cpu_backward(const at::Tensor &input, const at::Tensor &offset, + const at::Tensor &mask, const int kernel_h, + const int kernel_w, const int stride_h, const int stride_w, + const int pad_h, const int pad_w, const int dilation_h, + const int dilation_w, const int group, + const int group_channels, const float offset_scale, + const at::Tensor &grad_output, const int im2col_step); diff --git a/classification/ops_dcnv3/src/cuda/dcnv3_cuda.cu b/classification/ops_dcnv3/src/cuda/dcnv3_cuda.cu new file mode 100644 index 0000000..c8ee479 --- /dev/null +++ b/classification/ops_dcnv3/src/cuda/dcnv3_cuda.cu @@ -0,0 +1,174 @@ +/*! +************************************************************************************************** +* InternImage +* Copyright (c) 2022 OpenGVLab +* Licensed under The MIT License [see LICENSE for details] +************************************************************************************************** +* Modified from +*https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +************************************************************************************************** +*/ + +#include "cuda/dcnv3_im2col_cuda.cuh" +#include + +#include +#include +#include +#include +#include + +at::Tensor dcnv3_cuda_forward(const at::Tensor &input, const at::Tensor &offset, + const at::Tensor &mask, const int kernel_h, + const int kernel_w, const int stride_h, + const int stride_w, const int pad_h, + const int pad_w, const int dilation_h, + const int dilation_w, const int group, + const int group_channels, + const float offset_scale, const int im2col_step, const int remove_center) { + AT_ASSERTM(input.is_contiguous(), "input tensor has to be contiguous"); + AT_ASSERTM(offset.is_contiguous(), "offset tensor has to be contiguous"); + AT_ASSERTM(mask.is_contiguous(), "mask tensor has to be contiguous"); + AT_ASSERTM(input.type().is_cuda(), "input must be a CUDA tensor"); + AT_ASSERTM(offset.type().is_cuda(), "offset must be a CUDA tensor"); + AT_ASSERTM(mask.type().is_cuda(), "mask must be a CUDA tensor"); + + const int batch = input.size(0); + const int height_in = input.size(1); + const int width_in = input.size(2); + const int channels = input.size(3); + const int height_out = + (height_in + 2 * pad_h - (dilation_h * (kernel_h - 1) + 1)) / stride_h + + 1; + const int width_out = + (width_in + 2 * pad_w - (dilation_w * (kernel_w - 1) + 1)) / stride_w + + 1; + const int im2col_step_ = std::min(batch, im2col_step); + + AT_ASSERTM(batch % im2col_step_ == 0, + "batch(%d) must divide im2col_step(%d)", batch, im2col_step_); + AT_ASSERTM( + channels == (group * group_channels), + "Input channels and group times group channels wont match: (%d vs %d).", + channels, group * group_channels); + + auto output = + at::zeros({batch, height_out, width_out, group * group_channels}, + input.options()); + + const int batch_n = im2col_step_; + auto output_n = output.view({batch / batch_n, batch_n, height_out, + width_out, group * group_channels}); + auto per_input_size = height_in * width_in * group * group_channels; + auto per_offset_size = + height_out * width_out * group * (kernel_h * kernel_w - remove_center) * 2; + auto per_mask_size = height_out * width_out * group * (kernel_h * kernel_w - remove_center); + for (int n = 0; n < batch / im2col_step_; ++n) { + auto columns = output_n.select(0, n); + // AT_DISPATCH_FLOATING_TYPES( + AT_DISPATCH_FLOATING_TYPES_AND_HALF( + input.type(), "ms_deform_attn_forward_cuda", ([&] { + dcnv3_im2col_cuda( + at::cuda::getCurrentCUDAStream(), + input.data() + n * im2col_step_ * per_input_size, + offset.data() + + n * im2col_step_ * per_offset_size, + mask.data() + n * im2col_step_ * per_mask_size, + columns.data(), kernel_h, kernel_w, stride_h, + stride_w, pad_h, pad_w, dilation_h, dilation_w, group, + group_channels, batch_n, height_in, width_in, height_out, + width_out, offset_scale, remove_center); + })); + } + + return output; +} + +std::vector +dcnv3_cuda_backward(const at::Tensor &input, const at::Tensor &offset, + const at::Tensor &mask, const int kernel_h, + const int kernel_w, const int stride_h, const int stride_w, + const int pad_h, const int pad_w, const int dilation_h, + const int dilation_w, const int group, + const int group_channels, const float offset_scale, + const at::Tensor &grad_output, const int im2col_step, const int remove_center) { + + AT_ASSERTM(input.is_contiguous(), "input tensor has to be contiguous"); + AT_ASSERTM(offset.is_contiguous(), "offset tensor has to be contiguous"); + AT_ASSERTM(mask.is_contiguous(), "mask tensor has to be contiguous"); + AT_ASSERTM(grad_output.is_contiguous(), + "grad_output tensor has to be contiguous"); + AT_ASSERTM(input.type().is_cuda(), "input must be a CUDA tensor"); + AT_ASSERTM(offset.type().is_cuda(), "offset must be a CUDA tensor"); + AT_ASSERTM(mask.type().is_cuda(), "mask must be a CUDA tensor"); + AT_ASSERTM(grad_output.type().is_cuda(), + "grad_output must be a CUDA tensor"); + + const int batch = input.size(0); + const int height_in = input.size(1); + const int width_in = input.size(2); + const int channels = input.size(3); + const int height_out = + (height_in + 2 * pad_h - (dilation_h * (kernel_h - 1) + 1)) / stride_h + + 1; + const int width_out = + (width_in + 2 * pad_w - (dilation_w * (kernel_w - 1) + 1)) / stride_w + + 1; + const int im2col_step_ = std::min(batch, im2col_step); + + AT_ASSERTM(batch % im2col_step_ == 0, + "batch(%d) must divide im2col_step(%d)", batch, im2col_step_); + AT_ASSERTM( + channels == (group * group_channels), + "Input channels and group times group channels wont match: (%d vs %d).", + channels, group * group_channels); + + auto dtype = input.dtype(); + if (dtype == at::kHalf) { + dtype = at::kFloat; + } + + auto grad_input = at::zeros_like(input, dtype); + auto grad_offset = at::zeros_like(offset, dtype); + auto grad_mask = at::zeros_like(mask, dtype); + + const int batch_n = im2col_step_; + auto per_input_size = height_in * width_in * group * group_channels; + auto per_offset_size = + height_out * width_out * group * (kernel_h * kernel_w - remove_center) * 2; + auto per_mask_size = height_out * width_out * group * (kernel_h * kernel_w - remove_center); + auto grad_output_n = + grad_output.view({batch / im2col_step_, batch_n, height_out * width_out, + group, group_channels}); + + for (int n = 0; n < batch / im2col_step_; ++n) { + auto grad_output_g = grad_output_n.select(0, n); + // AT_DISPATCH_FLOATING_TYPES( + AT_DISPATCH_FLOATING_TYPES_AND_HALF( + input.type(), "ms_deform_attn_backward_cuda", ([&] { + dcnv3_col2im_cuda( + at::cuda::getCurrentCUDAStream(), + grad_output_g.data(), + input.data() + n * im2col_step_ * per_input_size, + offset.data() + + n * im2col_step_ * per_offset_size, + mask.data() + n * im2col_step_ * per_mask_size, + kernel_h, kernel_w, stride_h, stride_w, pad_h, pad_w, + dilation_h, dilation_w, group, group_channels, batch_n, + height_in, width_in, height_out, width_out, offset_scale, remove_center, + grad_input.data() + + n * im2col_step_ * per_input_size, + grad_offset.data() + + n * im2col_step_ * per_offset_size, + grad_mask.data() + + n * im2col_step_ * per_mask_size); + })); + } + + if (input.dtype() == torch::kHalf) { + return {grad_input.to(torch::kHalf), grad_offset.to(torch::kHalf), + grad_mask.to(torch::kHalf)}; + } else { + return {grad_input, grad_offset, grad_mask}; + } +} \ No newline at end of file diff --git a/classification/ops_dcnv3/src/cuda/dcnv3_cuda.h b/classification/ops_dcnv3/src/cuda/dcnv3_cuda.h new file mode 100644 index 0000000..d7ac024 --- /dev/null +++ b/classification/ops_dcnv3/src/cuda/dcnv3_cuda.h @@ -0,0 +1,31 @@ +/*! +************************************************************************************************** +* InternImage +* Copyright (c) 2022 OpenGVLab +* Licensed under The MIT License [see LICENSE for details] +************************************************************************************************** +* Modified from +*https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +************************************************************************************************** +*/ + +#pragma once +#include + +at::Tensor dcnv3_cuda_forward(const at::Tensor &input, const at::Tensor &offset, + const at::Tensor &mask, const int kernel_h, + const int kernel_w, const int stride_h, + const int stride_w, const int pad_h, + const int pad_w, const int dilation_h, + const int dilation_w, const int group, + const int group_channels, + const float offset_scale, const int im2col_step, const int remove_center); + +std::vector +dcnv3_cuda_backward(const at::Tensor &input, const at::Tensor &offset, + const at::Tensor &mask, const int kernel_h, + const int kernel_w, const int stride_h, const int stride_w, + const int pad_h, const int pad_w, const int dilation_h, + const int dilation_w, const int group, + const int group_channels, const float offset_scale, + const at::Tensor &grad_output, const int im2col_step, const int remove_center); diff --git a/classification/ops_dcnv3/src/cuda/dcnv3_im2col_cuda.cuh b/classification/ops_dcnv3/src/cuda/dcnv3_im2col_cuda.cuh new file mode 100644 index 0000000..b2bbf84 --- /dev/null +++ b/classification/ops_dcnv3/src/cuda/dcnv3_im2col_cuda.cuh @@ -0,0 +1,1094 @@ +/*! +************************************************************************************************** +* InternImage +* Copyright (c) 2022 OpenGVLab +* Licensed under The MIT License [see LICENSE for details] +************************************************************************************************** +* Modified from +*https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +************************************************************************************************** +*/ + +#include +#include +#include + +#include +#include +#include +#include + +#define CUDA_KERNEL_LOOP(i, n) \ + for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < (n); \ + i += blockDim.x * gridDim.x) + +const int CUDA_NUM_THREADS = 256; +inline int GET_BLOCKS(const int N, const int num_threads) { + return (N + num_threads - 1) / num_threads; +} + +#define opmath_t at::opmath_type + +template +__device__ opmath_t dcnv3_im2col_bilinear(const scalar_t *&bottom_data, + const int &height, const int &width, + const int &group, + const int &group_channels, + const opmath_t &h, const opmath_t &w, + const int &g, const int &c) { + const int h_low = floor(h); + const int w_low = floor(w); + const int h_high = h_low + 1; + const int w_high = w_low + 1; + + const opmath_t lh = h - h_low; + const opmath_t lw = w - w_low; + const opmath_t hh = 1 - lh, hw = 1 - lw; + + const int w_stride = group * group_channels; + const int h_stride = width * w_stride; + const int h_low_ptr_offset = h_low * h_stride; + const int h_high_ptr_offset = h_low_ptr_offset + h_stride; + const int w_low_ptr_offset = w_low * w_stride; + const int w_high_ptr_offset = w_low_ptr_offset + w_stride; + const int base_ptr = g * group_channels + c; + + opmath_t v1 = 0; + if (h_low >= 0 && w_low >= 0) { + const int ptr1 = h_low_ptr_offset + w_low_ptr_offset + base_ptr; + v1 = bottom_data[ptr1]; + } + opmath_t v2 = 0; + if (h_low >= 0 && w_high <= width - 1) { + const int ptr2 = h_low_ptr_offset + w_high_ptr_offset + base_ptr; + v2 = bottom_data[ptr2]; + } + opmath_t v3 = 0; + if (h_high <= height - 1 && w_low >= 0) { + const int ptr3 = h_high_ptr_offset + w_low_ptr_offset + base_ptr; + v3 = bottom_data[ptr3]; + } + opmath_t v4 = 0; + if (h_high <= height - 1 && w_high <= width - 1) { + const int ptr4 = h_high_ptr_offset + w_high_ptr_offset + base_ptr; + v4 = bottom_data[ptr4]; + } + const opmath_t w1 = hh * hw, w2 = hh * lw, w3 = lh * hw, w4 = lh * lw; + + const opmath_t val = (w1 * v1 + w2 * v2 + w3 * v3 + w4 * v4); + return val; +} + +template +__device__ void dcnv3_col2im_bilinear( + const scalar_t *&bottom_data, const int &height, const int &width, + const int &nheads, const int &group_channels, const opmath_t &h, + const opmath_t &w, const int &m, const int &c, const opmath_t offset_scale, + const opmath_t &top_grad, const opmath_t &mask, opmath_t *&grad_im, + opmath_t *grad_offset, opmath_t *grad_mask) { + const int h_low = floor(h); + const int w_low = floor(w); + const int h_high = h_low + 1; + const int w_high = w_low + 1; + + const opmath_t lh = h - h_low; + const opmath_t lw = w - w_low; + const opmath_t hh = 1 - lh, hw = 1 - lw; + + const int w_stride = nheads * group_channels; + const int h_stride = width * w_stride; + const int h_low_ptr_offset = h_low * h_stride; + const int h_high_ptr_offset = h_low_ptr_offset + h_stride; + const int w_low_ptr_offset = w_low * w_stride; + const int w_high_ptr_offset = w_low_ptr_offset + w_stride; + const int base_ptr = m * group_channels + c; + + const opmath_t w1 = hh * hw, w2 = hh * lw, w3 = lh * hw, w4 = lh * lw; + const opmath_t top_grad_im = top_grad * mask; + opmath_t grad_h_weight = 0, grad_w_weight = 0; + + opmath_t v1 = 0; + if (h_low >= 0 && w_low >= 0) { + const int ptr1 = h_low_ptr_offset + w_low_ptr_offset + base_ptr; + v1 = bottom_data[ptr1]; + grad_h_weight -= hw * v1; + grad_w_weight -= hh * v1; + atomicAdd(grad_im + ptr1, w1 * top_grad_im); + } + opmath_t v2 = 0; + if (h_low >= 0 && w_high <= width - 1) { + const int ptr2 = h_low_ptr_offset + w_high_ptr_offset + base_ptr; + v2 = bottom_data[ptr2]; + grad_h_weight -= lw * v2; + grad_w_weight += hh * v2; + atomicAdd(grad_im + ptr2, w2 * top_grad_im); + } + opmath_t v3 = 0; + if (h_high <= height - 1 && w_low >= 0) { + const int ptr3 = h_high_ptr_offset + w_low_ptr_offset + base_ptr; + v3 = bottom_data[ptr3]; + grad_h_weight += hw * v3; + grad_w_weight -= lh * v3; + atomicAdd(grad_im + ptr3, w3 * top_grad_im); + } + opmath_t v4 = 0; + if (h_high <= height - 1 && w_high <= width - 1) { + const int ptr4 = h_high_ptr_offset + w_high_ptr_offset + base_ptr; + v4 = bottom_data[ptr4]; + grad_h_weight += lw * v4; + grad_w_weight += lh * v4; + atomicAdd(grad_im + ptr4, w4 * top_grad_im); + } + + const opmath_t val = (w1 * v1 + w2 * v2 + w3 * v3 + w4 * v4); + *grad_mask = top_grad * val; + *grad_offset = offset_scale * grad_w_weight * top_grad_im; + *(grad_offset + 1) = offset_scale * grad_h_weight * top_grad_im; +} + +template +__device__ void dcnv3_col2im_bilinear_gm( + const scalar_t *&bottom_data, const int &height, const int &width, + const int &nheads, const int &group_channels, const opmath_t &h, + const opmath_t &w, const int &m, const int &c, const opmath_t offset_scale, + const opmath_t &top_grad, const opmath_t &mask, opmath_t *&grad_im, + opmath_t *grad_offset, opmath_t *grad_mask) { + const int h_low = floor(h); + const int w_low = floor(w); + const int h_high = h_low + 1; + const int w_high = w_low + 1; + + const opmath_t lh = h - h_low; + const opmath_t lw = w - w_low; + const opmath_t hh = 1 - lh, hw = 1 - lw; + + const int w_stride = nheads * group_channels; + const int h_stride = width * w_stride; + const int h_low_ptr_offset = h_low * h_stride; + const int h_high_ptr_offset = h_low_ptr_offset + h_stride; + const int w_low_ptr_offset = w_low * w_stride; + const int w_high_ptr_offset = w_low_ptr_offset + w_stride; + const int base_ptr = m * group_channels + c; + + const opmath_t w1 = hh * hw, w2 = hh * lw, w3 = lh * hw, w4 = lh * lw; + const opmath_t top_grad_im = top_grad * mask; + opmath_t grad_h_weight = 0, grad_w_weight = 0; + + opmath_t v1 = 0; + if (h_low >= 0 && w_low >= 0) { + const int ptr1 = h_low_ptr_offset + w_low_ptr_offset + base_ptr; + v1 = bottom_data[ptr1]; + grad_h_weight -= hw * v1; + grad_w_weight -= hh * v1; + atomicAdd(grad_im + ptr1, w1 * top_grad_im); + } + opmath_t v2 = 0; + if (h_low >= 0 && w_high <= width - 1) { + const int ptr2 = h_low_ptr_offset + w_high_ptr_offset + base_ptr; + v2 = bottom_data[ptr2]; + grad_h_weight -= lw * v2; + grad_w_weight += hh * v2; + atomicAdd(grad_im + ptr2, w2 * top_grad_im); + } + opmath_t v3 = 0; + if (h_high <= height - 1 && w_low >= 0) { + const int ptr3 = h_high_ptr_offset + w_low_ptr_offset + base_ptr; + v3 = bottom_data[ptr3]; + grad_h_weight += hw * v3; + grad_w_weight -= lh * v3; + atomicAdd(grad_im + ptr3, w3 * top_grad_im); + } + opmath_t v4 = 0; + if (h_high <= height - 1 && w_high <= width - 1) { + const int ptr4 = h_high_ptr_offset + w_high_ptr_offset + base_ptr; + v4 = bottom_data[ptr4]; + grad_h_weight += lw * v4; + grad_w_weight += lh * v4; + atomicAdd(grad_im + ptr4, w4 * top_grad_im); + } + + const opmath_t val = (w1 * v1 + w2 * v2 + w3 * v3 + w4 * v4); + atomicAdd(grad_mask, top_grad * val); + atomicAdd(grad_offset, offset_scale * grad_w_weight * top_grad_im); + atomicAdd(grad_offset + 1, offset_scale * grad_h_weight * top_grad_im); +} + +template +__global__ void dcnv3_im2col_gpu_kernel( + const int num_kernels, const scalar_t *data_im, const scalar_t *data_offset, + const scalar_t *data_mask, scalar_t *data_col, const int kernel_h, + const int kernel_w, const int stride_h, const int stride_w, const int pad_h, + const int pad_w, const int dilation_h, const int dilation_w, + const int group, const int group_channels, const int height_in, + const int width_in, const int height_out, const int width_out, + const opmath_t offset_scale, const int remove_center) { + CUDA_KERNEL_LOOP(index, num_kernels) { + int _temp = index; + const int c_col = _temp % group_channels; + _temp /= group_channels; + const int sampling_index = _temp; + const int g_col = _temp % group; + _temp /= group; + const int p0_w = ((dilation_w * (kernel_w - 1)) >> 1) - pad_w + + (_temp % width_out) * stride_w; + _temp /= width_out; + const int p0_h = ((dilation_h * (kernel_h - 1)) >> 1) - pad_h + + (_temp % height_out) * stride_h; + _temp /= height_out; + const int b_col = _temp; + + const int input_size = height_in * width_in; + scalar_t *data_col_ptr = data_col + index; + const int kernel_size = kernel_h * kernel_w - remove_center; + int data_weight_ptr = sampling_index * kernel_size; + int data_loc_w_ptr = data_weight_ptr << 1; + const int qid_stride = group * group_channels; + opmath_t col = 0; + const scalar_t *data_im_ptr = data_im + b_col * input_size * qid_stride; + // top-left + const opmath_t p0_w_ = + p0_w - ((dilation_w * (kernel_w - 1)) >> 1) * offset_scale; + const opmath_t p0_h_ = + p0_h - ((dilation_h * (kernel_h - 1)) >> 1) * offset_scale; + + const int center_h = kernel_h / 2; + const int center_w = kernel_w / 2; + + for (int i = 0; i < kernel_w; ++i) { + for (int j = 0; j < kernel_h; ++j) { + // if not remove center, or remove center and not the center + if (i!=center_w || j!=center_h || !remove_center) { + const opmath_t offset_w = data_offset[data_loc_w_ptr]; + const opmath_t offset_h = data_offset[data_loc_w_ptr + 1]; + const opmath_t loc_w = + p0_w_ + (i * dilation_w + offset_w) * offset_scale; + const opmath_t loc_h = + p0_h_ + (j * dilation_h + offset_h) * offset_scale; + const opmath_t weight = data_mask[data_weight_ptr]; + if (loc_h > -1 && loc_w > -1 && loc_h < height_in && + loc_w < width_in) { + col += dcnv3_im2col_bilinear( + data_im_ptr, height_in, width_in, group, + group_channels, loc_h, loc_w, g_col, c_col) * + weight; + } + data_weight_ptr += 1; + data_loc_w_ptr += 2; + } + } + } + *data_col_ptr = col; + } +} + +// debug +template +__global__ void dcnv3_col2im_gpu_kernel_shm_blocksize_aware_reduce_v1( + const int num_kernels, const scalar_t *grad_col, const scalar_t *data_im, + const scalar_t *data_offset, const scalar_t *data_mask, const int kernel_h, + const int kernel_w, const int stride_h, const int stride_w, const int pad_h, + const int pad_w, const int dilation_h, const int dilation_w, + const int group, const int group_channels, const int height_in, + const int width_in, const int height_out, const int width_out, + const opmath_t offset_scale, const int remove_center, opmath_t *grad_im, opmath_t *grad_offset, + opmath_t *grad_mask) { + CUDA_KERNEL_LOOP(index, num_kernels) { + __shared__ opmath_t cache_grad_offset[blockSize * 2]; + __shared__ opmath_t cache_grad_mask[blockSize]; + unsigned int tid = threadIdx.x; + int _temp = index; + const int c_col = _temp % group_channels; + _temp /= group_channels; + const int sampling_index = _temp; + const int g_col = _temp % group; + _temp /= group; + const int p0_w = ((dilation_w * (kernel_w - 1)) >> 1) - pad_w + + (_temp % width_out) * stride_w; + _temp /= width_out; + const int p0_h = ((dilation_h * (kernel_h - 1)) >> 1) - pad_h + + (_temp % height_out) * stride_h; + _temp /= height_out; + const int b_col = _temp; + + const opmath_t top_grad = grad_col[index]; + const int input_size = height_in * width_in; + const int kernel_size = kernel_h * kernel_w - remove_center; + int data_weight_ptr = sampling_index * kernel_size; + int data_loc_w_ptr = data_weight_ptr << 1; + const int grad_sampling_ptr = data_weight_ptr; + grad_offset += grad_sampling_ptr << 1; + grad_mask += grad_sampling_ptr; + const int qid_stride = group * group_channels; + const int im_ptr_offset = b_col * input_size * qid_stride; + const scalar_t *data_im_ptr = data_im + im_ptr_offset; + opmath_t *grad_im_ptr = grad_im + im_ptr_offset; + const opmath_t p0_w_ = + p0_w - ((dilation_w * (kernel_w - 1)) >> 1) * offset_scale; + const opmath_t p0_h_ = + p0_h - ((dilation_h * (kernel_h - 1)) >> 1) * offset_scale; + + const int center_h = kernel_h / 2; + const int center_w = kernel_w / 2; + + for (int i = 0; i < kernel_w; ++i) { + for (int j = 0; j < kernel_h; ++j) { + // if not remove center, or remove center and not the center + if (i!=center_w || j!=center_h || !remove_center) { + const opmath_t offset_w = data_offset[data_loc_w_ptr]; + const opmath_t offset_h = data_offset[data_loc_w_ptr + 1]; + const opmath_t loc_w = + p0_w_ + (i * dilation_w + offset_w) * offset_scale; + const opmath_t loc_h = + p0_h_ + (j * dilation_h + offset_h) * offset_scale; + const opmath_t weight = data_mask[data_weight_ptr]; + *(cache_grad_offset + (threadIdx.x << 1)) = 0; + *(cache_grad_offset + ((threadIdx.x << 1) + 1)) = 0; + *(cache_grad_mask + threadIdx.x) = 0; + if (loc_h > -1 && loc_w > -1 && loc_h < height_in && + loc_w < width_in) { + dcnv3_col2im_bilinear( + data_im_ptr, height_in, width_in, group, group_channels, + loc_h, loc_w, g_col, c_col, offset_scale, top_grad, + weight, grad_im_ptr, + cache_grad_offset + (threadIdx.x << 1), + cache_grad_mask + threadIdx.x); + } + + __syncthreads(); + if (tid == 0) { + opmath_t _grad_w = cache_grad_offset[0], + _grad_h = cache_grad_offset[1], + _grad_a = cache_grad_mask[0]; + int sid = 2; + for (unsigned int tid = 1; tid < blockSize; ++tid) { + _grad_w += cache_grad_offset[sid]; + _grad_h += cache_grad_offset[sid + 1]; + _grad_a += cache_grad_mask[tid]; + sid += 2; + } + + *grad_offset = _grad_w; + *(grad_offset + 1) = _grad_h; + *grad_mask = _grad_a; + } + __syncthreads(); + + data_weight_ptr += 1; + data_loc_w_ptr += 2; + grad_mask += 1; + grad_offset += 2; + } + } + } + } +} + +template +__global__ void dcnv3_col2im_gpu_kernel_shm_blocksize_aware_reduce_v2( + const int num_kernels, const scalar_t *grad_col, const scalar_t *data_im, + const scalar_t *data_offset, const scalar_t *data_mask, const int kernel_h, + const int kernel_w, const int stride_h, const int stride_w, const int pad_h, + const int pad_w, const int dilation_h, const int dilation_w, + const int group, const int group_channels, const int height_in, + const int width_in, const int height_out, const int width_out, + const opmath_t offset_scale, const int remove_center, opmath_t *grad_im, opmath_t *grad_offset, + opmath_t *grad_mask) { + CUDA_KERNEL_LOOP(index, num_kernels) { + __shared__ opmath_t cache_grad_offset[blockSize * 2]; + __shared__ opmath_t cache_grad_mask[blockSize]; + unsigned int tid = threadIdx.x; + int _temp = index; + const int c_col = _temp % group_channels; + _temp /= group_channels; + const int sampling_index = _temp; + const int g_col = _temp % group; + _temp /= group; + const int p0_w = ((dilation_w * (kernel_w - 1)) >> 1) - pad_w + + (_temp % width_out) * stride_w; + _temp /= width_out; + const int p0_h = ((dilation_h * (kernel_h - 1)) >> 1) - pad_h + + (_temp % height_out) * stride_h; + _temp /= height_out; + const int b_col = _temp; + + const opmath_t top_grad = grad_col[index]; + const int input_size = height_in * width_in; + const int kernel_size = kernel_h * kernel_w - remove_center; + int data_weight_ptr = sampling_index * kernel_size; + int data_loc_w_ptr = data_weight_ptr << 1; + const int grad_sampling_ptr = data_weight_ptr; + grad_offset += grad_sampling_ptr << 1; + grad_mask += grad_sampling_ptr; + const int qid_stride = group * group_channels; + const int im_ptr_offset = b_col * input_size * qid_stride; + const scalar_t *data_im_ptr = data_im + im_ptr_offset; + opmath_t *grad_im_ptr = grad_im + im_ptr_offset; + const opmath_t p0_w_ = + p0_w - ((dilation_w * (kernel_w - 1)) >> 1) * offset_scale; + const opmath_t p0_h_ = + p0_h - ((dilation_h * (kernel_h - 1)) >> 1) * offset_scale; + + const int center_h = kernel_h / 2; + const int center_w = kernel_w / 2; + + for (int i = 0; i < kernel_w; ++i) { + for (int j = 0; j < kernel_h; ++j) { + // if not remove center, or remove center and not the center + if (i!=center_w || j!=center_h || !remove_center) { + const opmath_t offset_w = data_offset[data_loc_w_ptr]; + const opmath_t offset_h = data_offset[data_loc_w_ptr + 1]; + const opmath_t loc_w = + p0_w_ + (i * dilation_w + offset_w) * offset_scale; + const opmath_t loc_h = + p0_h_ + (j * dilation_h + offset_h) * offset_scale; + const opmath_t weight = data_mask[data_weight_ptr]; + *(cache_grad_offset + (threadIdx.x << 1)) = 0; + *(cache_grad_offset + ((threadIdx.x << 1) + 1)) = 0; + *(cache_grad_mask + threadIdx.x) = 0; + if (loc_h > -1 && loc_w > -1 && loc_h < height_in && + loc_w < width_in) { + dcnv3_col2im_bilinear( + data_im_ptr, height_in, width_in, group, group_channels, + loc_h, loc_w, g_col, c_col, offset_scale, top_grad, + weight, grad_im_ptr, + cache_grad_offset + (threadIdx.x << 1), + cache_grad_mask + threadIdx.x); + } + + __syncthreads(); + + for (unsigned int s = blockSize / 2; s > 0; s >>= 1) { + if (tid < s) { + const unsigned int xid1 = tid << 1; + const unsigned int xid2 = (tid + s) << 1; + cache_grad_mask[tid] += cache_grad_mask[tid + s]; + cache_grad_offset[xid1] += cache_grad_offset[xid2]; + cache_grad_offset[xid1 + 1] += + cache_grad_offset[xid2 + 1]; + } + __syncthreads(); + } + + if (tid == 0) { + *grad_offset = cache_grad_offset[0]; + *(grad_offset + 1) = cache_grad_offset[1]; + *grad_mask = cache_grad_mask[0]; + } + __syncthreads(); + + data_weight_ptr += 1; + data_loc_w_ptr += 2; + grad_mask += 1; + grad_offset += 2; + } + } + } + } +} + +template +__global__ void dcnv3_col2im_gpu_kernel_shm_reduce_v1( + const int num_kernels, const scalar_t *grad_col, const scalar_t *data_im, + const scalar_t *data_offset, const scalar_t *data_mask, const int kernel_h, + const int kernel_w, const int stride_h, const int stride_w, const int pad_h, + const int pad_w, const int dilation_h, const int dilation_w, + const int group, const int group_channels, const int height_in, + const int width_in, const int height_out, const int width_out, + const opmath_t offset_scale, const int remove_center, opmath_t *grad_im, opmath_t *grad_offset, + opmath_t *grad_mask) { + CUDA_KERNEL_LOOP(index, num_kernels) { + extern __shared__ int _s[]; + opmath_t *cache_grad_offset = (opmath_t *)_s; + opmath_t *cache_grad_mask = cache_grad_offset + 2 * blockDim.x; + unsigned int tid = threadIdx.x; + int _temp = index; + const int c_col = _temp % group_channels; + _temp /= group_channels; + const int sampling_index = _temp; + const int g_col = _temp % group; + _temp /= group; + const int p0_w = ((dilation_w * (kernel_w - 1)) >> 1) - pad_w + + (_temp % width_out) * stride_w; + _temp /= width_out; + const int p0_h = ((dilation_h * (kernel_h - 1)) >> 1) - pad_h + + (_temp % height_out) * stride_h; + _temp /= height_out; + const int b_col = _temp; + + const opmath_t top_grad = grad_col[index]; + const int input_size = height_in * width_in; + const int kernel_size = kernel_h * kernel_w - remove_center; + int data_weight_ptr = sampling_index * kernel_size; + int data_loc_w_ptr = data_weight_ptr << 1; + const int grad_sampling_ptr = data_weight_ptr; + grad_offset += grad_sampling_ptr << 1; + grad_mask += grad_sampling_ptr; + const int qid_stride = group * group_channels; + const int im_ptr_offset = b_col * input_size * qid_stride; + const scalar_t *data_im_ptr = data_im + im_ptr_offset; + opmath_t *grad_im_ptr = grad_im + im_ptr_offset; + const opmath_t p0_w_ = + p0_w - ((dilation_w * (kernel_w - 1)) >> 1) * offset_scale; + const opmath_t p0_h_ = + p0_h - ((dilation_h * (kernel_h - 1)) >> 1) * offset_scale; + + const int center_h = kernel_h / 2; + const int center_w = kernel_w / 2; + + for (int i = 0; i < kernel_w; ++i) { + for (int j = 0; j < kernel_h; ++j) { + // if not remove center, or remove center and not the center + if (i!=center_w || j!=center_h || !remove_center) { + const opmath_t offset_w = data_offset[data_loc_w_ptr]; + const opmath_t offset_h = data_offset[data_loc_w_ptr + 1]; + const opmath_t loc_w = + p0_w_ + (i * dilation_w + offset_w) * offset_scale; + const opmath_t loc_h = + p0_h_ + (j * dilation_h + offset_h) * offset_scale; + const opmath_t weight = data_mask[data_weight_ptr]; + *(cache_grad_offset + (threadIdx.x << 1)) = 0; + *(cache_grad_offset + ((threadIdx.x << 1) + 1)) = 0; + *(cache_grad_mask + threadIdx.x) = 0; + if (loc_h > -1 && loc_w > -1 && loc_h < height_in && + loc_w < width_in) { + dcnv3_col2im_bilinear( + data_im_ptr, height_in, width_in, group, group_channels, + loc_h, loc_w, g_col, c_col, offset_scale, top_grad, + weight, grad_im_ptr, + cache_grad_offset + (threadIdx.x << 1), + cache_grad_mask + threadIdx.x); + } + + __syncthreads(); + if (tid == 0) { + opmath_t _grad_w = cache_grad_offset[0], + _grad_h = cache_grad_offset[1], + _grad_a = cache_grad_mask[0]; + int sid = 2; + for (unsigned int tid = 1; tid < blockDim.x; ++tid) { + _grad_w += cache_grad_offset[sid]; + _grad_h += cache_grad_offset[sid + 1]; + _grad_a += cache_grad_mask[tid]; + sid += 2; + } + + *grad_offset = _grad_w; + *(grad_offset + 1) = _grad_h; + *grad_mask = _grad_a; + } + __syncthreads(); + + data_weight_ptr += 1; + data_loc_w_ptr += 2; + grad_mask += 1; + grad_offset += 2; + } + } + } + } +} + +template +__global__ void dcnv3_col2im_gpu_kernel_shm_reduce_v2( + const int num_kernels, const scalar_t *grad_col, const scalar_t *data_im, + const scalar_t *data_offset, const scalar_t *data_mask, const int kernel_h, + const int kernel_w, const int stride_h, const int stride_w, const int pad_h, + const int pad_w, const int dilation_h, const int dilation_w, + const int group, const int group_channels, const int height_in, + const int width_in, const int height_out, const int width_out, + const opmath_t offset_scale, const int remove_center, opmath_t *grad_im, opmath_t *grad_offset, + opmath_t *grad_mask) { + CUDA_KERNEL_LOOP(index, num_kernels) { + extern __shared__ int _s[]; + opmath_t *cache_grad_offset = (opmath_t *)_s; + opmath_t *cache_grad_mask = cache_grad_offset + 2 * blockDim.x; + unsigned int tid = threadIdx.x; + int _temp = index; + const int c_col = _temp % group_channels; + _temp /= group_channels; + const int sampling_index = _temp; + const int g_col = _temp % group; + _temp /= group; + const int p0_w = ((dilation_w * (kernel_w - 1)) >> 1) - pad_w + + (_temp % width_out) * stride_w; + _temp /= width_out; + const int p0_h = ((dilation_h * (kernel_h - 1)) >> 1) - pad_h + + (_temp % height_out) * stride_h; + _temp /= height_out; + const int b_col = _temp; + + const opmath_t top_grad = grad_col[index]; + const int input_size = height_in * width_in; + const int kernel_size = kernel_h * kernel_w - remove_center; + int data_weight_ptr = sampling_index * kernel_size; + int data_loc_w_ptr = data_weight_ptr << 1; + const int grad_sampling_ptr = data_weight_ptr; + grad_offset += grad_sampling_ptr << 1; + grad_mask += grad_sampling_ptr; + const int qid_stride = group * group_channels; + const int im_ptr_offset = b_col * input_size * qid_stride; + const scalar_t *data_im_ptr = data_im + im_ptr_offset; + opmath_t *grad_im_ptr = grad_im + im_ptr_offset; + const opmath_t p0_w_ = + p0_w - ((dilation_w * (kernel_w - 1)) >> 1) * offset_scale; + const opmath_t p0_h_ = + p0_h - ((dilation_h * (kernel_h - 1)) >> 1) * offset_scale; + + const int center_h = kernel_h / 2; + const int center_w = kernel_w / 2; + + for (int i = 0; i < kernel_w; ++i) { + for (int j = 0; j < kernel_h; ++j) { + // if not remove center, or remove center and not the center + if (i!=center_w || j!=center_h || !remove_center) { + const opmath_t offset_w = data_offset[data_loc_w_ptr]; + const opmath_t offset_h = data_offset[data_loc_w_ptr + 1]; + const opmath_t loc_w = + p0_w_ + (i * dilation_w + offset_w) * offset_scale; + const opmath_t loc_h = + p0_h_ + (j * dilation_h + offset_h) * offset_scale; + const opmath_t weight = data_mask[data_weight_ptr]; + *(cache_grad_offset + (threadIdx.x << 1)) = 0; + *(cache_grad_offset + ((threadIdx.x << 1) + 1)) = 0; + *(cache_grad_mask + threadIdx.x) = 0; + if (loc_h > -1 && loc_w > -1 && loc_h < height_in && + loc_w < width_in) { + dcnv3_col2im_bilinear( + data_im_ptr, height_in, width_in, group, group_channels, + loc_h, loc_w, g_col, c_col, offset_scale, top_grad, + weight, grad_im_ptr, + cache_grad_offset + (threadIdx.x << 1), + cache_grad_mask + threadIdx.x); + } + + __syncthreads(); + + for (unsigned int s = blockDim.x / 2, spre = blockDim.x; s > 0; + s >>= 1, spre >>= 1) { + if (tid < s) { + const unsigned int xid1 = tid << 1; + const unsigned int xid2 = (tid + s) << 1; + cache_grad_mask[tid] += cache_grad_mask[tid + s]; + cache_grad_offset[xid1] += cache_grad_offset[xid2]; + cache_grad_offset[xid1 + 1] += + cache_grad_offset[xid2 + 1]; + if (tid + (s << 1) < spre) { + cache_grad_mask[tid] += + cache_grad_mask[tid + (s << 1)]; + cache_grad_offset[xid1] += + cache_grad_offset[xid2 + (s << 1)]; + cache_grad_offset[xid1 + 1] += + cache_grad_offset[xid2 + 1 + (s << 1)]; + } + } + __syncthreads(); + } + + if (tid == 0) { + *grad_offset = cache_grad_offset[0]; + *(grad_offset + 1) = cache_grad_offset[1]; + *grad_mask = cache_grad_mask[0]; + } + __syncthreads(); + + data_weight_ptr += 1; + data_loc_w_ptr += 2; + grad_mask += 1; + grad_offset += 2; + } + } + } + } +} + +template +__global__ void dcnv3_col2im_gpu_kernel_shm_reduce_v2_multi_blocks( + const int num_kernels, const scalar_t *grad_col, const scalar_t *data_im, + const scalar_t *data_offset, const scalar_t *data_mask, const int kernel_h, + const int kernel_w, const int stride_h, const int stride_w, const int pad_h, + const int pad_w, const int dilation_h, const int dilation_w, + const int group, const int group_channels, const int height_in, + const int width_in, const int height_out, const int width_out, + const opmath_t offset_scale, const int remove_center, opmath_t *grad_im, opmath_t *grad_offset, + opmath_t *grad_mask) { + CUDA_KERNEL_LOOP(index, num_kernels) { + extern __shared__ int _s[]; + opmath_t *cache_grad_offset = (opmath_t *)_s; + opmath_t *cache_grad_mask = cache_grad_offset + 2 * blockDim.x; + unsigned int tid = threadIdx.x; + int _temp = index; + const int c_col = _temp % group_channels; + _temp /= group_channels; + const int sampling_index = _temp; + const int g_col = _temp % group; + _temp /= group; + const int p0_w = ((dilation_w * (kernel_w - 1)) >> 1) - pad_w + + (_temp % width_out) * stride_w; + _temp /= width_out; + const int p0_h = ((dilation_h * (kernel_h - 1)) >> 1) - pad_h + + (_temp % height_out) * stride_h; + _temp /= height_out; + const int b_col = _temp; + + const opmath_t top_grad = grad_col[index]; + const int input_size = height_in * width_in; + const int kernel_size = kernel_h * kernel_w - remove_center; + int data_weight_ptr = sampling_index * kernel_size; + int data_loc_w_ptr = data_weight_ptr << 1; + const int grad_sampling_ptr = data_weight_ptr; + grad_offset += grad_sampling_ptr << 1; + grad_mask += grad_sampling_ptr; + const int qid_stride = group * group_channels; + const int im_ptr_offset = b_col * input_size * qid_stride; + const scalar_t *data_im_ptr = data_im + im_ptr_offset; + opmath_t *grad_im_ptr = grad_im + im_ptr_offset; + const opmath_t p0_w_ = + p0_w - ((dilation_w * (kernel_w - 1)) >> 1) * offset_scale; + const opmath_t p0_h_ = + p0_h - ((dilation_h * (kernel_h - 1)) >> 1) * offset_scale; + + const int center_h = kernel_h / 2; + const int center_w = kernel_w / 2; + + for (int i = 0; i < kernel_w; ++i) { + for (int j = 0; j < kernel_h; ++j) { + // if not remove center, or remove center and not the center + if (i!=center_w || j!=center_h || !remove_center) { + const opmath_t offset_w = data_offset[data_loc_w_ptr]; + const opmath_t offset_h = data_offset[data_loc_w_ptr + 1]; + const opmath_t loc_w = + p0_w_ + (i * dilation_w + offset_w) * offset_scale; + const opmath_t loc_h = + p0_h_ + (j * dilation_h + offset_h) * offset_scale; + const opmath_t weight = data_mask[data_weight_ptr]; + *(cache_grad_offset + (threadIdx.x << 1)) = 0; + *(cache_grad_offset + ((threadIdx.x << 1) + 1)) = 0; + *(cache_grad_mask + threadIdx.x) = 0; + if (loc_h > -1 && loc_w > -1 && loc_h < height_in && + loc_w < width_in) { + dcnv3_col2im_bilinear( + data_im_ptr, height_in, width_in, group, group_channels, + loc_h, loc_w, g_col, c_col, offset_scale, top_grad, + weight, grad_im_ptr, + cache_grad_offset + (threadIdx.x << 1), + cache_grad_mask + threadIdx.x); + } + + __syncthreads(); + + for (unsigned int s = blockDim.x / 2, spre = blockDim.x; s > 0; + s >>= 1, spre >>= 1) { + if (tid < s) { + const unsigned int xid1 = tid << 1; + const unsigned int xid2 = (tid + s) << 1; + cache_grad_mask[tid] += cache_grad_mask[tid + s]; + cache_grad_offset[xid1] += cache_grad_offset[xid2]; + cache_grad_offset[xid1 + 1] += + cache_grad_offset[xid2 + 1]; + if (tid + (s << 1) < spre) { + cache_grad_mask[tid] += + cache_grad_mask[tid + (s << 1)]; + cache_grad_offset[xid1] += + cache_grad_offset[xid2 + (s << 1)]; + cache_grad_offset[xid1 + 1] += + cache_grad_offset[xid2 + 1 + (s << 1)]; + } + } + __syncthreads(); + } + + if (tid == 0) { + atomicAdd(grad_offset, cache_grad_offset[0]); + atomicAdd(grad_offset + 1, cache_grad_offset[1]); + atomicAdd(grad_mask, cache_grad_mask[0]); + } + __syncthreads(); + + data_weight_ptr += 1; + data_loc_w_ptr += 2; + grad_mask += 1; + grad_offset += 2; + } + } + } + } +} + +template +__global__ void dcnv3_col2im_gpu_kernel_gm( + const int num_kernels, const scalar_t *grad_col, const scalar_t *data_im, + const scalar_t *data_offset, const scalar_t *data_mask, const int kernel_h, + const int kernel_w, const int stride_h, const int stride_w, const int pad_h, + const int pad_w, const int dilation_h, const int dilation_w, + const int group, const int group_channels, const int height_in, + const int width_in, const int height_out, const int width_out, + const opmath_t offset_scale, const int remove_center, opmath_t *grad_im, opmath_t *grad_offset, + opmath_t *grad_mask) { + CUDA_KERNEL_LOOP(index, num_kernels) { + int _temp = index; + const int c_col = _temp % group_channels; + _temp /= group_channels; + const int sampling_index = _temp; + const int g_col = _temp % group; + _temp /= group; + const int p0_w = ((dilation_w * (kernel_w - 1)) >> 1) - pad_w + + (_temp % width_out) * stride_w; + _temp /= width_out; + const int p0_h = ((dilation_h * (kernel_h - 1)) >> 1) - pad_h + + (_temp % height_out) * stride_h; + _temp /= height_out; + const int b_col = _temp; + + const opmath_t top_grad = grad_col[index]; + const int input_size = height_in * width_in; + const int kernel_size = kernel_h * kernel_w - remove_center; + int data_weight_ptr = sampling_index * kernel_size; + int data_loc_w_ptr = data_weight_ptr << 1; + const int grad_sampling_ptr = data_weight_ptr; + grad_offset += grad_sampling_ptr << 1; + grad_mask += grad_sampling_ptr; + const int qid_stride = group * group_channels; + const int im_ptr_offset = b_col * input_size * qid_stride; + const scalar_t *data_im_ptr = data_im + im_ptr_offset; + opmath_t *grad_im_ptr = grad_im + im_ptr_offset; + const opmath_t p0_w_ = + p0_w - ((dilation_w * (kernel_w - 1)) >> 1) * offset_scale; + const opmath_t p0_h_ = + p0_h - ((dilation_h * (kernel_h - 1)) >> 1) * offset_scale; + + const int center_h = kernel_h / 2; + const int center_w = kernel_w / 2; + + for (int i = 0; i < kernel_w; ++i) { + for (int j = 0; j < kernel_h; ++j) { + // if not remove center, or remove center and not the center + if (i!=center_w || j!=center_h || !remove_center) { + const opmath_t offset_w = data_offset[data_loc_w_ptr]; + const opmath_t offset_h = data_offset[data_loc_w_ptr + 1]; + const opmath_t loc_w = + p0_w_ + (i * dilation_w + offset_w) * offset_scale; + const opmath_t loc_h = + p0_h_ + (j * dilation_h + offset_h) * offset_scale; + const opmath_t weight = data_mask[data_weight_ptr]; + if (loc_h > -1 && loc_w > -1 && loc_h < height_in && + loc_w < width_in) { + dcnv3_col2im_bilinear_gm( + data_im_ptr, height_in, width_in, group, group_channels, + loc_h, loc_w, g_col, c_col, offset_scale, top_grad, + weight, grad_im_ptr, grad_offset, grad_mask); + } + data_weight_ptr += 1; + data_loc_w_ptr += 2; + grad_mask += 1; + grad_offset += 2; + } + } + } + } +} + +template +void dcnv3_im2col_cuda(cudaStream_t stream, const scalar_t *data_im, + const scalar_t *data_offset, const scalar_t *data_mask, + scalar_t *data_col, const int kernel_h, + const int kernel_w, const int stride_h, + const int stride_w, const int pad_h, const int pad_w, + const int dilation_h, const int dilation_w, + const int group, const int group_channels, + const int batch_n, const int height_in, + const int width_in, const int height_out, + const int width_out, const opmath_t offset_scale, const int remove_center) { + const int num_kernels = + batch_n * height_out * width_out * group * group_channels; + const int num_actual_kernels = + batch_n * height_out * width_out * group * group_channels; + const int num_threads = CUDA_NUM_THREADS; + dcnv3_im2col_gpu_kernel + <<>>(num_kernels, data_im, data_offset, data_mask, data_col, + kernel_h, kernel_w, stride_h, stride_w, pad_h, pad_w, + dilation_h, dilation_w, group, group_channels, height_in, + width_in, height_out, width_out, offset_scale, remove_center); + + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) { + printf("error in dcnv3_im2col_cuda: %s\n", cudaGetErrorString(err)); + } +} + +template +void dcnv3_col2im_cuda( + cudaStream_t stream, const scalar_t *grad_col, const scalar_t *data_im, + const scalar_t *data_offset, const scalar_t *data_mask, const int kernel_h, + const int kernel_w, const int stride_h, const int stride_w, const int pad_h, + const int pad_w, const int dilation_h, const int dilation_w, + const int group, const int group_channels, const int batch_n, + const int height_in, const int width_in, const int height_out, + const int width_out, const opmath_t offset_scale, const int remove_center, + opmath_t *grad_im, opmath_t *grad_offset, opmath_t *grad_mask) { + const int num_threads = + (group_channels > CUDA_NUM_THREADS) ? CUDA_NUM_THREADS : group_channels; + const int num_kernels = + batch_n * height_out * width_out * group * group_channels; + const int num_actual_kernels = + batch_n * height_out * width_out * group * group_channels; + if (group_channels > 1024) { + if ((group_channels & 1023) == 0) { + dcnv3_col2im_gpu_kernel_shm_reduce_v2_multi_blocks + <<>>( + num_kernels, grad_col, data_im, data_offset, data_mask, + kernel_h, kernel_w, stride_h, stride_w, pad_h, pad_w, + dilation_h, dilation_w, group, group_channels, height_in, + width_in, height_out, width_out, offset_scale, remove_center, grad_im, + grad_offset, grad_mask); + } else { + dcnv3_col2im_gpu_kernel_gm + <<>>(num_kernels, grad_col, data_im, data_offset, + data_mask, kernel_h, kernel_w, stride_h, stride_w, + pad_h, pad_w, dilation_h, dilation_w, group, + group_channels, height_in, width_in, height_out, + width_out, offset_scale, remove_center, grad_im, grad_offset, + grad_mask); + } + } else { + switch (group_channels) { + case 1: + dcnv3_col2im_gpu_kernel_shm_blocksize_aware_reduce_v1 + <<>>(num_kernels, grad_col, data_im, data_offset, + data_mask, kernel_h, kernel_w, stride_h, stride_w, + pad_h, pad_w, dilation_h, dilation_w, group, + group_channels, height_in, width_in, height_out, + width_out, offset_scale, remove_center, grad_im, grad_offset, + grad_mask); + break; + case 2: + dcnv3_col2im_gpu_kernel_shm_blocksize_aware_reduce_v1 + <<>>(num_kernels, grad_col, data_im, data_offset, + data_mask, kernel_h, kernel_w, stride_h, stride_w, + pad_h, pad_w, dilation_h, dilation_w, group, + group_channels, height_in, width_in, height_out, + width_out, offset_scale, remove_center, grad_im, grad_offset, + grad_mask); + break; + case 4: + dcnv3_col2im_gpu_kernel_shm_blocksize_aware_reduce_v1 + <<>>(num_kernels, grad_col, data_im, data_offset, + data_mask, kernel_h, kernel_w, stride_h, stride_w, + pad_h, pad_w, dilation_h, dilation_w, group, + group_channels, height_in, width_in, height_out, + width_out, offset_scale, remove_center, grad_im, grad_offset, + grad_mask); + break; + case 8: + dcnv3_col2im_gpu_kernel_shm_blocksize_aware_reduce_v1 + <<>>(num_kernels, grad_col, data_im, data_offset, + data_mask, kernel_h, kernel_w, stride_h, stride_w, + pad_h, pad_w, dilation_h, dilation_w, group, + group_channels, height_in, width_in, height_out, + width_out, offset_scale, remove_center, grad_im, grad_offset, + grad_mask); + break; + case 16: + dcnv3_col2im_gpu_kernel_shm_blocksize_aware_reduce_v1 + <<>>(num_kernels, grad_col, data_im, data_offset, + data_mask, kernel_h, kernel_w, stride_h, stride_w, + pad_h, pad_w, dilation_h, dilation_w, group, + group_channels, height_in, width_in, height_out, + width_out, offset_scale, remove_center, grad_im, grad_offset, + grad_mask); + break; + case 32: + dcnv3_col2im_gpu_kernel_shm_blocksize_aware_reduce_v1 + <<>>(num_kernels, grad_col, data_im, data_offset, + data_mask, kernel_h, kernel_w, stride_h, stride_w, + pad_h, pad_w, dilation_h, dilation_w, group, + group_channels, height_in, width_in, height_out, + width_out, offset_scale, remove_center, grad_im, grad_offset, + grad_mask); + break; + case 64: + dcnv3_col2im_gpu_kernel_shm_blocksize_aware_reduce_v2 + <<>>(num_kernels, grad_col, data_im, data_offset, + data_mask, kernel_h, kernel_w, stride_h, stride_w, + pad_h, pad_w, dilation_h, dilation_w, group, + group_channels, height_in, width_in, height_out, + width_out, offset_scale, remove_center, grad_im, grad_offset, + grad_mask); + break; + case 128: + dcnv3_col2im_gpu_kernel_shm_blocksize_aware_reduce_v2 + <<>>(num_kernels, grad_col, data_im, data_offset, + data_mask, kernel_h, kernel_w, stride_h, stride_w, + pad_h, pad_w, dilation_h, dilation_w, group, + group_channels, height_in, width_in, height_out, + width_out, offset_scale, remove_center, grad_im, grad_offset, + grad_mask); + break; + case 256: + dcnv3_col2im_gpu_kernel_shm_blocksize_aware_reduce_v2 + <<>>(num_kernels, grad_col, data_im, data_offset, + data_mask, kernel_h, kernel_w, stride_h, stride_w, + pad_h, pad_w, dilation_h, dilation_w, group, + group_channels, height_in, width_in, height_out, + width_out, offset_scale, remove_center, grad_im, grad_offset, + grad_mask); + break; + case 512: + dcnv3_col2im_gpu_kernel_shm_blocksize_aware_reduce_v2 + <<>>(num_kernels, grad_col, data_im, data_offset, + data_mask, kernel_h, kernel_w, stride_h, stride_w, + pad_h, pad_w, dilation_h, dilation_w, group, + group_channels, height_in, width_in, height_out, + width_out, offset_scale, remove_center, grad_im, grad_offset, + grad_mask); + break; + case 1024: + dcnv3_col2im_gpu_kernel_shm_blocksize_aware_reduce_v2 + <<>>(num_kernels, grad_col, data_im, data_offset, + data_mask, kernel_h, kernel_w, stride_h, stride_w, + pad_h, pad_w, dilation_h, dilation_w, group, + group_channels, height_in, width_in, height_out, + width_out, offset_scale, remove_center, grad_im, grad_offset, + grad_mask); + break; + default: + if (group_channels < 64) { + dcnv3_col2im_gpu_kernel_shm_reduce_v1 + <<>>( + num_kernels, grad_col, data_im, data_offset, data_mask, + kernel_h, kernel_w, stride_h, stride_w, pad_h, pad_w, + dilation_h, dilation_w, group, group_channels, + height_in, width_in, height_out, width_out, + offset_scale, remove_center, grad_im, grad_offset, grad_mask); + } else { + dcnv3_col2im_gpu_kernel_shm_reduce_v2 + <<>>( + num_kernels, grad_col, data_im, data_offset, data_mask, + kernel_h, kernel_w, stride_h, stride_w, pad_h, pad_w, + dilation_h, dilation_w, group, group_channels, + height_in, width_in, height_out, width_out, + offset_scale, remove_center, grad_im, grad_offset, grad_mask); + } + } + } + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) { + printf("error in dcnv3_col2im_cuda: %s\n", cudaGetErrorString(err)); + } +} \ No newline at end of file diff --git a/classification/ops_dcnv3/src/dcnv3.h b/classification/ops_dcnv3/src/dcnv3.h new file mode 100644 index 0000000..ce4500f --- /dev/null +++ b/classification/ops_dcnv3/src/dcnv3.h @@ -0,0 +1,59 @@ +/*! +************************************************************************************************** +* InternImage +* Copyright (c) 2022 OpenGVLab +* Licensed under The MIT License [see LICENSE for details] +************************************************************************************************** +* Modified from +*https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +************************************************************************************************** +*/ + +#pragma once + +#include "cpu/dcnv3_cpu.h" + +#ifdef WITH_CUDA +#include "cuda/dcnv3_cuda.h" +#endif + +at::Tensor dcnv3_forward(const at::Tensor &input, const at::Tensor &offset, + const at::Tensor &mask, const int kernel_h, + const int kernel_w, const int stride_h, + const int stride_w, const int pad_h, const int pad_w, + const int dilation_h, const int dilation_w, + const int group, const int group_channels, + const float offset_scale, const int im2col_step, const int remove_center) { + if (input.type().is_cuda()) { +#ifdef WITH_CUDA + return dcnv3_cuda_forward(input, offset, mask, kernel_h, kernel_w, + stride_h, stride_w, pad_h, pad_w, dilation_h, + dilation_w, group, group_channels, + offset_scale, im2col_step, remove_center); +#else + AT_ERROR("Not compiled with GPU support"); +#endif + } + AT_ERROR("Not implemented on the CPU"); +} + +std::vector +dcnv3_backward(const at::Tensor &input, const at::Tensor &offset, + const at::Tensor &mask, const int kernel_h, const int kernel_w, + const int stride_h, const int stride_w, const int pad_h, + const int pad_w, const int dilation_h, const int dilation_w, + const int group, const int group_channels, + const float offset_scale, const at::Tensor &grad_output, + const int im2col_step, const int remove_center) { + if (input.type().is_cuda()) { +#ifdef WITH_CUDA + return dcnv3_cuda_backward(input, offset, mask, kernel_h, kernel_w, + stride_h, stride_w, pad_h, pad_w, dilation_h, + dilation_w, group, group_channels, + offset_scale, grad_output, im2col_step, remove_center); +#else + AT_ERROR("Not compiled with GPU support"); +#endif + } + AT_ERROR("Not implemented on the CPU"); +} diff --git a/classification/ops_dcnv3/src/vision.cpp b/classification/ops_dcnv3/src/vision.cpp new file mode 100644 index 0000000..1f7a908 --- /dev/null +++ b/classification/ops_dcnv3/src/vision.cpp @@ -0,0 +1,17 @@ +/*! +************************************************************************************************** +* InternImage +* Copyright (c) 2022 OpenGVLab +* Licensed under The MIT License [see LICENSE for details] +************************************************************************************************** +* Modified from +*https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +************************************************************************************************** +*/ + +#include "dcnv3.h" + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("dcnv3_forward", &dcnv3_forward, "dcnv3_forward"); + m.def("dcnv3_backward", &dcnv3_backward, "dcnv3_backward"); +} diff --git a/classification/ops_dcnv3/test.py b/classification/ops_dcnv3/test.py new file mode 100644 index 0000000..e2174d2 --- /dev/null +++ b/classification/ops_dcnv3/test.py @@ -0,0 +1,264 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + +from __future__ import absolute_import +from __future__ import print_function +from __future__ import division + +import time +import torch +import torch.nn as nn +import math +from torch.autograd import gradcheck + +from functions.dcnv3_func import DCNv3Function, dcnv3_core_pytorch + +H_in, W_in = 8, 8 +N, M, D = 2, 4, 16 +Kh, Kw = 3, 3 +remove_center = False +P = Kh * Kw - remove_center +offset_scale = 2.0 +pad = 1 +dilation = 1 +stride = 1 +H_out = (H_in + 2 * pad - (dilation * (Kh - 1) + 1)) // stride + 1 +W_out = (W_in + 2 * pad - (dilation * (Kw - 1) + 1)) // stride + 1 + +torch.manual_seed(3) + + +@torch.no_grad() +def check_forward_equal_with_pytorch_double(): + input = torch.rand(N, H_in, W_in, M*D).cuda() * 0.01 + offset = torch.rand(N, H_out, W_out, M*P*2).cuda() * 10 + mask = torch.rand(N, H_out, W_out, M, P).cuda() + 1e-5 + mask /= mask.sum(-1, keepdim=True) + mask = mask.reshape(N, H_out, W_out, M*P) + + output_pytorch = dcnv3_core_pytorch( + input.double(), + offset.double(), + mask.double(), + Kh, Kw, stride, stride, Kh // 2, Kw // 2, dilation, dilation, M, D, offset_scale, remove_center).detach().cpu() + + im2col_step = 2 + output_cuda = DCNv3Function.apply( + input.double(), + offset.double(), + mask.double(), + Kh, Kw, stride, stride, Kh // 2, Kw // 2, dilation, dilation, M, D, offset_scale, + im2col_step, remove_center).detach().cpu() + + fwdok = torch.allclose(output_cuda, output_pytorch) + max_abs_err = (output_cuda - output_pytorch).abs().max() + max_rel_err = ((output_cuda - output_pytorch).abs() / + output_pytorch.abs()).max() + print('>>> forward double') + print(f'* {fwdok} check_forward_equal_with_pytorch_double: max_abs_err {max_abs_err:.2e} max_rel_err {max_rel_err:.2e}') + + +@torch.no_grad() +def check_forward_equal_with_pytorch_float(): + input = torch.rand(N, H_in, W_in, M*D).cuda() * 0.01 + offset = torch.rand(N, H_out, W_out, M*P*2).cuda() * 10 + mask = torch.rand(N, H_out, W_out, M, P).cuda() + 1e-5 + mask /= mask.sum(-1, keepdim=True) + mask = mask.reshape(N, H_out, W_out, M*P) + + output_pytorch = dcnv3_core_pytorch( + input, + offset, + mask, + Kh, Kw, stride, stride, Kh // 2, Kw // 2, dilation, dilation, M, D, offset_scale, remove_center).detach().cpu() + + im2col_step = 2 + output_cuda = DCNv3Function.apply( + input, + offset, + mask, + Kh, Kw, stride, stride, Kh // 2, Kw // 2, dilation, dilation, M, D, offset_scale, + im2col_step, remove_center).detach().cpu() + + fwdok = torch.allclose(output_cuda, output_pytorch, rtol=1e-2, atol=1e-3) + max_abs_err = (output_cuda - output_pytorch).abs().max() + max_rel_err = ((output_cuda - output_pytorch).abs() / + output_pytorch.abs()).max() + print('>>> forward float') + print(f'* {fwdok} check_forward_equal_with_pytorch_float: max_abs_err {max_abs_err:.2e} max_rel_err {max_rel_err:.2e}') + + +def check_backward_equal_with_pytorch_double(channels=4, grad_input=True, grad_offset=True, grad_mask=True): + # H_in, W_in = 4, 4 + N = 2 + M = 2 + H_out = (H_in + 2 * pad - (dilation * (Kh - 1) + 1)) // stride + 1 + W_out = (W_in + 2 * pad - (dilation * (Kw - 1) + 1)) // stride + 1 + + D = channels + input0 = torch.rand(N, H_in, W_in, M*D).cuda() * 0.01 + offset0 = torch.rand(N, H_out, W_out, M*P*2).cuda() * 10 + mask0 = torch.rand(N, H_out, W_out, M, P).cuda() + 1e-5 + mask0 /= mask0.sum(-1, keepdim=True) + mask0 = mask0.reshape(N, H_out, W_out, M*P) + input0.requires_grad = grad_input + offset0.requires_grad = grad_offset + mask0.requires_grad = grad_mask + + output_pytorch = dcnv3_core_pytorch( + input0.double(), + offset0.double(), + mask0.double(), + Kh, Kw, stride, stride, Kh // 2, Kw // 2, dilation, dilation, M, D, offset_scale, remove_center) + output_pytorch.sum().backward() + + input1 = input0.detach() + offset1 = offset0.detach() + mask1 = mask0.detach() + input1.requires_grad = grad_input + offset1.requires_grad = grad_offset + mask1.requires_grad = grad_mask + + im2col_step = 2 + output_cuda = DCNv3Function.apply( + input1.double(), + offset1.double(), + mask1.double(), + Kh, Kw, stride, stride, Kh // 2, Kw // 2, dilation, dilation, M, D, offset_scale, + im2col_step, remove_center) + output_cuda.sum().backward() + + print(f'>>> backward double: channels {D}') + bwdok = torch.allclose(input0.grad, input1.grad, rtol=1e-2, atol=1e-3) + max_abs_err = (input0.grad - input1.grad).abs().max() + max_rel_err = ((input0.grad - input1.grad).abs() / + input0.grad.abs()).max() + print( + f'* {bwdok} input_grad check_backward_equal_with_pytorch_double: max_abs_err {max_abs_err:.2e} max_rel_err {max_rel_err:.2e}') + + bwdok = torch.allclose(offset0.grad, offset1.grad, rtol=1e-2, atol=1e-3) + max_abs_err = (offset0.grad - offset1.grad).abs().max() + max_rel_err = ((offset0.grad - offset1.grad).abs() / + offset0.grad.abs()).max() + print( + f'* {bwdok} offset_grad check_backward_equal_with_pytorch_double: max_abs_err {max_abs_err:.2e} max_rel_err {max_rel_err:.2e}') + + bwdok = torch.allclose(mask0.grad, mask1.grad, rtol=1e-2, atol=1e-3) + max_abs_err = (mask0.grad - mask1.grad).abs().max() + max_rel_err = ((mask0.grad - mask1.grad).abs() / + mask0.grad.abs()).max() + print( + f'* {bwdok} mask_grad check_backward_equal_with_pytorch_double: max_abs_err {max_abs_err:.2e} max_rel_err {max_rel_err:.2e}') + + +def check_backward_equal_with_pytorch_float(channels=4, grad_input=True, grad_offset=True, grad_mask=True): + # H_in, W_in = 4, 4 + N = 2 + M = 2 + H_out = (H_in + 2 * pad - (dilation * (Kh - 1) + 1)) // stride + 1 + W_out = (W_in + 2 * pad - (dilation * (Kw - 1) + 1)) // stride + 1 + + D = channels + input0 = torch.rand(N, H_in, W_in, M*D).cuda() * 0.01 + offset0 = torch.rand(N, H_out, W_out, M*P*2).cuda() * 10 + mask0 = torch.rand(N, H_out, W_out, M, P).cuda() + 1e-5 + mask0 /= mask0.sum(-1, keepdim=True) + mask0 = mask0.reshape(N, H_out, W_out, M*P) + input0.requires_grad = grad_input + offset0.requires_grad = grad_offset + mask0.requires_grad = grad_mask + + output_pytorch = dcnv3_core_pytorch( + input0, + offset0, + mask0, + Kh, Kw, stride, stride, Kh // 2, Kw // 2, dilation, dilation, M, D, offset_scale, remove_center) + output_pytorch.sum().backward() + + input1 = input0.detach() + offset1 = offset0.detach() + mask1 = mask0.detach() + input1.requires_grad = grad_input + offset1.requires_grad = grad_offset + mask1.requires_grad = grad_mask + + im2col_step = 2 + output_cuda = DCNv3Function.apply( + input1, + offset1, + mask1, + Kh, Kw, stride, stride, Kh // 2, Kw // 2, dilation, dilation, M, D, offset_scale, + im2col_step, remove_center) + output_cuda.sum().backward() + + print(f'>>> backward float: channels {D}') + bwdok = torch.allclose(input0.grad, input1.grad, rtol=1e-2, atol=1e-3) + max_abs_err = (input0.grad - input1.grad).abs().max() + max_rel_err = ((input0.grad - input1.grad).abs() / + input0.grad.abs()).max() + print( + f'* {bwdok} input_grad check_backward_equal_with_pytorch_float: max_abs_err {max_abs_err:.2e} max_rel_err {max_rel_err:.2e}') + + bwdok = torch.allclose(offset0.grad, offset1.grad, rtol=1e-2, atol=1e-3) + max_abs_err = (offset0.grad - offset1.grad).abs().max() + max_rel_err = ((offset0.grad - offset1.grad).abs() / + offset0.grad.abs()).max() + print( + f'* {bwdok} offset_grad check_backward_equal_with_pytorch_float: max_abs_err {max_abs_err:.2e} max_rel_err {max_rel_err:.2e}') + + bwdok = torch.allclose(mask0.grad, mask1.grad, rtol=1e-2, atol=1e-3) + max_abs_err = (mask0.grad - mask1.grad).abs().max() + max_rel_err = ((mask0.grad - mask1.grad).abs() / + mask0.grad.abs()).max() + print( + f'* {bwdok} mask_grad check_backward_equal_with_pytorch_float: max_abs_err {max_abs_err:.2e} max_rel_err {max_rel_err:.2e}') + + +@torch.no_grad() +def check_time_cost(im2col_step=128): + N = 512 + H_in, W_in = 64, 64 + H_out = (H_in + 2 * pad - (dilation * (Kh - 1) + 1)) // stride + 1 + W_out = (W_in + 2 * pad - (dilation * (Kw - 1) + 1)) // stride + 1 + + input = torch.rand(N, H_in, W_in, M*D).cuda() * 0.01 + offset = torch.rand(N, H_out, W_out, M*P*2).cuda() * 10 + mask = torch.rand(N, H_out, W_out, M, P).cuda() + 1e-5 + mask /= mask.sum(-1, keepdim=True) + mask = mask.reshape(N, H_out, W_out, M*P) + print( + f'>>> time cost: im2col_step {im2col_step}; input {input.shape}; points {P} ') + repeat = 100 + for i in range(repeat): + output_cuda = DCNv3Function.apply( + input, + offset, + mask, + Kh, Kw, stride, stride, Kh // 2, Kw // 2, dilation, dilation, M, D, 1.0, + im2col_step, remove_center) + torch.cuda.synchronize() + start = time.time() + for i in range(repeat): + output_cuda = DCNv3Function.apply( + input, + offset, + mask, + Kh, Kw, stride, stride, Kh // 2, Kw // 2, dilation, dilation, M, D, 1.0, + im2col_step, remove_center) + torch.cuda.synchronize() + print(f'foward time cost: {(time.time() - start) / repeat}') + + +if __name__ == '__main__': + check_forward_equal_with_pytorch_double() + check_forward_equal_with_pytorch_float() + for channels in [1, 16, 30, 32, 64, 71, 1025]: + check_backward_equal_with_pytorch_double(channels, True, True, True) + for channels in [1, 16, 30, 32, 64, 71, 1025]: + check_backward_equal_with_pytorch_float(channels, True, True, True) + for i in range(3): + im2col_step = 128 * (2 ** i) + check_time_cost(im2col_step) diff --git a/classification/optimizer.py b/classification/optimizer.py new file mode 100644 index 0000000..70ee30a --- /dev/null +++ b/classification/optimizer.py @@ -0,0 +1,159 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + + +from torch import optim as optim +from torch.distributed.optim import ZeroRedundancyOptimizer + + +def build_optimizer(config, model): + """ + Build optimizer, set weight decay of normalization to 0 by default. + """ + skip = {} + skip_keywords = {} + if hasattr(model, 'no_weight_decay'): + skip = model.no_weight_decay() + if hasattr(model, 'no_weight_decay_keywords'): + skip_keywords = model.no_weight_decay_keywords() + + parameters = set_weight_decay_and_lr( + model, + config.TRAIN.WEIGHT_DECAY, + config.TRAIN.BASE_LR, + skip, + skip_keywords, + lr_layer_decay=config.TRAIN.LR_LAYER_DECAY, + lr_layer_decay_ratio=config.TRAIN.LR_LAYER_DECAY_RATIO, + freeze_backbone=config.TRAIN.OPTIMIZER.FREEZE_BACKBONE, + dcn_lr_mul=config.TRAIN.OPTIMIZER.DCN_LR_MUL, + ) + + opt_lower = config.TRAIN.OPTIMIZER.NAME.lower() + optimizer = None + use_zero = config.TRAIN.OPTIMIZER.USE_ZERO + if use_zero: + print(f"\nUse Zero!") + if opt_lower == 'sgd': + # an ugly implementation + # this problem is fixed after torch 1.12 + # https://github.com/pytorch/pytorch/issues/71347 + + # before 1.12, we could only pass list to zero optimizer, so we first pass parameters[0] with its lr and weight decay, + # then we add other parameter via parameter group. + + optimizer = ZeroRedundancyOptimizer( + parameters[0]['params'], + optimizer_class=optim.SGD, + momentum=config.TRAIN.OPTIMIZER.MOMENTUM, nesterov=True, + lr=parameters[0]['lr'], weight_decay=parameters[0]['weight_decay'] + ) + if len(parameters) > 1: + for param_group in parameters[1:]: + optimizer.add_param_group(param_group) + elif opt_lower == 'adamw': + optimizer = ZeroRedundancyOptimizer( + parameters[0]['params'], + optimizer_class=optim.AdamW, + eps=config.TRAIN.OPTIMIZER.EPS, betas=config.TRAIN.OPTIMIZER.BETAS, + lr=parameters[0]['lr'], weight_decay=parameters[0]['weight_decay'] + ) + if len(parameters) > 1: + for param_group in parameters[1:]: + optimizer.add_param_group(param_group) + else: + if opt_lower == 'sgd': + optimizer = optim.SGD(parameters, + momentum=config.TRAIN.OPTIMIZER.MOMENTUM, + nesterov=True, + lr=config.TRAIN.BASE_LR, + weight_decay=config.TRAIN.WEIGHT_DECAY) + elif opt_lower == 'adamw': + optimizer = optim.AdamW(parameters, + eps=config.TRAIN.OPTIMIZER.EPS, + betas=config.TRAIN.OPTIMIZER.BETAS, + lr=config.TRAIN.BASE_LR, + weight_decay=config.TRAIN.WEIGHT_DECAY) + + return optimizer + + +def check_keywords_in_name(name, keywords=()): + isin = False + for keyword in keywords: + if keyword in name: + isin = True + return isin + + +def check_keywords_in_dict(name, keywords_dict): + for k, v in keywords_dict.items(): + if k in name: + return v + return None + + +def set_weight_decay_and_lr( + model, + weight_decay, + base_lr, + skip_list=(), + skip_keywords=(), + lr_layer_decay=None, + lr_layer_decay_ratio=None, + freeze_backbone=None, + dcn_lr_mul=None, + layerwise_lr=True, +): + parameters = [] + no_decay_name = [] + lr_ratio_log = {} + + for name, param in model.named_parameters(): + if not param.requires_grad: + continue # frozen weights + if freeze_backbone: + for i in freeze_backbone: + if f'levels.{i}' in name: + param.requires_grad = False + # 1. check wd + if len(param.shape) == 1 or name.endswith(".bias") or ( + name in skip_list) or check_keywords_in_name( + name, skip_keywords): + wd = 0. + no_decay_name.append(name) + else: + wd = weight_decay + + if lr_layer_decay: + print('layer-wise lr decay is used !') + assert hasattr(model, 'lr_decay_keywards') + lr_ratio_keywards = model.lr_decay_keywards(lr_layer_decay_ratio) + + # 2. check lr + ratio = check_keywords_in_dict(name, lr_ratio_keywards) + if ratio is not None: + lr = ratio * base_lr + else: + lr = base_lr + + # dcn lr + if dcn_lr_mul is not None: + if 'offset' in name or 'attention_weights' in name or 'center_feature_scale_proj' in name or 'alpha_beta' in name: + lr = dcn_lr_mul * lr + + lr_ratio_log[name] = (base_lr, ratio, wd, param.requires_grad) + else: + lr = base_lr + parameters.append({'params': [param], 'weight_decay': wd, 'lr': lr, 'name': name}) + + print('no decay params: {no_decay_name}') + if layerwise_lr: + print('lr_ratio_params:') + for k, v in lr_ratio_log.items(): + print(k, v) + + return parameters diff --git a/classification/train_in1k.sh b/classification/train_in1k.sh new file mode 100755 index 0000000..3b35a1e --- /dev/null +++ b/classification/train_in1k.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +set -x + +PARTITION=$1 +JOB_NAME=$2 +CONFIG=$3 +WORK_DIR=$4 +GPUS=${GPUS:-1} +GPUS_PER_NODE=${GPUS_PER_NODE:-1} +CPUS_PER_TASK=${CPUS_PER_TASK:-10} +SRUN_ARGS=${SRUN_ARGS:-""} +PY_ARGS=${@:5} + +PYTHONPATH="$(dirname $0)/..":$PYTHONPATH \ +srun -p ${PARTITION} \ + --job-name=${JOB_NAME} \ + --gres=gpu:${GPUS_PER_NODE} \ + --ntasks=${GPUS} \ + --ntasks-per-node=${GPUS_PER_NODE} \ + --cpus-per-task=${CPUS_PER_TASK} \ + --kill-on-bad-exit=1 \ + --quotatype=reserved \ + ${SRUN_ARGS} \ +python -u main.py \ + --cfg ${CONFIG} \ + --accumulation-steps 1 \ + --local-rank 0 \ + --batch-size 128 \ + --data-path /mnt/petrelfs/share/images \ + --output work_dirs ${@:4} --launcher="slurm" diff --git a/classification/train_in1k_deepspeed.sh b/classification/train_in1k_deepspeed.sh new file mode 100644 index 0000000..788e8b8 --- /dev/null +++ b/classification/train_in1k_deepspeed.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +set -x + +PARTITION=$1 +JOB_NAME=$2 +CONFIG=$3 +GPUS=${GPUS:-8} +GPUS_PER_NODE=${GPUS_PER_NODE:-8} +CPUS_PER_TASK=${CPUS_PER_TASK:-12} +SRUN_ARGS=${SRUN_ARGS:-""} + +PYTHONPATH="$(dirname $0)/..":$PYTHONPATH \ + srun -p ${PARTITION} \ + --job-name=${JOB_NAME} \ + --gres=gpu:${GPUS_PER_NODE} \ + --ntasks=${GPUS} \ + --ntasks-per-node=${GPUS_PER_NODE} \ + --cpus-per-task=${CPUS_PER_TASK} \ + --kill-on-bad-exit=1 \ + --quotatype=spot \ + ${SRUN_ARGS} \ + python -u main_deepspeed.py \ + --cfg ${CONFIG} \ + --local-rank 0 \ + --data-path /mnt/lustre/share/images \ + --output work_dirs_deepspeed ${@:4} diff --git a/classification/utils.py b/classification/utils.py new file mode 100644 index 0000000..3122cef --- /dev/null +++ b/classification/utils.py @@ -0,0 +1,423 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + +import os +import math +import torch +import numpy as np +import torch.distributed as dist +from collections import OrderedDict +from timm.utils import get_state_dict +try: + # noinspection PyUnresolvedReferences + from apex import amp +except ImportError: + amp = None + + +def load_ema_checkpoint(config, model_ema, logger): + logger.info( + f'==============> Resuming form {config.MODEL.RESUME}....................' + ) + if config.MODEL.RESUME.startswith('https'): + checkpoint = torch.hub.load_state_dict_from_url(config.MODEL.RESUME, + map_location='cpu', + check_hash=True) + else: + checkpoint = torch.load(config.MODEL.RESUME, map_location='cpu') + + assert isinstance(checkpoint, dict) + if 'model_ema' in checkpoint: + new_state_dict = OrderedDict() + for k, v in checkpoint['model_ema'].items(): + if model_ema.ema_has_module: + name = 'module.' + k if not k.startswith('module') else k + else: + name = k + new_state_dict[name] = v + msg = model_ema.ema.load_state_dict(new_state_dict, strict=False) + logger.info(msg) + logger.info('Loaded state_dict_ema') + else: + logger.warning( + 'Failed to find state_dict_ema, starting from loaded model weights' + ) + + max_accuracy_ema = 0 + if 'max_accuracy_ema' in checkpoint: + max_accuracy_ema = checkpoint['max_accuracy_ema'] + if 'ema_decay' in checkpoint: + model_ema.decay = checkpoint['ema_decay'] + return max_accuracy_ema + + +def load_checkpoint(config, model, optimizer, lr_scheduler, scaler, logger): + logger.info( + f'==============> Resuming form {config.MODEL.RESUME}....................' + ) + if config.MODEL.RESUME.startswith('https'): + checkpoint = torch.hub.load_state_dict_from_url(config.MODEL.RESUME, + map_location='cpu', + check_hash=True) + else: + checkpoint = torch.load(config.MODEL.RESUME, map_location='cpu') + + print('resuming model') + msg = model.load_state_dict(checkpoint['model'], strict=False) + logger.info(msg) + max_accuracy = 0.0 + if not config.EVAL_MODE and 'optimizer' in checkpoint and 'lr_scheduler' in checkpoint and 'epoch' in checkpoint: + if optimizer is not None: + print('resuming optimizer') + try: + optimizer.load_state_dict(checkpoint['optimizer']) + except: + print('resume optimizer failed') + if lr_scheduler is not None: + print('resuming lr_scheduler') + lr_scheduler.load_state_dict(checkpoint['lr_scheduler']) + config.defrost() + config.TRAIN.START_EPOCH = checkpoint['epoch'] + 1 + config.freeze() + if 'amp' in checkpoint and config.AMP_OPT_LEVEL != 'O0' and checkpoint[ + 'config'].AMP_OPT_LEVEL != 'O0': + scaler.load_state_dict(checkpoint['amp']) + logger.info( + f"=> loaded successfully {config.MODEL.RESUME} (epoch {checkpoint['epoch']})" + ) + if 'max_accuracy' in checkpoint: + max_accuracy = checkpoint['max_accuracy'] + + del checkpoint + torch.cuda.empty_cache() + + return max_accuracy + + +def load_pretrained(config, model, logger): + logger.info( + f'==============> Loading weight {config.MODEL.PRETRAINED} for fine-tuning......' + ) + checkpoint = torch.load(config.MODEL.PRETRAINED, map_location='cpu') + + state_dict = checkpoint + if 'model' in checkpoint: + state_dict = checkpoint['model'] + elif 'module' in checkpoint: + state_dict = checkpoint['module'] + + first_key = list(state_dict.keys())[0] + # delete teacher weights + if 'student' in first_key or 'teacher' in first_key: + new_state_dict = OrderedDict() + for k, v in state_dict.items(): + if 'student_proj' in k: + continue + if 'student' in k: + new_k = k.replace('student.', '') + new_state_dict[new_k] = v + state_dict = new_state_dict + + # weights from sim + if 'mask_token' in first_key: + new_state_dict = OrderedDict() + for k, v in state_dict.items(): + if 'mm_dcnv3' in k: + continue + if 'dcnv3' not in k and 'clip_projector' not in k: + continue + new_k = k.replace('dcnv3.', '') + new_state_dict[new_k] = v + new_state_dict['fc_norm.weight'] = state_dict[ + 'clip.classifier_ln.weight'] + new_state_dict['fc_norm.bias'] = state_dict['clip.classifier_ln.bias'] + new_state_dict['head.weight'] = state_dict['clip.classifier.weight'] + new_state_dict['head.bias'] = state_dict['clip.classifier.bias'] + state_dict = new_state_dict + + # delete relative_position_index since we always re-init it + relative_position_index_keys = [ + k for k in state_dict.keys() if 'relative_position_index' in k + ] + for k in relative_position_index_keys: + del state_dict[k] + + # delete relative_coords_table since we always re-init it + relative_position_index_keys = [ + k for k in state_dict.keys() if 'relative_coords_table' in k + ] + for k in relative_position_index_keys: + del state_dict[k] + + # delete attn_mask since we always re-init it + attn_mask_keys = [k for k in state_dict.keys() if 'attn_mask' in k] + for k in attn_mask_keys: + del state_dict[k] + + # bicubic interpolate relative_position_bias_table if not match + relative_position_bias_table_keys = [ + k for k in state_dict.keys() if 'relative_position_bias_table' in k + ] + for k in relative_position_bias_table_keys: + relative_position_bias_table_pretrained = state_dict[k] + relative_position_bias_table_current = model.state_dict()[k] + L1, nH1 = relative_position_bias_table_pretrained.size() + L2, nH2 = relative_position_bias_table_current.size() + if nH1 != nH2: + logger.warning(f'Error in loading {k}, passing......') + else: + if L1 != L2: + # bicubic interpolate relative_position_bias_table if not match + S1 = int(L1**0.5) + S2 = int(L2**0.5) + relative_position_bias_table_pretrained_resized = torch.nn.functional.interpolate( + relative_position_bias_table_pretrained.permute(1, 0).view( + 1, nH1, S1, S1), + size=(S2, S2), + mode='bicubic') + state_dict[ + k] = relative_position_bias_table_pretrained_resized.view( + nH2, L2).permute(1, 0) + + # bicubic interpolate absolute_pos_embed if not match + absolute_pos_embed_keys = [ + k for k in state_dict.keys() if 'absolute_pos_embed' in k + ] + for k in absolute_pos_embed_keys: + # dpe + absolute_pos_embed_pretrained = state_dict[k] + absolute_pos_embed_current = model.state_dict()[k] + _, L1, C1 = absolute_pos_embed_pretrained.size() + _, L2, C2 = absolute_pos_embed_current.size() + if C1 != C1: + logger.warning(f'Error in loading {k}, passing......') + else: + if L1 != L2: + S1 = int(L1**0.5) + S2 = int(L2**0.5) + absolute_pos_embed_pretrained = absolute_pos_embed_pretrained.reshape( + -1, S1, S1, C1) + absolute_pos_embed_pretrained = absolute_pos_embed_pretrained.permute( + 0, 3, 1, 2) + absolute_pos_embed_pretrained_resized = torch.nn.functional.interpolate( + absolute_pos_embed_pretrained, + size=(S2, S2), + mode='bicubic') + absolute_pos_embed_pretrained_resized = absolute_pos_embed_pretrained_resized.permute( + 0, 2, 3, 1) + absolute_pos_embed_pretrained_resized = absolute_pos_embed_pretrained_resized.flatten( + 1, 2) + state_dict[k] = absolute_pos_embed_pretrained_resized + + # check classifier, if not match, then re-init classifier to zero + if 'head.bias' in state_dict: + head_bias_pretrained = state_dict['head.bias'] + Nc1 = head_bias_pretrained.shape[0] + Nc2 = model.head.bias.shape[0] + + if (Nc1 != Nc2): + if config.TRAIN.RAND_INIT_FT_HEAD: + model.head.weight.data = model.head.weight.data * 0.001 + model.head.bias.data = model.head.bias.data * 0.001 + del state_dict['head.weight'] + del state_dict['head.bias'] + logger.warning( + f'Error in loading classifier head, re-init classifier head to 0' + ) + elif Nc1 == 21841 and Nc2 == 1000: + logger.info( + 'loading ImageNet-22K weight to ImageNet-1K ......') + map22kto1k_path = 'meta_data/map22kto1k.txt' + logger.info(map22kto1k_path) + with open(map22kto1k_path) as f: + map22kto1k = f.readlines() + map22kto1k = [int(id22k.strip()) for id22k in map22kto1k] + state_dict['head.weight'] = state_dict['head.weight'][ + map22kto1k, :] + state_dict['head.bias'] = state_dict['head.bias'][map22kto1k] + + msg = model.load_state_dict(state_dict, strict=False) + logger.warning(msg) + + # from IPython import embed + # embed() + + logger.info(f'=> loaded successfully {config.MODEL.PRETRAINED}') + + del checkpoint + torch.cuda.empty_cache() + + +def convert_22k_head_to_1k(model, logger): + head_weight = model.module.head.weight + head_bias = model.module.head.bias + Nc1 = head_bias.shape[0] + + if Nc1 == 21841: + logger.info('converting ImageNet-22K head to ImageNet-1K ......') + map22kto1k_path = 'meta_data/map22kto1k.txt' + logger.info(map22kto1k_path) + with open(map22kto1k_path) as f: + map22kto1k = f.readlines() + map22kto1k = [int(id22k.strip()) for id22k in map22kto1k] + model.module.head.weight = torch.nn.Parameter( + head_weight[map22kto1k, :]) + model.module.head.bias = torch.nn.Parameter(head_bias[map22kto1k]) + else: + logger.warning(f'Error in converting classifier head') + + return model + + +def save_checkpoint(config, + epoch, + model, + max_accuracy, + optimizer, + lr_scheduler, + scaler, + logger, + model_ema=None, + max_accuracy_ema=None, + ema_decay=None, + model_ems=None, + max_accuracy_ems=None, + ems_model_num=None, + best=None): + + save_state = { + 'model': model.state_dict(), + 'optimizer': optimizer.state_dict(), + 'lr_scheduler': lr_scheduler.state_dict(), + 'max_accuracy': max_accuracy, + 'epoch': epoch, + 'config': config + } + if model_ema is not None: + save_state['model_ema'] = get_state_dict(model_ema) + if max_accuracy_ema is not None: + save_state['max_accuracy_ema'] = max_accuracy_ema + if ema_decay is not None: + save_state['ema_decay'] = ema_decay + if model_ems is not None: + save_state['model_ems'] = get_state_dict(model_ems) + if max_accuracy_ems is not None: + save_state['max_accuracy_ems'] = max_accuracy_ems + if ems_model_num is not None: + save_state['ems_model_num'] = ems_model_num + if config.AMP_OPT_LEVEL != 'O0': + # save_state['amp'] = amp.state_dict() + save_state['amp'] = scaler.state_dict() + if best is None: + save_path = os.path.join(config.OUTPUT, f'ckpt_epoch_{epoch}.pth') + else: + save_path = os.path.join(config.OUTPUT, f'ckpt_epoch_{best}.pth') + logger.info(f'{save_path} saving......') + torch.save(save_state, save_path) + logger.info(f'{save_path} saved !!!') + + if dist.get_rank() == 0 and isinstance(epoch, int): + to_del = epoch - config.SAVE_CKPT_NUM * config.SAVE_FREQ + old_ckpt = os.path.join(config.OUTPUT, f'ckpt_epoch_{to_del}.pth') + if os.path.exists(old_ckpt): + os.remove(old_ckpt) + + +def get_grad_norm(parameters, norm_type=2): + if isinstance(parameters, torch.Tensor): + parameters = [parameters] + parameters = list(filter(lambda p: p.grad is not None, parameters)) + norm_type = float(norm_type) + total_norm = 0 + for p in parameters: + param_norm = p.grad.data.norm(norm_type) + total_norm += param_norm.item()**norm_type + total_norm = total_norm**(1. / norm_type) + return total_norm + + +def auto_resume_helper(output_dir): + checkpoints = os.listdir(output_dir) + checkpoints = [ckpt for ckpt in checkpoints if ckpt.endswith('pth')] + print(f'All checkpoints founded in {output_dir}: {checkpoints}') + if len(checkpoints) > 0: + latest_checkpoint = max( + [os.path.join(output_dir, d) for d in checkpoints], + key=os.path.getmtime) + print(f'The latest checkpoint founded: {latest_checkpoint}') + resume_file = latest_checkpoint + else: + resume_file = None + return resume_file + + +def reduce_tensor(tensor): + rt = tensor.clone() + dist.all_reduce(rt, op=dist.ReduceOp.SUM) + rt /= dist.get_world_size() + return rt + + +# https://github.com/facebookresearch/ConvNeXt/blob/main/utils.py +class NativeScalerWithGradNormCount: + state_dict_key = 'amp_scaler' + + def __init__(self): + self._scaler = torch.cuda.amp.GradScaler() + + def __call__(self, + loss, + optimizer, + clip_grad=None, + parameters=None, + create_graph=False, + update_grad=True): + self._scaler.scale(loss).backward(create_graph=create_graph) + if update_grad: + if clip_grad is not None: + assert parameters is not None + self._scaler.unscale_( + optimizer + ) # unscale the gradients of optimizer's assigned params in-place + norm = torch.nn.utils.clip_grad_norm_(parameters, clip_grad) + else: + self._scaler.unscale_(optimizer) + norm = get_grad_norm(parameters) + self._scaler.step(optimizer) + self._scaler.update() + else: + norm = None + return norm + + def state_dict(self): + return self._scaler.state_dict() + + def load_state_dict(self, state_dict): + self._scaler.load_state_dict(state_dict) + + +class MyAverageMeter(object): + """Computes and stores the average and current value.""" + + def __init__(self, max_len=-1): + self.val_list = [] + self.count = [] + self.max_len = max_len + self.val = 0 + self.avg = 0 + self.var = 0 + + def update(self, val): + self.val = val + self.avg = 0 + self.var = 0 + if not math.isnan(val) and not math.isinf(val): + self.val_list.append(val) + if self.max_len > 0 and len(self.val_list) > self.max_len: + self.val_list = self.val_list[-self.max_len:] + if len(self.val_list) > 0: + self.avg = np.mean(np.array(self.val_list)) + self.var = np.std(np.array(self.val_list)) diff --git a/detection/README.md b/detection/README.md new file mode 100644 index 0000000..a4c0094 --- /dev/null +++ b/detection/README.md @@ -0,0 +1,93 @@ +# FlashInternImage for Object Detection + +This folder contains the implementation of the FlashInternImage for object detection. + +Our detection code is developed on top of [MMDetection v2.28.1](https://github.com/open-mmlab/mmdetection/tree/v2.28.1). + + +## Usage + +### Install + +- Clone this repo: + +```bash +git clone https://github.com/OpenGVLab/DCNv4.git +cd DCNv4 +``` + +- Create a conda virtual environment and activate it: + +```bash +conda create -n dcnv4 python=3.7 -y +conda activate dcnv4 +``` + +- Install `CUDA>=10.2` with `cudnn>=7` following + the [official installation instructions](https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html) +- Install `PyTorch>=1.10.0` and `torchvision>=0.9.0` with `CUDA>=10.2`: + +For examples, to install torch==1.11 with CUDA==11.3: +```bash +pip install torch==1.11.0+cu113 torchvision==0.12.0+cu113 -f https://download.pytorch.org/whl/torch_stable.html +``` + +- Install `timm==0.6.11` and `mmcv-full==1.5.0`: + +```bash +pip install -U openmim +mim install mmcv-full==1.5.0 +pip install timm==0.6.11 mmdet==2.28.1 +``` + +- Install other requirements: + +```bash +pip install opencv-python termcolor yacs pyyaml scipy +``` + +- Install DCNv4 +```bash +pip install DCNv4==latest +``` + + +### Data Preparation + +Prepare COCO according to the guidelines in [MMDetection v2.28.1](https://github.com/open-mmlab/mmdetection/resolve/master/docs/en/1_exist_data_model.md). + + +### Evaluation + +To evaluate our `FlashInternImage` on COCO val, run: + +```bash +sh dist_test.sh --eval bbox segm +``` + +For example, to evaluate the `FlashInternImage-T` with a single GPU: + +```bash +python test.py configs/coco/mask_rcnn_flash_intern_image_t_fpn_1x_coco.py checkpoint_dir/det/mask_rcnn_flash_internimage_t_fpn_1x_coco.pth --eval bbox segm +``` + +For example, to evaluate the `FlashInternImage-B` with a single node with 8 GPUs: + +```bash +sh dist_test.sh configs/coco/mask_rcnn_flash_intern_image_b_fpn_1x_coco.py checkpoint_dir/det/mask_rcnn_flash_internimage_b_fpn_1x_coco.py 8 --eval bbox segm +``` + +### Training on COCO + +To train an `FlashInternImage` on COCO, run: + +```bash +sh dist_train.sh +``` + +For example, to train `FlashInternImage-T` with 8 GPU on 1 node, run: + +```bash +sh dist_train.sh configs/coco/mask_rcnn_flash_intern_image_t_fpn_1x_coco.py 8 +``` + diff --git a/detection/configs/_base_/datasets/coco_detection.py b/detection/configs/_base_/datasets/coco_detection.py new file mode 100644 index 0000000..737e50c --- /dev/null +++ b/detection/configs/_base_/datasets/coco_detection.py @@ -0,0 +1,49 @@ +# dataset settings +dataset_type = 'CocoDataset' +data_root = 'data/coco/' +img_norm_cfg = dict( + mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) +train_pipeline = [ + dict(type='LoadImageFromFile'), + dict(type='LoadAnnotations', with_bbox=True), + dict(type='Resize', img_scale=(1333, 800), keep_ratio=True), + dict(type='RandomFlip', flip_ratio=0.5), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size_divisor=32), + dict(type='DefaultFormatBundle'), + dict(type='Collect', keys=['img', 'gt_bboxes', 'gt_labels']), +] +test_pipeline = [ + dict(type='LoadImageFromFile'), + dict( + type='MultiScaleFlipAug', + img_scale=(1333, 800), + flip=False, + transforms=[ + dict(type='Resize', keep_ratio=True), + dict(type='RandomFlip'), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size_divisor=32), + dict(type='ImageToTensor', keys=['img']), + dict(type='Collect', keys=['img']), + ]) +] +data = dict( + samples_per_gpu=2, + workers_per_gpu=2, + train=dict( + type=dataset_type, + ann_file=data_root + 'annotations/instances_train2017.json', + img_prefix=data_root + 'train2017/', + pipeline=train_pipeline), + val=dict( + type=dataset_type, + ann_file=data_root + 'annotations/instances_val2017.json', + img_prefix=data_root + 'val2017/', + pipeline=test_pipeline), + test=dict( + type=dataset_type, + ann_file=data_root + 'annotations/instances_val2017.json', + img_prefix=data_root + 'val2017/', + pipeline=test_pipeline)) +evaluation = dict(interval=1, metric='bbox', classwise=True) \ No newline at end of file diff --git a/detection/configs/_base_/datasets/coco_instance.py b/detection/configs/_base_/datasets/coco_instance.py new file mode 100644 index 0000000..91461aa --- /dev/null +++ b/detection/configs/_base_/datasets/coco_instance.py @@ -0,0 +1,49 @@ +# dataset settings +dataset_type = 'CocoDataset' +data_root = 'data/coco/' +img_norm_cfg = dict( + mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) +train_pipeline = [ + dict(type='LoadImageFromFile'), + dict(type='LoadAnnotations', with_bbox=True, with_mask=True), + dict(type='Resize', img_scale=(1333, 800), keep_ratio=True), + dict(type='RandomFlip', flip_ratio=0.5), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size_divisor=32), + dict(type='DefaultFormatBundle'), + dict(type='Collect', keys=['img', 'gt_bboxes', 'gt_labels', 'gt_masks']), +] +test_pipeline = [ + dict(type='LoadImageFromFile'), + dict( + type='MultiScaleFlipAug', + img_scale=(1333, 800), + flip=False, + transforms=[ + dict(type='Resize', keep_ratio=True), + dict(type='RandomFlip'), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size_divisor=32), + dict(type='ImageToTensor', keys=['img']), + dict(type='Collect', keys=['img']), + ]) +] +data = dict( + samples_per_gpu=2, + workers_per_gpu=2, + train=dict( + type=dataset_type, + ann_file=data_root + 'annotations/instances_train2017.json', + img_prefix=data_root + 'train2017/', + pipeline=train_pipeline), + val=dict( + type=dataset_type, + ann_file=data_root + 'annotations/instances_val2017.json', + img_prefix=data_root + 'val2017/', + pipeline=test_pipeline), + test=dict( + type=dataset_type, + ann_file=data_root + 'annotations/instances_val2017.json', + img_prefix=data_root + 'val2017/', + pipeline=test_pipeline)) +evaluation = dict(metric=['bbox', 'segm'], classwise=True) diff --git a/detection/configs/_base_/datasets/crowd_human.py b/detection/configs/_base_/datasets/crowd_human.py new file mode 100644 index 0000000..b1c4921 --- /dev/null +++ b/detection/configs/_base_/datasets/crowd_human.py @@ -0,0 +1,54 @@ +# dataset settings +dataset_type = 'CrowdHumanDataset' +data_root = 'data/CrowdHuman/' +classes = ('person',) +img_norm_cfg = dict( + mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) +train_pipeline = [ + dict(type='LoadImageFromFile'), + dict(type='LoadAnnotations', with_bbox=True), + dict(type='Resize', img_scale=(1333, 800), keep_ratio=True), + dict(type='RandomFlip', flip_ratio=0.5), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size_divisor=32), + dict(type='DefaultFormatBundle'), + dict(type='Collect', keys=['img', 'gt_bboxes', 'gt_labels']), +] +test_pipeline = [ + dict(type='LoadImageFromFile'), + dict( + type='MultiScaleFlipAug', + img_scale=(1333, 800), + flip=False, + transforms=[ + dict(type='Resize', keep_ratio=True), + dict(type='RandomFlip'), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size_divisor=32), + dict(type='ImageToTensor', keys=['img']), + dict(type='Collect', keys=['img']), + ]) +] +data = dict( + samples_per_gpu=2, + workers_per_gpu=2, + train=dict( + type=dataset_type, + classes=classes, + filter_empty_gt=True, + ann_file=data_root + 'annotations/annotation_train.json', + img_prefix=data_root + 'Images', + pipeline=train_pipeline), + val=dict( + type=dataset_type, + classes=classes, + ann_file=data_root + 'annotations/annotation_val.json', + img_prefix=data_root + 'Images', + pipeline=test_pipeline), + test=dict( + type=dataset_type, + classes=classes, + ann_file=data_root + 'annotations/annotation_val.json', + img_prefix=data_root + 'Images', + pipeline=test_pipeline)) +evaluation = dict(interval=100, metric='bbox') diff --git a/detection/configs/_base_/default_runtime.py b/detection/configs/_base_/default_runtime.py new file mode 100644 index 0000000..55097c5 --- /dev/null +++ b/detection/configs/_base_/default_runtime.py @@ -0,0 +1,16 @@ +checkpoint_config = dict(interval=1) +# yapf:disable +log_config = dict( + interval=50, + hooks=[ + dict(type='TextLoggerHook'), + # dict(type='TensorboardLoggerHook') + ]) +# yapf:enable +custom_hooks = [dict(type='NumClassCheckHook')] + +dist_params = dict(backend='nccl') +log_level = 'INFO' +load_from = None +resume_from = None +workflow = [('train', 1)] diff --git a/detection/configs/_base_/models/cascade_mask_rcnn_convnext_fpn.py b/detection/configs/_base_/models/cascade_mask_rcnn_convnext_fpn.py new file mode 100644 index 0000000..beb9db1 --- /dev/null +++ b/detection/configs/_base_/models/cascade_mask_rcnn_convnext_fpn.py @@ -0,0 +1,208 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. + +# All rights reserved. + +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + + +# model settings +model = dict( + type='CascadeRCNN', + pretrained=None, + backbone=dict( + type='ConvNeXt_speed', + in_chans=3, + depths=[3, 3, 9, 3], + dims=[96, 192, 384, 768], + drop_path_rate=0.2, + layer_scale_init_value=1e-6, + out_indices=[0, 1, 2, 3], + ), + neck=dict( + type='FPN', + in_channels=[128, 256, 512, 1024], + out_channels=256, + num_outs=5), + rpn_head=dict( + type='RPNHead', + in_channels=256, + feat_channels=256, + anchor_generator=dict( + type='AnchorGenerator', + scales=[8], + ratios=[0.5, 1.0, 2.0], + strides=[4, 8, 16, 32, 64]), + bbox_coder=dict( + type='DeltaXYWHBBoxCoder', + target_means=[.0, .0, .0, .0], + target_stds=[1.0, 1.0, 1.0, 1.0]), + loss_cls=dict( + type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0), + loss_bbox=dict(type='SmoothL1Loss', beta=1.0 / 9.0, loss_weight=1.0)), + roi_head=dict( + type='CascadeRoIHead', + num_stages=3, + stage_loss_weights=[1, 0.5, 0.25], + bbox_roi_extractor=dict( + type='SingleRoIExtractor', + roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0), + out_channels=256, + featmap_strides=[4, 8, 16, 32]), + bbox_head=[ + dict( + type='Shared2FCBBoxHead', + in_channels=256, + fc_out_channels=1024, + roi_feat_size=7, + num_classes=80, + bbox_coder=dict( + type='DeltaXYWHBBoxCoder', + target_means=[0., 0., 0., 0.], + target_stds=[0.1, 0.1, 0.2, 0.2]), + reg_class_agnostic=True, + loss_cls=dict( + type='CrossEntropyLoss', + use_sigmoid=False, + loss_weight=1.0), + loss_bbox=dict(type='SmoothL1Loss', beta=1.0, + loss_weight=1.0)), + dict( + type='Shared2FCBBoxHead', + in_channels=256, + fc_out_channels=1024, + roi_feat_size=7, + num_classes=80, + bbox_coder=dict( + type='DeltaXYWHBBoxCoder', + target_means=[0., 0., 0., 0.], + target_stds=[0.05, 0.05, 0.1, 0.1]), + reg_class_agnostic=True, + loss_cls=dict( + type='CrossEntropyLoss', + use_sigmoid=False, + loss_weight=1.0), + loss_bbox=dict(type='SmoothL1Loss', beta=1.0, + loss_weight=1.0)), + dict( + type='Shared2FCBBoxHead', + in_channels=256, + fc_out_channels=1024, + roi_feat_size=7, + num_classes=80, + bbox_coder=dict( + type='DeltaXYWHBBoxCoder', + target_means=[0., 0., 0., 0.], + target_stds=[0.033, 0.033, 0.067, 0.067]), + reg_class_agnostic=True, + loss_cls=dict( + type='CrossEntropyLoss', + use_sigmoid=False, + loss_weight=1.0), + loss_bbox=dict(type='SmoothL1Loss', beta=1.0, loss_weight=1.0)) + ], + mask_roi_extractor=dict( + type='SingleRoIExtractor', + roi_layer=dict(type='RoIAlign', output_size=14, sampling_ratio=0), + out_channels=256, + featmap_strides=[4, 8, 16, 32]), + mask_head=dict( + type='FCNMaskHead', + num_convs=4, + in_channels=256, + conv_out_channels=256, + num_classes=80, + loss_mask=dict( + type='CrossEntropyLoss', use_mask=True, loss_weight=1.0))), + # model training and testing settings + train_cfg = dict( + rpn=dict( + assigner=dict( + type='MaxIoUAssigner', + pos_iou_thr=0.7, + neg_iou_thr=0.3, + min_pos_iou=0.3, + match_low_quality=True, + ignore_iof_thr=-1), + sampler=dict( + type='RandomSampler', + num=256, + pos_fraction=0.5, + neg_pos_ub=-1, + add_gt_as_proposals=False), + allowed_border=0, + pos_weight=-1, + debug=False), + rpn_proposal=dict( + nms_across_levels=False, + nms_pre=2000, + nms_post=2000, + max_per_img=2000, + nms=dict(type='nms', iou_threshold=0.7), + min_bbox_size=0), + rcnn=[ + dict( + assigner=dict( + type='MaxIoUAssigner', + pos_iou_thr=0.5, + neg_iou_thr=0.5, + min_pos_iou=0.5, + match_low_quality=False, + ignore_iof_thr=-1), + sampler=dict( + type='RandomSampler', + num=512, + pos_fraction=0.25, + neg_pos_ub=-1, + add_gt_as_proposals=True), + mask_size=28, + pos_weight=-1, + debug=False), + dict( + assigner=dict( + type='MaxIoUAssigner', + pos_iou_thr=0.6, + neg_iou_thr=0.6, + min_pos_iou=0.6, + match_low_quality=False, + ignore_iof_thr=-1), + sampler=dict( + type='RandomSampler', + num=512, + pos_fraction=0.25, + neg_pos_ub=-1, + add_gt_as_proposals=True), + mask_size=28, + pos_weight=-1, + debug=False), + dict( + assigner=dict( + type='MaxIoUAssigner', + pos_iou_thr=0.7, + neg_iou_thr=0.7, + min_pos_iou=0.7, + match_low_quality=False, + ignore_iof_thr=-1), + sampler=dict( + type='RandomSampler', + num=512, + pos_fraction=0.25, + neg_pos_ub=-1, + add_gt_as_proposals=True), + mask_size=28, + pos_weight=-1, + debug=False) + ]), + test_cfg = dict( + rpn=dict( + nms_across_levels=False, + nms_pre=1000, + nms_post=1000, + max_per_img=1000, + nms=dict(type='nms', iou_threshold=0.7), + min_bbox_size=0), + rcnn=dict( + score_thr=0.05, + nms=dict(type='nms', iou_threshold=0.5), + max_per_img=100, + mask_thr_binary=0.5))) \ No newline at end of file diff --git a/detection/configs/_base_/models/cascade_mask_rcnn_r50_fpn.py b/detection/configs/_base_/models/cascade_mask_rcnn_r50_fpn.py new file mode 100644 index 0000000..2902cca --- /dev/null +++ b/detection/configs/_base_/models/cascade_mask_rcnn_r50_fpn.py @@ -0,0 +1,196 @@ +# model settings +model = dict( + type='CascadeRCNN', + backbone=dict( + type='ResNet', + depth=50, + num_stages=4, + out_indices=(0, 1, 2, 3), + frozen_stages=1, + norm_cfg=dict(type='BN', requires_grad=True), + norm_eval=True, + style='pytorch', + init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet50')), + neck=dict( + type='FPN', + in_channels=[256, 512, 1024, 2048], + out_channels=256, + num_outs=5), + rpn_head=dict( + type='RPNHead', + in_channels=256, + feat_channels=256, + anchor_generator=dict( + type='AnchorGenerator', + scales=[8], + ratios=[0.5, 1.0, 2.0], + strides=[4, 8, 16, 32, 64]), + bbox_coder=dict( + type='DeltaXYWHBBoxCoder', + target_means=[.0, .0, .0, .0], + target_stds=[1.0, 1.0, 1.0, 1.0]), + loss_cls=dict( + type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0), + loss_bbox=dict(type='SmoothL1Loss', beta=1.0 / 9.0, loss_weight=1.0)), + roi_head=dict( + type='CascadeRoIHead', + num_stages=3, + stage_loss_weights=[1, 0.5, 0.25], + bbox_roi_extractor=dict( + type='SingleRoIExtractor', + roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0), + out_channels=256, + featmap_strides=[4, 8, 16, 32]), + bbox_head=[ + dict( + type='Shared2FCBBoxHead', + in_channels=256, + fc_out_channels=1024, + roi_feat_size=7, + num_classes=80, + bbox_coder=dict( + type='DeltaXYWHBBoxCoder', + target_means=[0., 0., 0., 0.], + target_stds=[0.1, 0.1, 0.2, 0.2]), + reg_class_agnostic=True, + loss_cls=dict( + type='CrossEntropyLoss', + use_sigmoid=False, + loss_weight=1.0), + loss_bbox=dict(type='SmoothL1Loss', beta=1.0, + loss_weight=1.0)), + dict( + type='Shared2FCBBoxHead', + in_channels=256, + fc_out_channels=1024, + roi_feat_size=7, + num_classes=80, + bbox_coder=dict( + type='DeltaXYWHBBoxCoder', + target_means=[0., 0., 0., 0.], + target_stds=[0.05, 0.05, 0.1, 0.1]), + reg_class_agnostic=True, + loss_cls=dict( + type='CrossEntropyLoss', + use_sigmoid=False, + loss_weight=1.0), + loss_bbox=dict(type='SmoothL1Loss', beta=1.0, + loss_weight=1.0)), + dict( + type='Shared2FCBBoxHead', + in_channels=256, + fc_out_channels=1024, + roi_feat_size=7, + num_classes=80, + bbox_coder=dict( + type='DeltaXYWHBBoxCoder', + target_means=[0., 0., 0., 0.], + target_stds=[0.033, 0.033, 0.067, 0.067]), + reg_class_agnostic=True, + loss_cls=dict( + type='CrossEntropyLoss', + use_sigmoid=False, + loss_weight=1.0), + loss_bbox=dict(type='SmoothL1Loss', beta=1.0, loss_weight=1.0)) + ], + mask_roi_extractor=dict( + type='SingleRoIExtractor', + roi_layer=dict(type='RoIAlign', output_size=14, sampling_ratio=0), + out_channels=256, + featmap_strides=[4, 8, 16, 32]), + mask_head=dict( + type='FCNMaskHead', + num_convs=4, + in_channels=256, + conv_out_channels=256, + num_classes=80, + loss_mask=dict( + type='CrossEntropyLoss', use_mask=True, loss_weight=1.0))), + # model training and testing settings + train_cfg=dict( + rpn=dict( + assigner=dict( + type='MaxIoUAssigner', + pos_iou_thr=0.7, + neg_iou_thr=0.3, + min_pos_iou=0.3, + match_low_quality=True, + ignore_iof_thr=-1), + sampler=dict( + type='RandomSampler', + num=256, + pos_fraction=0.5, + neg_pos_ub=-1, + add_gt_as_proposals=False), + allowed_border=0, + pos_weight=-1, + debug=False), + rpn_proposal=dict( + nms_pre=2000, + max_per_img=2000, + nms=dict(type='nms', iou_threshold=0.7), + min_bbox_size=0), + rcnn=[ + dict( + assigner=dict( + type='MaxIoUAssigner', + pos_iou_thr=0.5, + neg_iou_thr=0.5, + min_pos_iou=0.5, + match_low_quality=False, + ignore_iof_thr=-1), + sampler=dict( + type='RandomSampler', + num=512, + pos_fraction=0.25, + neg_pos_ub=-1, + add_gt_as_proposals=True), + mask_size=28, + pos_weight=-1, + debug=False), + dict( + assigner=dict( + type='MaxIoUAssigner', + pos_iou_thr=0.6, + neg_iou_thr=0.6, + min_pos_iou=0.6, + match_low_quality=False, + ignore_iof_thr=-1), + sampler=dict( + type='RandomSampler', + num=512, + pos_fraction=0.25, + neg_pos_ub=-1, + add_gt_as_proposals=True), + mask_size=28, + pos_weight=-1, + debug=False), + dict( + assigner=dict( + type='MaxIoUAssigner', + pos_iou_thr=0.7, + neg_iou_thr=0.7, + min_pos_iou=0.7, + match_low_quality=False, + ignore_iof_thr=-1), + sampler=dict( + type='RandomSampler', + num=512, + pos_fraction=0.25, + neg_pos_ub=-1, + add_gt_as_proposals=True), + mask_size=28, + pos_weight=-1, + debug=False) + ]), + test_cfg=dict( + rpn=dict( + nms_pre=1000, + max_per_img=1000, + nms=dict(type='nms', iou_threshold=0.7), + min_bbox_size=0), + rcnn=dict( + score_thr=0.05, + nms=dict(type='nms', iou_threshold=0.5), + max_per_img=100, + mask_thr_binary=0.5))) diff --git a/detection/configs/_base_/models/cascade_mask_rcnn_r50_fpn_crowdhuman.py b/detection/configs/_base_/models/cascade_mask_rcnn_r50_fpn_crowdhuman.py new file mode 100644 index 0000000..8addb67 --- /dev/null +++ b/detection/configs/_base_/models/cascade_mask_rcnn_r50_fpn_crowdhuman.py @@ -0,0 +1,183 @@ +# model settings +model = dict( + type='CascadeRCNN', + backbone=dict( + type='ResNet', + depth=50, + num_stages=4, + out_indices=(0, 1, 2, 3), + frozen_stages=1, + norm_cfg=dict(type='BN', requires_grad=True), + norm_eval=True, + style='pytorch', + init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet50')), + neck=dict( + type='FPN', + in_channels=[256, 512, 1024, 2048], + out_channels=256, + num_outs=5), + rpn_head=dict( + type='RPNHead', + in_channels=256, + feat_channels=256, + anchor_generator=dict( + type='AnchorGenerator', + scales=[8], + ratios=[0.5, 1.0, 2.0], + strides=[4, 8, 16, 32, 64]), + bbox_coder=dict( + type='DeltaXYWHBBoxCoder', + target_means=[.0, .0, .0, .0], + target_stds=[1.0, 1.0, 1.0, 1.0]), + loss_cls=dict( + type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0), + loss_bbox=dict(type='SmoothL1Loss', beta=1.0 / 9.0, loss_weight=1.0)), + roi_head=dict( + type='CascadeRoIHead', + num_stages=3, + stage_loss_weights=[1, 0.5, 0.25], + bbox_roi_extractor=dict( + type='SingleRoIExtractor', + roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0), + out_channels=256, + featmap_strides=[4, 8, 16, 32]), + bbox_head=[ + dict( + type='Shared2FCBBoxHead', + in_channels=256, + fc_out_channels=1024, + roi_feat_size=7, + num_classes=80, + bbox_coder=dict( + type='DeltaXYWHBBoxCoder', + target_means=[0., 0., 0., 0.], + target_stds=[0.1, 0.1, 0.2, 0.2]), + reg_class_agnostic=True, + loss_cls=dict( + type='CrossEntropyLoss', + use_sigmoid=False, + loss_weight=1.0), + loss_bbox=dict(type='SmoothL1Loss', beta=1.0, + loss_weight=1.0)), + dict( + type='Shared2FCBBoxHead', + in_channels=256, + fc_out_channels=1024, + roi_feat_size=7, + num_classes=80, + bbox_coder=dict( + type='DeltaXYWHBBoxCoder', + target_means=[0., 0., 0., 0.], + target_stds=[0.05, 0.05, 0.1, 0.1]), + reg_class_agnostic=True, + loss_cls=dict( + type='CrossEntropyLoss', + use_sigmoid=False, + loss_weight=1.0), + loss_bbox=dict(type='SmoothL1Loss', beta=1.0, + loss_weight=1.0)), + dict( + type='Shared2FCBBoxHead', + in_channels=256, + fc_out_channels=1024, + roi_feat_size=7, + num_classes=80, + bbox_coder=dict( + type='DeltaXYWHBBoxCoder', + target_means=[0., 0., 0., 0.], + target_stds=[0.033, 0.033, 0.067, 0.067]), + reg_class_agnostic=True, + loss_cls=dict( + type='CrossEntropyLoss', + use_sigmoid=False, + loss_weight=1.0), + loss_bbox=dict(type='SmoothL1Loss', beta=1.0, loss_weight=1.0)) + ],), + # model training and testing settings + train_cfg=dict( + rpn=dict( + assigner=dict( + type='MaxIoUAssigner', + pos_iou_thr=0.7, + neg_iou_thr=0.3, + min_pos_iou=0.3, + match_low_quality=True, + ignore_iof_thr=-1), + sampler=dict( + type='RandomSampler', + num=256, + pos_fraction=0.5, + neg_pos_ub=-1, + add_gt_as_proposals=False), + allowed_border=0, + pos_weight=-1, + debug=False), + rpn_proposal=dict( + nms_pre=2000, + max_per_img=2000, + nms=dict(type='nms', iou_threshold=0.7), + min_bbox_size=0), + rcnn=[ + dict( + assigner=dict( + type='MaxIoUAssigner', + pos_iou_thr=0.5, + neg_iou_thr=0.5, + min_pos_iou=0.5, + match_low_quality=False, + ignore_iof_thr=-1), + sampler=dict( + type='RandomSampler', + num=512, + pos_fraction=0.25, + neg_pos_ub=-1, + add_gt_as_proposals=True), + mask_size=28, + pos_weight=-1, + debug=False), + dict( + assigner=dict( + type='MaxIoUAssigner', + pos_iou_thr=0.6, + neg_iou_thr=0.6, + min_pos_iou=0.6, + match_low_quality=False, + ignore_iof_thr=-1), + sampler=dict( + type='RandomSampler', + num=512, + pos_fraction=0.25, + neg_pos_ub=-1, + add_gt_as_proposals=True), + mask_size=28, + pos_weight=-1, + debug=False), + dict( + assigner=dict( + type='MaxIoUAssigner', + pos_iou_thr=0.7, + neg_iou_thr=0.7, + min_pos_iou=0.7, + match_low_quality=False, + ignore_iof_thr=-1), + sampler=dict( + type='RandomSampler', + num=512, + pos_fraction=0.25, + neg_pos_ub=-1, + add_gt_as_proposals=True), + mask_size=28, + pos_weight=-1, + debug=False) + ]), + test_cfg=dict( + rpn=dict( + nms_pre=1000, + max_per_img=1000, + nms=dict(type='nms', iou_threshold=0.7), + min_bbox_size=0), + rcnn=dict( + score_thr=0.05, + nms=dict(type='nms', iou_threshold=0.5), + max_per_img=100, + mask_thr_binary=0.5))) diff --git a/detection/configs/_base_/models/cascade_rcnn_r50_fpn.py b/detection/configs/_base_/models/cascade_rcnn_r50_fpn.py new file mode 100644 index 0000000..42f74ae --- /dev/null +++ b/detection/configs/_base_/models/cascade_rcnn_r50_fpn.py @@ -0,0 +1,179 @@ +# model settings +model = dict( + type='CascadeRCNN', + backbone=dict( + type='ResNet', + depth=50, + num_stages=4, + out_indices=(0, 1, 2, 3), + frozen_stages=1, + norm_cfg=dict(type='BN', requires_grad=True), + norm_eval=True, + style='pytorch', + init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet50')), + neck=dict( + type='FPN', + in_channels=[256, 512, 1024, 2048], + out_channels=256, + num_outs=5), + rpn_head=dict( + type='RPNHead', + in_channels=256, + feat_channels=256, + anchor_generator=dict( + type='AnchorGenerator', + scales=[8], + ratios=[0.5, 1.0, 2.0], + strides=[4, 8, 16, 32, 64]), + bbox_coder=dict( + type='DeltaXYWHBBoxCoder', + target_means=[.0, .0, .0, .0], + target_stds=[1.0, 1.0, 1.0, 1.0]), + loss_cls=dict( + type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0), + loss_bbox=dict(type='SmoothL1Loss', beta=1.0 / 9.0, loss_weight=1.0)), + roi_head=dict( + type='CascadeRoIHead', + num_stages=3, + stage_loss_weights=[1, 0.5, 0.25], + bbox_roi_extractor=dict( + type='SingleRoIExtractor', + roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0), + out_channels=256, + featmap_strides=[4, 8, 16, 32]), + bbox_head=[ + dict( + type='Shared2FCBBoxHead', + in_channels=256, + fc_out_channels=1024, + roi_feat_size=7, + num_classes=80, + bbox_coder=dict( + type='DeltaXYWHBBoxCoder', + target_means=[0., 0., 0., 0.], + target_stds=[0.1, 0.1, 0.2, 0.2]), + reg_class_agnostic=True, + loss_cls=dict( + type='CrossEntropyLoss', + use_sigmoid=False, + loss_weight=1.0), + loss_bbox=dict(type='SmoothL1Loss', beta=1.0, + loss_weight=1.0)), + dict( + type='Shared2FCBBoxHead', + in_channels=256, + fc_out_channels=1024, + roi_feat_size=7, + num_classes=80, + bbox_coder=dict( + type='DeltaXYWHBBoxCoder', + target_means=[0., 0., 0., 0.], + target_stds=[0.05, 0.05, 0.1, 0.1]), + reg_class_agnostic=True, + loss_cls=dict( + type='CrossEntropyLoss', + use_sigmoid=False, + loss_weight=1.0), + loss_bbox=dict(type='SmoothL1Loss', beta=1.0, + loss_weight=1.0)), + dict( + type='Shared2FCBBoxHead', + in_channels=256, + fc_out_channels=1024, + roi_feat_size=7, + num_classes=80, + bbox_coder=dict( + type='DeltaXYWHBBoxCoder', + target_means=[0., 0., 0., 0.], + target_stds=[0.033, 0.033, 0.067, 0.067]), + reg_class_agnostic=True, + loss_cls=dict( + type='CrossEntropyLoss', + use_sigmoid=False, + loss_weight=1.0), + loss_bbox=dict(type='SmoothL1Loss', beta=1.0, loss_weight=1.0)) + ]), + # model training and testing settings + train_cfg=dict( + rpn=dict( + assigner=dict( + type='MaxIoUAssigner', + pos_iou_thr=0.7, + neg_iou_thr=0.3, + min_pos_iou=0.3, + match_low_quality=True, + ignore_iof_thr=-1), + sampler=dict( + type='RandomSampler', + num=256, + pos_fraction=0.5, + neg_pos_ub=-1, + add_gt_as_proposals=False), + allowed_border=0, + pos_weight=-1, + debug=False), + rpn_proposal=dict( + nms_pre=2000, + max_per_img=2000, + nms=dict(type='nms', iou_threshold=0.7), + min_bbox_size=0), + rcnn=[ + dict( + assigner=dict( + type='MaxIoUAssigner', + pos_iou_thr=0.5, + neg_iou_thr=0.5, + min_pos_iou=0.5, + match_low_quality=False, + ignore_iof_thr=-1), + sampler=dict( + type='RandomSampler', + num=512, + pos_fraction=0.25, + neg_pos_ub=-1, + add_gt_as_proposals=True), + pos_weight=-1, + debug=False), + dict( + assigner=dict( + type='MaxIoUAssigner', + pos_iou_thr=0.6, + neg_iou_thr=0.6, + min_pos_iou=0.6, + match_low_quality=False, + ignore_iof_thr=-1), + sampler=dict( + type='RandomSampler', + num=512, + pos_fraction=0.25, + neg_pos_ub=-1, + add_gt_as_proposals=True), + pos_weight=-1, + debug=False), + dict( + assigner=dict( + type='MaxIoUAssigner', + pos_iou_thr=0.7, + neg_iou_thr=0.7, + min_pos_iou=0.7, + match_low_quality=False, + ignore_iof_thr=-1), + sampler=dict( + type='RandomSampler', + num=512, + pos_fraction=0.25, + neg_pos_ub=-1, + add_gt_as_proposals=True), + pos_weight=-1, + debug=False) + ]), + test_cfg=dict( + rpn=dict( + nms_pre=1000, + max_per_img=1000, + nms=dict(type='nms', iou_threshold=0.7), + min_bbox_size=0), + rcnn=dict( + score_thr=0.05, + nms=dict(type='nms', iou_threshold=0.5), + max_per_img=100))) diff --git a/detection/configs/_base_/models/fast_rcnn_r50_fpn.py b/detection/configs/_base_/models/fast_rcnn_r50_fpn.py new file mode 100644 index 0000000..9982fe0 --- /dev/null +++ b/detection/configs/_base_/models/fast_rcnn_r50_fpn.py @@ -0,0 +1,62 @@ +# model settings +model = dict( + type='FastRCNN', + backbone=dict( + type='ResNet', + depth=50, + num_stages=4, + out_indices=(0, 1, 2, 3), + frozen_stages=1, + norm_cfg=dict(type='BN', requires_grad=True), + norm_eval=True, + style='pytorch', + init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet50')), + neck=dict( + type='FPN', + in_channels=[256, 512, 1024, 2048], + out_channels=256, + num_outs=5), + roi_head=dict( + type='StandardRoIHead', + bbox_roi_extractor=dict( + type='SingleRoIExtractor', + roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0), + out_channels=256, + featmap_strides=[4, 8, 16, 32]), + bbox_head=dict( + type='Shared2FCBBoxHead', + in_channels=256, + fc_out_channels=1024, + roi_feat_size=7, + num_classes=80, + bbox_coder=dict( + type='DeltaXYWHBBoxCoder', + target_means=[0., 0., 0., 0.], + target_stds=[0.1, 0.1, 0.2, 0.2]), + reg_class_agnostic=False, + loss_cls=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0), + loss_bbox=dict(type='L1Loss', loss_weight=1.0))), + # model training and testing settings + train_cfg=dict( + rcnn=dict( + assigner=dict( + type='MaxIoUAssigner', + pos_iou_thr=0.5, + neg_iou_thr=0.5, + min_pos_iou=0.5, + match_low_quality=False, + ignore_iof_thr=-1), + sampler=dict( + type='RandomSampler', + num=512, + pos_fraction=0.25, + neg_pos_ub=-1, + add_gt_as_proposals=True), + pos_weight=-1, + debug=False)), + test_cfg=dict( + rcnn=dict( + score_thr=0.05, + nms=dict(type='nms', iou_threshold=0.5), + max_per_img=100))) diff --git a/detection/configs/_base_/models/faster_rcnn_r50_caffe_c4.py b/detection/configs/_base_/models/faster_rcnn_r50_caffe_c4.py new file mode 100644 index 0000000..51b5db4 --- /dev/null +++ b/detection/configs/_base_/models/faster_rcnn_r50_caffe_c4.py @@ -0,0 +1,114 @@ +# model settings +norm_cfg = dict(type='BN', requires_grad=False) +model = dict( + type='FasterRCNN', + backbone=dict( + type='ResNet', + depth=50, + num_stages=3, + strides=(1, 2, 2), + dilations=(1, 1, 1), + out_indices=(2, ), + frozen_stages=1, + norm_cfg=norm_cfg, + norm_eval=True, + style='caffe', + init_cfg=dict( + type='Pretrained', + checkpoint='open-mmlab://detectron2/resnet50_caffe')), + rpn_head=dict( + type='RPNHead', + in_channels=1024, + feat_channels=1024, + anchor_generator=dict( + type='AnchorGenerator', + scales=[2, 4, 8, 16, 32], + ratios=[0.5, 1.0, 2.0], + strides=[16]), + bbox_coder=dict( + type='DeltaXYWHBBoxCoder', + target_means=[.0, .0, .0, .0], + target_stds=[1.0, 1.0, 1.0, 1.0]), + loss_cls=dict( + type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0), + loss_bbox=dict(type='L1Loss', loss_weight=1.0)), + roi_head=dict( + type='StandardRoIHead', + shared_head=dict( + type='ResLayer', + depth=50, + stage=3, + stride=2, + dilation=1, + style='caffe', + norm_cfg=norm_cfg, + norm_eval=True), + bbox_roi_extractor=dict( + type='SingleRoIExtractor', + roi_layer=dict(type='RoIAlign', output_size=14, sampling_ratio=0), + out_channels=1024, + featmap_strides=[16]), + bbox_head=dict( + type='BBoxHead', + with_avg_pool=True, + roi_feat_size=7, + in_channels=2048, + num_classes=80, + bbox_coder=dict( + type='DeltaXYWHBBoxCoder', + target_means=[0., 0., 0., 0.], + target_stds=[0.1, 0.1, 0.2, 0.2]), + reg_class_agnostic=False, + loss_cls=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0), + loss_bbox=dict(type='L1Loss', loss_weight=1.0))), + # model training and testing settings + train_cfg=dict( + rpn=dict( + assigner=dict( + type='MaxIoUAssigner', + pos_iou_thr=0.7, + neg_iou_thr=0.3, + min_pos_iou=0.3, + match_low_quality=True, + ignore_iof_thr=-1), + sampler=dict( + type='RandomSampler', + num=256, + pos_fraction=0.5, + neg_pos_ub=-1, + add_gt_as_proposals=False), + allowed_border=0, + pos_weight=-1, + debug=False), + rpn_proposal=dict( + nms_pre=12000, + max_per_img=2000, + nms=dict(type='nms', iou_threshold=0.7), + min_bbox_size=0), + rcnn=dict( + assigner=dict( + type='MaxIoUAssigner', + pos_iou_thr=0.5, + neg_iou_thr=0.5, + min_pos_iou=0.5, + match_low_quality=False, + ignore_iof_thr=-1), + sampler=dict( + type='RandomSampler', + num=512, + pos_fraction=0.25, + neg_pos_ub=-1, + add_gt_as_proposals=True), + pos_weight=-1, + debug=False)), + test_cfg=dict( + rpn=dict( + nms_pre=6000, + max_per_img=1000, + nms=dict(type='nms', iou_threshold=0.7), + min_bbox_size=0), + rcnn=dict( + score_thr=0.05, + nms=dict(type='nms', iou_threshold=0.5), + max_per_img=100))) diff --git a/detection/configs/_base_/models/faster_rcnn_r50_caffe_dc5.py b/detection/configs/_base_/models/faster_rcnn_r50_caffe_dc5.py new file mode 100644 index 0000000..a377a6f --- /dev/null +++ b/detection/configs/_base_/models/faster_rcnn_r50_caffe_dc5.py @@ -0,0 +1,105 @@ +# model settings +norm_cfg = dict(type='BN', requires_grad=False) +model = dict( + type='FasterRCNN', + backbone=dict( + type='ResNet', + depth=50, + num_stages=4, + strides=(1, 2, 2, 1), + dilations=(1, 1, 1, 2), + out_indices=(3, ), + frozen_stages=1, + norm_cfg=norm_cfg, + norm_eval=True, + style='caffe', + init_cfg=dict( + type='Pretrained', + checkpoint='open-mmlab://detectron2/resnet50_caffe')), + rpn_head=dict( + type='RPNHead', + in_channels=2048, + feat_channels=2048, + anchor_generator=dict( + type='AnchorGenerator', + scales=[2, 4, 8, 16, 32], + ratios=[0.5, 1.0, 2.0], + strides=[16]), + bbox_coder=dict( + type='DeltaXYWHBBoxCoder', + target_means=[.0, .0, .0, .0], + target_stds=[1.0, 1.0, 1.0, 1.0]), + loss_cls=dict( + type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0), + loss_bbox=dict(type='L1Loss', loss_weight=1.0)), + roi_head=dict( + type='StandardRoIHead', + bbox_roi_extractor=dict( + type='SingleRoIExtractor', + roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0), + out_channels=2048, + featmap_strides=[16]), + bbox_head=dict( + type='Shared2FCBBoxHead', + in_channels=2048, + fc_out_channels=1024, + roi_feat_size=7, + num_classes=80, + bbox_coder=dict( + type='DeltaXYWHBBoxCoder', + target_means=[0., 0., 0., 0.], + target_stds=[0.1, 0.1, 0.2, 0.2]), + reg_class_agnostic=False, + loss_cls=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0), + loss_bbox=dict(type='L1Loss', loss_weight=1.0))), + # model training and testing settings + train_cfg=dict( + rpn=dict( + assigner=dict( + type='MaxIoUAssigner', + pos_iou_thr=0.7, + neg_iou_thr=0.3, + min_pos_iou=0.3, + match_low_quality=True, + ignore_iof_thr=-1), + sampler=dict( + type='RandomSampler', + num=256, + pos_fraction=0.5, + neg_pos_ub=-1, + add_gt_as_proposals=False), + allowed_border=0, + pos_weight=-1, + debug=False), + rpn_proposal=dict( + nms_pre=12000, + max_per_img=2000, + nms=dict(type='nms', iou_threshold=0.7), + min_bbox_size=0), + rcnn=dict( + assigner=dict( + type='MaxIoUAssigner', + pos_iou_thr=0.5, + neg_iou_thr=0.5, + min_pos_iou=0.5, + match_low_quality=False, + ignore_iof_thr=-1), + sampler=dict( + type='RandomSampler', + num=512, + pos_fraction=0.25, + neg_pos_ub=-1, + add_gt_as_proposals=True), + pos_weight=-1, + debug=False)), + test_cfg=dict( + rpn=dict( + nms=dict(type='nms', iou_threshold=0.7), + nms_pre=6000, + max_per_img=1000, + min_bbox_size=0), + rcnn=dict( + score_thr=0.05, + nms=dict(type='nms', iou_threshold=0.5), + max_per_img=100))) diff --git a/detection/configs/_base_/models/faster_rcnn_r50_fpn.py b/detection/configs/_base_/models/faster_rcnn_r50_fpn.py new file mode 100644 index 0000000..1ef8e7b --- /dev/null +++ b/detection/configs/_base_/models/faster_rcnn_r50_fpn.py @@ -0,0 +1,108 @@ +# model settings +model = dict( + type='FasterRCNN', + backbone=dict( + type='ResNet', + depth=50, + num_stages=4, + out_indices=(0, 1, 2, 3), + frozen_stages=1, + norm_cfg=dict(type='BN', requires_grad=True), + norm_eval=True, + style='pytorch', + init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet50')), + neck=dict( + type='FPN', + in_channels=[256, 512, 1024, 2048], + out_channels=256, + num_outs=5), + rpn_head=dict( + type='RPNHead', + in_channels=256, + feat_channels=256, + anchor_generator=dict( + type='AnchorGenerator', + scales=[8], + ratios=[0.5, 1.0, 2.0], + strides=[4, 8, 16, 32, 64]), + bbox_coder=dict( + type='DeltaXYWHBBoxCoder', + target_means=[.0, .0, .0, .0], + target_stds=[1.0, 1.0, 1.0, 1.0]), + loss_cls=dict( + type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0), + loss_bbox=dict(type='L1Loss', loss_weight=1.0)), + roi_head=dict( + type='StandardRoIHead', + bbox_roi_extractor=dict( + type='SingleRoIExtractor', + roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0), + out_channels=256, + featmap_strides=[4, 8, 16, 32]), + bbox_head=dict( + type='Shared2FCBBoxHead', + in_channels=256, + fc_out_channels=1024, + roi_feat_size=7, + num_classes=80, + bbox_coder=dict( + type='DeltaXYWHBBoxCoder', + target_means=[0., 0., 0., 0.], + target_stds=[0.1, 0.1, 0.2, 0.2]), + reg_class_agnostic=False, + loss_cls=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0), + loss_bbox=dict(type='L1Loss', loss_weight=1.0))), + # model training and testing settings + train_cfg=dict( + rpn=dict( + assigner=dict( + type='MaxIoUAssigner', + pos_iou_thr=0.7, + neg_iou_thr=0.3, + min_pos_iou=0.3, + match_low_quality=True, + ignore_iof_thr=-1), + sampler=dict( + type='RandomSampler', + num=256, + pos_fraction=0.5, + neg_pos_ub=-1, + add_gt_as_proposals=False), + allowed_border=-1, + pos_weight=-1, + debug=False), + rpn_proposal=dict( + nms_pre=2000, + max_per_img=1000, + nms=dict(type='nms', iou_threshold=0.7), + min_bbox_size=0), + rcnn=dict( + assigner=dict( + type='MaxIoUAssigner', + pos_iou_thr=0.5, + neg_iou_thr=0.5, + min_pos_iou=0.5, + match_low_quality=False, + ignore_iof_thr=-1), + sampler=dict( + type='RandomSampler', + num=512, + pos_fraction=0.25, + neg_pos_ub=-1, + add_gt_as_proposals=True), + pos_weight=-1, + debug=False)), + test_cfg=dict( + rpn=dict( + nms_pre=1000, + max_per_img=1000, + nms=dict(type='nms', iou_threshold=0.7), + min_bbox_size=0), + rcnn=dict( + score_thr=0.05, + nms=dict(type='nms', iou_threshold=0.5), + max_per_img=100) + # soft-nms is also supported for rcnn testing + # e.g., nms=dict(type='soft_nms', iou_threshold=0.5, min_score=0.05) + )) diff --git a/detection/configs/_base_/models/mask_rcnn_convnext_fpn.py b/detection/configs/_base_/models/mask_rcnn_convnext_fpn.py new file mode 100644 index 0000000..84d02c6 --- /dev/null +++ b/detection/configs/_base_/models/mask_rcnn_convnext_fpn.py @@ -0,0 +1,128 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. + +# All rights reserved. + +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + + +# model settings +model = dict( + type='MaskRCNN', + pretrained=None, + backbone=dict( + type='ConvNeXt', + in_chans=3, + depths=[3, 3, 9, 3], + dims=[96, 192, 384, 768], + drop_path_rate=0.2, + layer_scale_init_value=1e-6, + out_indices=[0, 1, 2, 3], + ), + neck=dict( + type='FPN', + in_channels=[128, 256, 512, 1024], + out_channels=256, + num_outs=5), + rpn_head=dict( + type='RPNHead', + in_channels=256, + feat_channels=256, + anchor_generator=dict( + type='AnchorGenerator', + scales=[8], + ratios=[0.5, 1.0, 2.0], + strides=[4, 8, 16, 32, 64]), + bbox_coder=dict( + type='DeltaXYWHBBoxCoder', + target_means=[.0, .0, .0, .0], + target_stds=[1.0, 1.0, 1.0, 1.0]), + loss_cls=dict( + type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0), + loss_bbox=dict(type='L1Loss', loss_weight=1.0)), + roi_head=dict( + type='StandardRoIHead', + bbox_roi_extractor=dict( + type='SingleRoIExtractor', + roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0), + out_channels=256, + featmap_strides=[4, 8, 16, 32]), + bbox_head=dict( + type='Shared2FCBBoxHead', + in_channels=256, + fc_out_channels=1024, + roi_feat_size=7, + num_classes=80, + bbox_coder=dict( + type='DeltaXYWHBBoxCoder', + target_means=[0., 0., 0., 0.], + target_stds=[0.1, 0.1, 0.2, 0.2]), + reg_class_agnostic=False, + loss_cls=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0), + loss_bbox=dict(type='L1Loss', loss_weight=1.0)), + mask_roi_extractor=dict( + type='SingleRoIExtractor', + roi_layer=dict(type='RoIAlign', output_size=14, sampling_ratio=0), + out_channels=256, + featmap_strides=[4, 8, 16, 32]), + mask_head=dict( + type='FCNMaskHead', + num_convs=4, + in_channels=256, + conv_out_channels=256, + num_classes=80, + loss_mask=dict( + type='CrossEntropyLoss', use_mask=True, loss_weight=1.0))), + # model training and testing settings + train_cfg=dict( + rpn=dict( + assigner=dict( + type='MaxIoUAssigner', + pos_iou_thr=0.7, + neg_iou_thr=0.3, + min_pos_iou=0.3, + match_low_quality=True, + ignore_iof_thr=-1), + sampler=dict( + type='RandomSampler', + num=256, + pos_fraction=0.5, + neg_pos_ub=-1, + add_gt_as_proposals=False), + allowed_border=-1, + pos_weight=-1, + debug=False), + rpn_proposal=dict( + nms_pre=2000, + max_per_img=1000, + nms=dict(type='nms', iou_threshold=0.7), + min_bbox_size=0), + rcnn=dict( + assigner=dict( + type='MaxIoUAssigner', + pos_iou_thr=0.5, + neg_iou_thr=0.5, + min_pos_iou=0.5, + match_low_quality=True, + ignore_iof_thr=-1), + sampler=dict( + type='RandomSampler', + num=512, + pos_fraction=0.25, + neg_pos_ub=-1, + add_gt_as_proposals=True), + mask_size=28, + pos_weight=-1, + debug=False)), + test_cfg=dict( + rpn=dict( + nms_pre=1000, + max_per_img=1000, + nms=dict(type='nms', iou_threshold=0.7), + min_bbox_size=0), + rcnn=dict( + score_thr=0.05, + nms=dict(type='nms', iou_threshold=0.5), + max_per_img=100, + mask_thr_binary=0.5))) \ No newline at end of file diff --git a/detection/configs/_base_/models/mask_rcnn_r50_caffe_c4.py b/detection/configs/_base_/models/mask_rcnn_r50_caffe_c4.py new file mode 100644 index 0000000..122202e --- /dev/null +++ b/detection/configs/_base_/models/mask_rcnn_r50_caffe_c4.py @@ -0,0 +1,125 @@ +# model settings +norm_cfg = dict(type='BN', requires_grad=False) +model = dict( + type='MaskRCNN', + backbone=dict( + type='ResNet', + depth=50, + num_stages=3, + strides=(1, 2, 2), + dilations=(1, 1, 1), + out_indices=(2, ), + frozen_stages=1, + norm_cfg=norm_cfg, + norm_eval=True, + style='caffe', + init_cfg=dict( + type='Pretrained', + checkpoint='open-mmlab://detectron2/resnet50_caffe')), + rpn_head=dict( + type='RPNHead', + in_channels=1024, + feat_channels=1024, + anchor_generator=dict( + type='AnchorGenerator', + scales=[2, 4, 8, 16, 32], + ratios=[0.5, 1.0, 2.0], + strides=[16]), + bbox_coder=dict( + type='DeltaXYWHBBoxCoder', + target_means=[.0, .0, .0, .0], + target_stds=[1.0, 1.0, 1.0, 1.0]), + loss_cls=dict( + type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0), + loss_bbox=dict(type='L1Loss', loss_weight=1.0)), + roi_head=dict( + type='StandardRoIHead', + shared_head=dict( + type='ResLayer', + depth=50, + stage=3, + stride=2, + dilation=1, + style='caffe', + norm_cfg=norm_cfg, + norm_eval=True), + bbox_roi_extractor=dict( + type='SingleRoIExtractor', + roi_layer=dict(type='RoIAlign', output_size=14, sampling_ratio=0), + out_channels=1024, + featmap_strides=[16]), + bbox_head=dict( + type='BBoxHead', + with_avg_pool=True, + roi_feat_size=7, + in_channels=2048, + num_classes=80, + bbox_coder=dict( + type='DeltaXYWHBBoxCoder', + target_means=[0., 0., 0., 0.], + target_stds=[0.1, 0.1, 0.2, 0.2]), + reg_class_agnostic=False, + loss_cls=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0), + loss_bbox=dict(type='L1Loss', loss_weight=1.0)), + mask_roi_extractor=None, + mask_head=dict( + type='FCNMaskHead', + num_convs=0, + in_channels=2048, + conv_out_channels=256, + num_classes=80, + loss_mask=dict( + type='CrossEntropyLoss', use_mask=True, loss_weight=1.0))), + # model training and testing settings + train_cfg=dict( + rpn=dict( + assigner=dict( + type='MaxIoUAssigner', + pos_iou_thr=0.7, + neg_iou_thr=0.3, + min_pos_iou=0.3, + match_low_quality=True, + ignore_iof_thr=-1), + sampler=dict( + type='RandomSampler', + num=256, + pos_fraction=0.5, + neg_pos_ub=-1, + add_gt_as_proposals=False), + allowed_border=0, + pos_weight=-1, + debug=False), + rpn_proposal=dict( + nms_pre=12000, + max_per_img=2000, + nms=dict(type='nms', iou_threshold=0.7), + min_bbox_size=0), + rcnn=dict( + assigner=dict( + type='MaxIoUAssigner', + pos_iou_thr=0.5, + neg_iou_thr=0.5, + min_pos_iou=0.5, + match_low_quality=False, + ignore_iof_thr=-1), + sampler=dict( + type='RandomSampler', + num=512, + pos_fraction=0.25, + neg_pos_ub=-1, + add_gt_as_proposals=True), + mask_size=14, + pos_weight=-1, + debug=False)), + test_cfg=dict( + rpn=dict( + nms_pre=6000, + nms=dict(type='nms', iou_threshold=0.7), + max_per_img=1000, + min_bbox_size=0), + rcnn=dict( + score_thr=0.05, + nms=dict(type='nms', iou_threshold=0.5), + max_per_img=100, + mask_thr_binary=0.5))) diff --git a/detection/configs/_base_/models/mask_rcnn_r50_fpn.py b/detection/configs/_base_/models/mask_rcnn_r50_fpn.py new file mode 100644 index 0000000..d903e55 --- /dev/null +++ b/detection/configs/_base_/models/mask_rcnn_r50_fpn.py @@ -0,0 +1,120 @@ +# model settings +model = dict( + type='MaskRCNN', + backbone=dict( + type='ResNet', + depth=50, + num_stages=4, + out_indices=(0, 1, 2, 3), + frozen_stages=1, + norm_cfg=dict(type='BN', requires_grad=True), + norm_eval=True, + style='pytorch', + init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet50')), + neck=dict( + type='FPN', + in_channels=[256, 512, 1024, 2048], + out_channels=256, + num_outs=5), + rpn_head=dict( + type='RPNHead', + in_channels=256, + feat_channels=256, + anchor_generator=dict( + type='AnchorGenerator', + scales=[8], + ratios=[0.5, 1.0, 2.0], + strides=[4, 8, 16, 32, 64]), + bbox_coder=dict( + type='DeltaXYWHBBoxCoder', + target_means=[.0, .0, .0, .0], + target_stds=[1.0, 1.0, 1.0, 1.0]), + loss_cls=dict( + type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0), + loss_bbox=dict(type='L1Loss', loss_weight=1.0)), + roi_head=dict( + type='StandardRoIHead', + bbox_roi_extractor=dict( + type='SingleRoIExtractor', + roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0), + out_channels=256, + featmap_strides=[4, 8, 16, 32]), + bbox_head=dict( + type='Shared2FCBBoxHead', + in_channels=256, + fc_out_channels=1024, + roi_feat_size=7, + num_classes=80, + bbox_coder=dict( + type='DeltaXYWHBBoxCoder', + target_means=[0., 0., 0., 0.], + target_stds=[0.1, 0.1, 0.2, 0.2]), + reg_class_agnostic=False, + loss_cls=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0), + loss_bbox=dict(type='L1Loss', loss_weight=1.0)), + mask_roi_extractor=dict( + type='SingleRoIExtractor', + roi_layer=dict(type='RoIAlign', output_size=14, sampling_ratio=0), + out_channels=256, + featmap_strides=[4, 8, 16, 32]), + mask_head=dict( + type='FCNMaskHead', + num_convs=4, + in_channels=256, + conv_out_channels=256, + num_classes=80, + loss_mask=dict( + type='CrossEntropyLoss', use_mask=True, loss_weight=1.0))), + # model training and testing settings + train_cfg=dict( + rpn=dict( + assigner=dict( + type='MaxIoUAssigner', + pos_iou_thr=0.7, + neg_iou_thr=0.3, + min_pos_iou=0.3, + match_low_quality=True, + ignore_iof_thr=-1), + sampler=dict( + type='RandomSampler', + num=256, + pos_fraction=0.5, + neg_pos_ub=-1, + add_gt_as_proposals=False), + allowed_border=-1, + pos_weight=-1, + debug=False), + rpn_proposal=dict( + nms_pre=2000, + max_per_img=1000, + nms=dict(type='nms', iou_threshold=0.7), + min_bbox_size=0), + rcnn=dict( + assigner=dict( + type='MaxIoUAssigner', + pos_iou_thr=0.5, + neg_iou_thr=0.5, + min_pos_iou=0.5, + match_low_quality=True, + ignore_iof_thr=-1), + sampler=dict( + type='RandomSampler', + num=512, + pos_fraction=0.25, + neg_pos_ub=-1, + add_gt_as_proposals=True), + mask_size=28, + pos_weight=-1, + debug=False)), + test_cfg=dict( + rpn=dict( + nms_pre=1000, + max_per_img=1000, + nms=dict(type='nms', iou_threshold=0.7), + min_bbox_size=0), + rcnn=dict( + score_thr=0.05, + nms=dict(type='nms', iou_threshold=0.5), + max_per_img=100, + mask_thr_binary=0.5))) diff --git a/detection/configs/_base_/models/retinanet_r50_fpn.py b/detection/configs/_base_/models/retinanet_r50_fpn.py new file mode 100644 index 0000000..56e43fa --- /dev/null +++ b/detection/configs/_base_/models/retinanet_r50_fpn.py @@ -0,0 +1,60 @@ +# model settings +model = dict( + type='RetinaNet', + backbone=dict( + type='ResNet', + depth=50, + num_stages=4, + out_indices=(0, 1, 2, 3), + frozen_stages=1, + norm_cfg=dict(type='BN', requires_grad=True), + norm_eval=True, + style='pytorch', + init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet50')), + neck=dict( + type='FPN', + in_channels=[256, 512, 1024, 2048], + out_channels=256, + start_level=1, + add_extra_convs='on_input', + num_outs=5), + bbox_head=dict( + type='RetinaHead', + num_classes=80, + in_channels=256, + stacked_convs=4, + feat_channels=256, + anchor_generator=dict( + type='AnchorGenerator', + octave_base_scale=4, + scales_per_octave=3, + ratios=[0.5, 1.0, 2.0], + strides=[8, 16, 32, 64, 128]), + bbox_coder=dict( + type='DeltaXYWHBBoxCoder', + target_means=[.0, .0, .0, .0], + target_stds=[1.0, 1.0, 1.0, 1.0]), + loss_cls=dict( + type='FocalLoss', + use_sigmoid=True, + gamma=2.0, + alpha=0.25, + loss_weight=1.0), + loss_bbox=dict(type='L1Loss', loss_weight=1.0)), + # model training and testing settings + train_cfg=dict( + assigner=dict( + type='MaxIoUAssigner', + pos_iou_thr=0.5, + neg_iou_thr=0.4, + min_pos_iou=0, + ignore_iof_thr=-1), + allowed_border=-1, + pos_weight=-1, + debug=False), + test_cfg=dict( + nms_pre=1000, + min_bbox_size=0, + score_thr=0.05, + nms=dict(type='nms', iou_threshold=0.5), + max_per_img=100)) diff --git a/detection/configs/_base_/models/rpn_r50_caffe_c4.py b/detection/configs/_base_/models/rpn_r50_caffe_c4.py new file mode 100644 index 0000000..8b32ca9 --- /dev/null +++ b/detection/configs/_base_/models/rpn_r50_caffe_c4.py @@ -0,0 +1,58 @@ +# model settings +model = dict( + type='RPN', + backbone=dict( + type='ResNet', + depth=50, + num_stages=3, + strides=(1, 2, 2), + dilations=(1, 1, 1), + out_indices=(2, ), + frozen_stages=1, + norm_cfg=dict(type='BN', requires_grad=False), + norm_eval=True, + style='caffe', + init_cfg=dict( + type='Pretrained', + checkpoint='open-mmlab://detectron2/resnet50_caffe')), + neck=None, + rpn_head=dict( + type='RPNHead', + in_channels=1024, + feat_channels=1024, + anchor_generator=dict( + type='AnchorGenerator', + scales=[2, 4, 8, 16, 32], + ratios=[0.5, 1.0, 2.0], + strides=[16]), + bbox_coder=dict( + type='DeltaXYWHBBoxCoder', + target_means=[.0, .0, .0, .0], + target_stds=[1.0, 1.0, 1.0, 1.0]), + loss_cls=dict( + type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0), + loss_bbox=dict(type='L1Loss', loss_weight=1.0)), + # model training and testing settings + train_cfg=dict( + rpn=dict( + assigner=dict( + type='MaxIoUAssigner', + pos_iou_thr=0.7, + neg_iou_thr=0.3, + min_pos_iou=0.3, + ignore_iof_thr=-1), + sampler=dict( + type='RandomSampler', + num=256, + pos_fraction=0.5, + neg_pos_ub=-1, + add_gt_as_proposals=False), + allowed_border=0, + pos_weight=-1, + debug=False)), + test_cfg=dict( + rpn=dict( + nms_pre=12000, + max_per_img=2000, + nms=dict(type='nms', iou_threshold=0.7), + min_bbox_size=0))) diff --git a/detection/configs/_base_/models/rpn_r50_fpn.py b/detection/configs/_base_/models/rpn_r50_fpn.py new file mode 100644 index 0000000..edaf4d4 --- /dev/null +++ b/detection/configs/_base_/models/rpn_r50_fpn.py @@ -0,0 +1,58 @@ +# model settings +model = dict( + type='RPN', + backbone=dict( + type='ResNet', + depth=50, + num_stages=4, + out_indices=(0, 1, 2, 3), + frozen_stages=1, + norm_cfg=dict(type='BN', requires_grad=True), + norm_eval=True, + style='pytorch', + init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet50')), + neck=dict( + type='FPN', + in_channels=[256, 512, 1024, 2048], + out_channels=256, + num_outs=5), + rpn_head=dict( + type='RPNHead', + in_channels=256, + feat_channels=256, + anchor_generator=dict( + type='AnchorGenerator', + scales=[8], + ratios=[0.5, 1.0, 2.0], + strides=[4, 8, 16, 32, 64]), + bbox_coder=dict( + type='DeltaXYWHBBoxCoder', + target_means=[.0, .0, .0, .0], + target_stds=[1.0, 1.0, 1.0, 1.0]), + loss_cls=dict( + type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0), + loss_bbox=dict(type='L1Loss', loss_weight=1.0)), + # model training and testing settings + train_cfg=dict( + rpn=dict( + assigner=dict( + type='MaxIoUAssigner', + pos_iou_thr=0.7, + neg_iou_thr=0.3, + min_pos_iou=0.3, + ignore_iof_thr=-1), + sampler=dict( + type='RandomSampler', + num=256, + pos_fraction=0.5, + neg_pos_ub=-1, + add_gt_as_proposals=False), + allowed_border=0, + pos_weight=-1, + debug=False)), + test_cfg=dict( + rpn=dict( + nms_pre=2000, + max_per_img=1000, + nms=dict(type='nms', iou_threshold=0.7), + min_bbox_size=0))) diff --git a/detection/configs/_base_/models/ssd300.py b/detection/configs/_base_/models/ssd300.py new file mode 100644 index 0000000..f17df01 --- /dev/null +++ b/detection/configs/_base_/models/ssd300.py @@ -0,0 +1,56 @@ +# model settings +input_size = 300 +model = dict( + type='SingleStageDetector', + backbone=dict( + type='SSDVGG', + depth=16, + with_last_pool=False, + ceil_mode=True, + out_indices=(3, 4), + out_feature_indices=(22, 34), + init_cfg=dict( + type='Pretrained', checkpoint='open-mmlab://vgg16_caffe')), + neck=dict( + type='SSDNeck', + in_channels=(512, 1024), + out_channels=(512, 1024, 512, 256, 256, 256), + level_strides=(2, 2, 1, 1), + level_paddings=(1, 1, 0, 0), + l2_norm_scale=20), + bbox_head=dict( + type='SSDHead', + in_channels=(512, 1024, 512, 256, 256, 256), + num_classes=80, + anchor_generator=dict( + type='SSDAnchorGenerator', + scale_major=False, + input_size=input_size, + basesize_ratio_range=(0.15, 0.9), + strides=[8, 16, 32, 64, 100, 300], + ratios=[[2], [2, 3], [2, 3], [2, 3], [2], [2]]), + bbox_coder=dict( + type='DeltaXYWHBBoxCoder', + target_means=[.0, .0, .0, .0], + target_stds=[0.1, 0.1, 0.2, 0.2])), + # model training and testing settings + train_cfg=dict( + assigner=dict( + type='MaxIoUAssigner', + pos_iou_thr=0.5, + neg_iou_thr=0.5, + min_pos_iou=0., + ignore_iof_thr=-1, + gt_max_assign_all=False), + smoothl1_beta=1., + allowed_border=-1, + pos_weight=-1, + neg_pos_ratio=3, + debug=False), + test_cfg=dict( + nms_pre=1000, + nms=dict(type='nms', iou_threshold=0.45), + min_bbox_size=0, + score_thr=0.02, + max_per_img=200)) +cudnn_benchmark = True diff --git a/detection/configs/_base_/schedules/schedule_1x.py b/detection/configs/_base_/schedules/schedule_1x.py new file mode 100644 index 0000000..13b3783 --- /dev/null +++ b/detection/configs/_base_/schedules/schedule_1x.py @@ -0,0 +1,11 @@ +# optimizer +optimizer = dict(type='SGD', lr=0.02, momentum=0.9, weight_decay=0.0001) +optimizer_config = dict(grad_clip=None) +# learning policy +lr_config = dict( + policy='step', + warmup='linear', + warmup_iters=500, + warmup_ratio=0.001, + step=[8, 11]) +runner = dict(type='EpochBasedRunner', max_epochs=12) diff --git a/detection/configs/_base_/schedules/schedule_3x.py b/detection/configs/_base_/schedules/schedule_3x.py new file mode 100644 index 0000000..e2dc0a5 --- /dev/null +++ b/detection/configs/_base_/schedules/schedule_3x.py @@ -0,0 +1,11 @@ +# optimizer +optimizer = dict(type='SGD', lr=0.02, momentum=0.9, weight_decay=0.0001) +optimizer_config = dict(grad_clip=None) +# learning policy +lr_config = dict( + policy='step', + warmup='linear', + warmup_iters=500, + warmup_ratio=0.001, + step=[27, 33]) +runner = dict(type='EpochBasedRunner', max_epochs=36) diff --git a/detection/configs/coco/README.md b/detection/configs/coco/README.md new file mode 100644 index 0000000..e4eda2f --- /dev/null +++ b/detection/configs/coco/README.md @@ -0,0 +1,51 @@ +# COCO + +## Introduction + +Introduced by Lin et al. in [Microsoft COCO: Common Objects in Context](https://arxiv.org/pdf/1405.0312v3.pdf) + +The MS COCO (Microsoft Common Objects in Context) dataset is a large-scale object detection, segmentation, key-point detection, and captioning dataset. The dataset consists of 328K images. + +Splits: The first version of MS COCO dataset was released in 2014. It contains 164K images split into training (83K), validation (41K) and test (41K) sets. In 2015 additional test set of 81K images was released, including all the previous test images and 40K new images. + +Based on community feedback, in 2017 the training/validation split was changed from 83K/41K to 118K/5K. The new split uses the same images and annotations. The 2017 test set is a subset of 41K images of the 2015 test set. Additionally, the 2017 release contains a new unannotated dataset of 123K images. + + +## Model Zoo + +### Mask R-CNN + FlashInternImage + + +| backbone | schd | box mAP | mask mAP |Config | Download | +| :-----------------: | :---------: | :-----: |:------: | :-----: | :---: | +| FlashInternImage-T | 1x | 48.0 | 43.1 | [config](./mask_rcnn_flash_intern_image_t_fpn_1x_coco.py) | [ckpt](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/mask_rcnn_flash_internimage_t_fpn_1x_coco.pth) \| [log](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/mask_rcnn_flash_internimage_t_fpn_1x_coco.log) | +| FlashInternImage-T | 3x | 49.5 | 44.0 | [config](././mask_rcnn_flash_intern_image_t_fpn_3x_coco.py) | [ckpt](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/mask_rcnn_flash_internimage_t_fpn_3x_coco.pth) \| [log](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/mask_rcnn_flash_internimage_t_fpn_3x_coco.log) | +| FlashInternImage-S | 1x | 49.2 | 44.0 | [config](./mask_rcnn_flash_intern_image_s_fpn_1x_coco.py) | [ckpt](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/mask_rcnn_flash_internimage_s_fpn_1x_coco.pth) \| [log](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/mask_rcnn_flash_internimage_s_fpn_1x_coco.log) | +| FlashInternImage-S | 3x | 50.5 | 44.9 | [config](./mask_rcnn_flash_intern_image_s_fpn_3x_coco.py) | [ckpt](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/mask_rcnn_flash_internimage_s_fpn_3x_coco.pth) \| [log](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/mask_rcnn_flash_internimage_s_fpn_3x_coco.log) | +| FlashInternImage-B | 1x | 50.1 | 44.5 | [config](./mask_rcnn_flash_intern_image_b_fpn_1x_coco.py) | [ckpt](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/mask_rcnn_flash_internimage_b_fpn_1x_coco.pth) \| [log](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/mask_rcnn_flash_internimage_b_fpn_1x_coco.log) | +| FlashInternImage-B | 3x | 50.6 | 45.4 | [config](./mask_rcnn_flash_intern_image_b_fpn_3x_coco.py)| [ckpt](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/mask_rcnn_flash_internimage_b_fpn_3x_coco.pth) \| [log](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/mask_rcnn_flash_internimage_b_fpn_3x_coco.log) | + +- Training speed is measured with A100 GPUs using current code and may be faster than the speed in logs. +- Some logs are our recent newly trained ones. There might be slight differences between the results in logs and our paper. +- Please set `with_cp=True` to save memory if you meet `out-of-memory` issues. + +### Cascade Mask R-CNN + FlashInternImage + +| backbone | schd | box mAP | mask mAP | Config | Download | +| :------------: | :---------: | :-----: | :------: | :---: | :---: | +| FlashInternImage-L | 1x | 55.6 | 48.2 | [config](./cascade_flash_intern_image_l_fpn_1x_coco.py) | [ckpt](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/cascade_flash_internimage_l_fpn_1x_coco.pth) \| [log](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/cascade_flash_internimage_l_fpn_1x_coco.log) +| FlashInternImage-L | 3x | 56.7 | 48.9 | [config](./cascade_flash_intern_image_l_fpn_3x_coco.py) | [ckpt](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/cascade_flash_internimage_l_fpn_3x_coco.pth) | + +- Training speed is measured with A100 GPUs using current code and may be faster than the speed in logs. +- Some logs are our recent newly trained ones. There might be slight differences between the results in logs and our paper. +- Please set `with_cp=True` to save memory if you meet `out-of-memory` issues. + + +### DINO + FlashInternImage +| backbone | lr type | pretrain | schd | box mAP | Config | Download | +| :------------: | :---------: |:---------: | :---------: | :-----: | :---: | :-----: +| FlashInternImage-T | layer-wise lr | ImageNet-1K | 1x | 54.7 | [config](./dino_4scale_flash_internimage_t_1x_coco.py) | [ckpt](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/dino_4scale_flash_internimage_t_1x_coco.pth) \| [log](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/dino_4scale_flash_internimage_t_1x_coco.json) | +| FlashInternImage-S | layer-wise lr | ImageNet-1K | 1x | 55.3 | [config](./dino_4scale_flash_internimage_s_1x_coco.py) | [ckpt](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/dino_4scale_flash_internimage_s_1x_coco.pth) \| [log](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/dino_4scale_flash_internimage_s_1x_coco.log) | +| FlashInternImage-B | layer-wise lr | ImageNet-1K | 1x | 56.0 | [config](./dino_4scale_flash_internimage_b_1x_coco.py) | [ckpt](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/dino_4scale_flash_internimage_b_1x_coco.pth) \| [log](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/dino_4scale_flash_internimage_b_1x_coco.log) | +| FlashInternImage-L | 0.1x backbone lr | ImageNet-22K | 1x | 58.8 | [config](./dino_4scale_flash_internimage_l_1x_coco.py) | [ckpt](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/dino_4scale_flash_internimage_l_1x_coco.pth) \| [log](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/dino_4scale_flash_internimage_l_1x_coco.log) | + diff --git a/detection/configs/coco/cascade_flash_intern_image_l_fpn_1x_coco.py b/detection/configs/coco/cascade_flash_intern_image_l_fpn_1x_coco.py new file mode 100644 index 0000000..c6b0823 --- /dev/null +++ b/detection/configs/coco/cascade_flash_intern_image_l_fpn_1x_coco.py @@ -0,0 +1,148 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2023 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- +_base_ = [ + '../_base_/models/cascade_mask_rcnn_r50_fpn.py', + '../_base_/datasets/coco_instance.py', + '../_base_/schedules/schedule_1x.py', + '../_base_/default_runtime.py' +] +pretrained = 'https://huggingface.co/OpenGVLab/DCNv4/resolve/main/flash_intern_image_l_22k_384.pth' +model = dict( + backbone=dict( + _delete_=True, + type='FlashInternImage', + core_op='DCNv4', + channels=160, + depths=[5, 5, 22, 5], + groups=[10, 20, 40, 80], + mlp_ratio=4., + drop_path_rate=0.4, + norm_layer='LN', + layer_scale=1.0, + offset_scale=2.0, + post_norm=True, + with_cp=False, + dcn_output_bias=True, + mlp_fc2_bias=True, + dw_kernel_size=3, + out_indices=(0, 1, 2, 3), + init_cfg=dict(type='Pretrained', checkpoint=pretrained)), + # We leverage the FPN implemented in ViTDet for stable training, + # and we don't benefit from this FPN in terms of performance. + neck=dict( + type='FPN_vitdet', + in_channels=[160, 320, 640, 1280], + out_channels=256, + norm_cfg=dict(type='LN', requires_grad=True), + use_residual=True, + num_outs=5), + roi_head=dict( + bbox_head=[ + dict( + type='DCNv4FCBBoxHead', + with_dcn=False, + num_shared_convs=4, + num_shared_fcs=1, + in_channels=256, + conv_out_channels=256, + fc_out_channels=1024, + roi_feat_size=7, + num_classes=80, + bbox_coder=dict( + type='DeltaXYWHBBoxCoder', + target_means=[0., 0., 0., 0.], + target_stds=[0.1, 0.1, 0.2, 0.2]), + reg_class_agnostic=False, + reg_decoded_bbox=True, + norm_cfg=dict(type='SyncBN', requires_grad=True), + loss_cls=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0), + loss_bbox=dict(type='GIoULoss', loss_weight=10.0)), + dict( + type='DCNv4FCBBoxHead', + with_dcn=False, + num_shared_convs=4, + num_shared_fcs=1, + in_channels=256, + conv_out_channels=256, + fc_out_channels=1024, + roi_feat_size=7, + num_classes=80, + bbox_coder=dict( + type='DeltaXYWHBBoxCoder', + target_means=[0., 0., 0., 0.], + target_stds=[0.05, 0.05, 0.1, 0.1]), + reg_class_agnostic=False, + reg_decoded_bbox=True, + norm_cfg=dict(type='SyncBN', requires_grad=True), + loss_cls=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0), + loss_bbox=dict(type='GIoULoss', loss_weight=10.0)), + dict( + type='DCNv4FCBBoxHead', + with_dcn=False, + num_shared_convs=4, + num_shared_fcs=1, + in_channels=256, + conv_out_channels=256, + fc_out_channels=1024, + roi_feat_size=7, + num_classes=80, + bbox_coder=dict( + type='DeltaXYWHBBoxCoder', + target_means=[0., 0., 0., 0.], + target_stds=[0.033, 0.033, 0.067, 0.067]), + reg_class_agnostic=False, + reg_decoded_bbox=True, + norm_cfg=dict(type='SyncBN', requires_grad=True), + loss_cls=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0), + loss_bbox=dict(type='GIoULoss', loss_weight=10.0)) +])) +# By default, models are trained on 8 GPUs with 2 images per GPU +data = dict(samples_per_gpu=2) +optimizer = dict( + _delete_=True, type='AdamW', lr=0.0001, weight_decay=0.05, + constructor='CustomLayerDecayOptimizerConstructor', + paramwise_cfg=dict(num_layers=37, layer_decay_rate=0.94, + depths=[5, 5, 22, 5])) +optimizer_config = dict(grad_clip=None) +# fp16 = dict(loss_scale=dict(init_scale=512)) +evaluation = dict(save_best='auto') +checkpoint_config = dict( + interval=1, + max_keep_ckpts=1, + save_last=True, +) + +# Bbox +# Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.556 +# Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=1000 ] = 0.744 +# Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=1000 ] = 0.604 +# Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=1000 ] = 0.388 +# Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=1000 ] = 0.598 +# Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=1000 ] = 0.720 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.670 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=300 ] = 0.670 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=1000 ] = 0.670 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=1000 ] = 0.505 +# Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=1000 ] = 0.714 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=1000 ] = 0.823 + +# Segm +# Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.482 +# Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=1000 ] = 0.720 +# Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=1000 ] = 0.526 +# Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=1000 ] = 0.289 +# Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=1000 ] = 0.514 +# Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=1000 ] = 0.676 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.588 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=300 ] = 0.588 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=1000 ] = 0.588 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=1000 ] = 0.424 +# Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=1000 ] = 0.629 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=1000 ] = 0.749 + diff --git a/detection/configs/coco/cascade_flash_intern_image_l_fpn_3x_coco.py b/detection/configs/coco/cascade_flash_intern_image_l_fpn_3x_coco.py new file mode 100644 index 0000000..d29f811 --- /dev/null +++ b/detection/configs/coco/cascade_flash_intern_image_l_fpn_3x_coco.py @@ -0,0 +1,196 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2023 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- +_base_ = [ + '../_base_/models/cascade_mask_rcnn_r50_fpn.py', + '../_base_/datasets/coco_instance.py', + '../_base_/schedules/schedule_3x.py', + '../_base_/default_runtime.py' +] +pretrained = 'https://huggingface.co/OpenGVLab/DCNv4/resolve/main/flash_intern_image_l_22k_384.pth' +model = dict( + backbone=dict( + _delete_=True, + type='FlashInternImage', + core_op='DCNv4', + channels=160, + depths=[5, 5, 22, 5], + groups=[10, 20, 40, 80], + mlp_ratio=4., + drop_path_rate=0.4, + norm_layer='LN', + layer_scale=1.0, + offset_scale=2.0, + post_norm=True, + with_cp=True, + dcn_output_bias=True, + mlp_fc2_bias=True, + dw_kernel_size=3, + out_indices=(0, 1, 2, 3), + init_cfg=dict(type='Pretrained', checkpoint=pretrained)), + # We leverage the FPN implemented in ViTDet for stable training, + # and we don't benefit from this FPN in terms of performance. + neck=dict( + type='FPN_vitdet', + in_channels=[160, 320, 640, 1280], + out_channels=256, + norm_cfg=dict(type='LN', requires_grad=True), + use_residual=True, + num_outs=5), + roi_head=dict( + bbox_head=[ + dict( + type='ConvFCBBoxHead', + num_shared_convs=4, + num_shared_fcs=1, + in_channels=256, + conv_out_channels=256, + fc_out_channels=1024, + roi_feat_size=7, + num_classes=80, + bbox_coder=dict( + type='DeltaXYWHBBoxCoder', + target_means=[0., 0., 0., 0.], + target_stds=[0.1, 0.1, 0.2, 0.2]), + reg_class_agnostic=False, + reg_decoded_bbox=True, + norm_cfg=dict(type='SyncBN', requires_grad=True), + loss_cls=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0), + loss_bbox=dict(type='GIoULoss', loss_weight=10.0)), + dict( + type='ConvFCBBoxHead', + num_shared_convs=4, + num_shared_fcs=1, + in_channels=256, + conv_out_channels=256, + fc_out_channels=1024, + roi_feat_size=7, + num_classes=80, + bbox_coder=dict( + type='DeltaXYWHBBoxCoder', + target_means=[0., 0., 0., 0.], + target_stds=[0.05, 0.05, 0.1, 0.1]), + reg_class_agnostic=False, + reg_decoded_bbox=True, + norm_cfg=dict(type='SyncBN', requires_grad=True), + loss_cls=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0), + loss_bbox=dict(type='GIoULoss', loss_weight=10.0)), + dict( + type='ConvFCBBoxHead', + num_shared_convs=4, + num_shared_fcs=1, + in_channels=256, + conv_out_channels=256, + fc_out_channels=1024, + roi_feat_size=7, + num_classes=80, + bbox_coder=dict( + type='DeltaXYWHBBoxCoder', + target_means=[0., 0., 0., 0.], + target_stds=[0.033, 0.033, 0.067, 0.067]), + reg_class_agnostic=False, + reg_decoded_bbox=True, + norm_cfg=dict(type='SyncBN', requires_grad=True), + loss_cls=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0), + loss_bbox=dict(type='GIoULoss', loss_weight=10.0)) +])) +img_norm_cfg = dict( + mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) +# augmentation strategy originates from DETR / Sparse RCNN +train_pipeline = [ + dict(type='LoadImageFromFile'), + dict(type='LoadAnnotations', with_bbox=True, with_mask=True), + dict(type='RandomFlip', flip_ratio=0.5), + dict(type='AutoAugment', + policies=[ + [ + dict(type='Resize', + img_scale=[(480, 1333), (512, 1333), (544, 1333), (576, 1333), + (608, 1333), (640, 1333), (672, 1333), (704, 1333), + (736, 1333), (768, 1333), (800, 1333)], + multiscale_mode='value', + keep_ratio=True) + ], + [ + dict(type='Resize', + img_scale=[(400, 1333), (500, 1333), (600, 1333)], + multiscale_mode='value', + keep_ratio=True), + dict(type='RandomCrop', + crop_type='absolute_range', + crop_size=(384, 600), + allow_negative_crop=True), + dict(type='Resize', + img_scale=[(480, 1333), (512, 1333), (544, 1333), + (576, 1333), (608, 1333), (640, 1333), + (672, 1333), (704, 1333), (736, 1333), + (768, 1333), (800, 1333)], + multiscale_mode='value', + override=True, + keep_ratio=True) + ] + ]), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size_divisor=32), + dict(type='DefaultFormatBundle'), + dict(type='Collect', keys=['img', 'gt_bboxes', 'gt_labels', 'gt_masks']), +] +# we use 4 nodes to train this model, with a total batch size of 64 +data = dict( + samples_per_gpu=4, + train=dict(pipeline=train_pipeline)) +# optimizer +optimizer = dict( + _delete_=True, type='AdamW', lr=0.0001 * 2, weight_decay=0.05, + constructor='CustomLayerDecayOptimizerConstructor', + paramwise_cfg=dict(num_layers=37, layer_decay_rate=0.90, + depths=[5, 5, 22, 5], offset_lr_scale=0.01)) +optimizer_config = dict(grad_clip=None) +# fp16 = dict(loss_scale=dict(init_scale=512)) +evaluation = dict(save_best='auto') +checkpoint_config = dict( + interval=1, + max_keep_ckpts=1, + save_last=True, +) +resume_from = None +custom_hooks = [ + dict( + type='ExpMomentumEMAHook', + resume_from=resume_from, + momentum=0.0001, + priority=49) +] + +# Bbox +# Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.567 +# Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=1000 ] = 0.754 +# Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=1000 ] = 0.615 +# Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=1000 ] = 0.410 +# Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=1000 ] = 0.612 +# Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=1000 ] = 0.729 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.685 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=300 ] = 0.685 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=1000 ] = 0.685 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=1000 ] = 0.532 +# Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=1000 ] = 0.733 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=1000 ] = 0.825 + +# Segm +# Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.490 +# Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=1000 ] = 0.732 +# Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=1000 ] = 0.537 +# Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=1000 ] = 0.301 +# Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=1000 ] = 0.527 +# Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=1000 ] = 0.677 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.600 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=300 ] = 0.600 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=1000 ] = 0.600 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=1000 ] = 0.445 +# Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=1000 ] = 0.644 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=1000 ] = 0.751 \ No newline at end of file diff --git a/detection/configs/coco/dino_4scale_flash_internimage_b_1x_coco.py b/detection/configs/coco/dino_4scale_flash_internimage_b_1x_coco.py new file mode 100644 index 0000000..e26cae3 --- /dev/null +++ b/detection/configs/coco/dino_4scale_flash_internimage_b_1x_coco.py @@ -0,0 +1,180 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- +_base_ = [ + '../_base_/datasets/coco_detection.py', + '../_base_/default_runtime.py', + '../_base_/schedules/schedule_1x.py', +] +pretrained = 'https://huggingface.co/OpenGVLab/DCNv4/resolve/main/flash_intern_image_b_1k_224.pth' +model = dict( + type='DINO', + backbone=dict( + type='FlashInternImage', + core_op='DCNv4', + channels=112, + depths=[4, 4, 21, 4], + groups=[7, 14, 28, 56], + mlp_ratio=4., + drop_path_rate=0.3, + norm_layer='LN', + layer_scale=1.0, + offset_scale=0.5, + post_norm=True, + with_cp=True, + dw_kernel_size=3, + out_indices=(1, 2, 3), + init_cfg=dict(type='Pretrained', checkpoint=pretrained)), + neck=dict( + type='ChannelMapper', + in_channels=[224, 448, 896], + kernel_size=1, + out_channels=256, + act_cfg=None, + norm_cfg=dict(type='GN', num_groups=32), + num_outs=4), + bbox_head=dict( + type='DINOHead', + num_query=900, + num_classes=80, + in_channels=2048, + sync_cls_avg_factor=True, + as_two_stage=True, + with_box_refine=True, + dn_cfg=dict( + type='CdnQueryGenerator', + noise_scale=dict(label=0.5, box=1.0), + group_cfg=dict(dynamic=True, num_groups=None, num_dn_queries=100)), + transformer=dict( + type='DinoTransformer', + two_stage_num_proposals=900, + encoder=dict( + type='DetrTransformerEncoder', + num_layers=6, + transformerlayers=dict( + type='BaseTransformerLayer', + attn_cfgs=dict( + type='MultiScaleDeformableAttention', + embed_dims=256, + dropout=0.0), + feedforward_channels=2048, + ffn_dropout=0.0, # 0.1 for DeformDETR + + operation_order=('self_attn', 'norm', 'ffn', 'norm'))), + decoder=dict( + type='DinoTransformerDecoder', + num_layers=6, + return_intermediate=True, + transformerlayers=dict( + type='DetrTransformerDecoderLayer', + attn_cfgs=[ + dict( + type='MultiheadAttention', + embed_dims=256, + num_heads=8, + dropout=0.0), + dict( + type='MultiScaleDeformableAttention', + embed_dims=256, + dropout=0.0), + ], + feedforward_channels=2048, + ffn_dropout=0.0, + operation_order=('self_attn', 'norm', 'cross_attn', 'norm', + 'ffn', 'norm')))), + positional_encoding=dict( + type='SinePositionalEncoding', + num_feats=128, + temperature=20, + normalize=True), + loss_cls=dict( + type='FocalLoss', + use_sigmoid=True, + gamma=2.0, + alpha=0.25, + loss_weight=1.0), + loss_bbox=dict(type='L1Loss', loss_weight=5.0), + loss_iou=dict(type='GIoULoss', loss_weight=2.0)), + # training and testing settings + train_cfg=dict( + assigner=dict( + type='HungarianAssigner', + cls_cost=dict(type='FocalLossCost', weight=2.0), + reg_cost=dict(type='BBoxL1Cost', weight=5.0, box_format='xywh'), + iou_cost=dict(type='IoUCost', iou_mode='giou', weight=2.0))), + test_cfg=dict(max_per_img=300)) +img_norm_cfg = dict( + mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) +# train_pipeline, NOTE the img_scale and the Pad's size_divisor is different +# from the default setting in mmdet. +train_pipeline = [ + dict(type='LoadImageFromFile'), + dict(type='LoadAnnotations', with_bbox=True), + dict(type='RandomFlip', flip_ratio=0.5), + dict( + type='AutoAugment', + policies=[ + [ + dict( + type='Resize', + img_scale=[(480, 1333), (512, 1333), (544, 1333), + (576, 1333), (608, 1333), (640, 1333), + (672, 1333), (704, 1333), (736, 1333), + (768, 1333), (800, 1333)], + multiscale_mode='value', + keep_ratio=True) + ], + [ + dict( + type='Resize', + img_scale=[(400, 4200), (500, 4200), (600, 4200)], + multiscale_mode='value', + keep_ratio=True), + dict( + type='RandomCrop', + crop_type='absolute_range', + crop_size=(384, 600), + allow_negative_crop=False), + dict( + type='Resize', + img_scale=[(480, 1333), (512, 1333), (544, 1333), + (576, 1333), (608, 1333), (640, 1333), + (672, 1333), (704, 1333), (736, 1333), + (768, 1333), (800, 1333)], + multiscale_mode='value', + override=True, + keep_ratio=True) + ] + ]), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size_divisor=32), + dict(type='DefaultFormatBundle'), + dict(type='Collect', keys=['img', 'gt_bboxes', 'gt_labels']) +] +# By default, models are trained on 8 GPUs with 4 images per GPU +data = dict( + samples_per_gpu=4, + train=dict(pipeline=train_pipeline)) +# optimizer +optimizer = dict( + _delete_=True, type='AdamW', lr=0.0002, weight_decay=0.0001, + constructor='CustomLayerDecayOptimizerConstructor', + paramwise_cfg=dict(num_layers=33, layer_decay_rate=0.9, + depths=[4, 4, 21, 4])) +optimizer_config = dict(_delete_=True, grad_clip=dict(max_norm=0.1, norm_type=2)) +# learning policy +lr_config = dict( + policy='step', + warmup='linear', + warmup_iters=500, + warmup_ratio=0.001, + step=[11]) +evaluation = dict(save_best='auto') +checkpoint_config = dict( + interval=1, + max_keep_ckpts=1, + save_last=True, +) +# fp16 = dict(loss_scale=512.) \ No newline at end of file diff --git a/detection/configs/coco/dino_4scale_flash_internimage_l_1x_coco.py b/detection/configs/coco/dino_4scale_flash_internimage_l_1x_coco.py new file mode 100644 index 0000000..52ad3f1 --- /dev/null +++ b/detection/configs/coco/dino_4scale_flash_internimage_l_1x_coco.py @@ -0,0 +1,184 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- +_base_ = [ + '../_base_/datasets/coco_detection.py', + '../_base_/default_runtime.py', + '../_base_/schedules/schedule_1x.py', +] +pretrained = 'https://huggingface.co/OpenGVLab/DCNv4/resolve/main/flash_intern_image_l_22k_384.pth' +model = dict( + type='DINO', + backbone=dict( + type='FlashInternImage', + core_op='DCNv4', + channels=160, + depths=[5, 5, 22, 5], + groups=[10, 20, 40, 80], + mlp_ratio=4., + drop_path_rate=0.4, + norm_layer='LN', + layer_scale=1.0, + offset_scale=2.0, + post_norm=True, + with_cp=True, + dcn_output_bias=True, + mlp_fc2_bias=True, + dw_kernel_size=3, + out_indices=(1, 2, 3), + init_cfg=dict(type='Pretrained', checkpoint=pretrained)), + neck=dict( + type='ChannelMapper', + in_channels=[320, 640, 1280], + kernel_size=1, + out_channels=256, + act_cfg=None, + # norm_cfg=norm_cfg, + norm_cfg=dict(type='GN', num_groups=32), + num_outs=4), + bbox_head=dict( + type='DINOHead', + num_query=900, + num_classes=80, + in_channels=2048, + sync_cls_avg_factor=True, + as_two_stage=True, + with_box_refine=True, + dn_cfg=dict( + type='CdnQueryGenerator', + noise_scale=dict(label=0.5, box=1.0), + group_cfg=dict(dynamic=True, num_groups=None, num_dn_queries=100)), + transformer=dict( + type='DinoTransformer', + two_stage_num_proposals=900, + encoder=dict( + type='DetrTransformerEncoder', + num_layers=6, + transformerlayers=dict( + type='BaseTransformerLayer', + attn_cfgs=dict( + type='MultiScaleDeformableAttention', + embed_dims=256, + dropout=0.0), + feedforward_channels=2048, + ffn_dropout=0.0, # 0.1 for DeformDETR + + operation_order=('self_attn', 'norm', 'ffn', 'norm'))), + decoder=dict( + type='DinoTransformerDecoder', + num_layers=6, + return_intermediate=True, + transformerlayers=dict( + type='DetrTransformerDecoderLayer', + attn_cfgs=[ + dict( + type='MultiheadAttention', + embed_dims=256, + num_heads=8, + dropout=0.0), + dict( + type='MultiScaleDeformableAttention', + embed_dims=256, + dropout=0.0), + ], + feedforward_channels=2048, + ffn_dropout=0.0, + operation_order=('self_attn', 'norm', 'cross_attn', 'norm', + 'ffn', 'norm')))), + positional_encoding=dict( + type='SinePositionalEncoding', + num_feats=128, + temperature=20, + normalize=True), + loss_cls=dict( + type='FocalLoss', + use_sigmoid=True, + gamma=2.0, + alpha=0.25, + loss_weight=1.0), + loss_bbox=dict(type='L1Loss', loss_weight=5.0), + loss_iou=dict(type='GIoULoss', loss_weight=2.0)), + # training and testing settings + train_cfg=dict( + assigner=dict( + type='HungarianAssigner', + cls_cost=dict(type='FocalLossCost', weight=2.0), + reg_cost=dict(type='BBoxL1Cost', weight=5.0, box_format='xywh'), + iou_cost=dict(type='IoUCost', iou_mode='giou', weight=2.0))), + test_cfg=dict(max_per_img=300)) +img_norm_cfg = dict( + mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) +# train_pipeline, NOTE the img_scale and the Pad's size_divisor is different +# from the default setting in mmdet. +train_pipeline = [ + dict(type='LoadImageFromFile'), + dict(type='LoadAnnotations', with_bbox=True), + dict(type='RandomFlip', flip_ratio=0.5), + dict( + type='AutoAugment', + policies=[ + [ + dict( + type='Resize', + img_scale=[(480, 1333), (512, 1333), (544, 1333), + (576, 1333), (608, 1333), (640, 1333), + (672, 1333), (704, 1333), (736, 1333), + (768, 1333), (800, 1333)], + multiscale_mode='value', + keep_ratio=True) + ], + [ + dict( + type='Resize', + img_scale=[(400, 4200), (500, 4200), (600, 4200)], + multiscale_mode='value', + keep_ratio=True), + dict( + type='RandomCrop', + crop_type='absolute_range', + crop_size=(384, 600), + allow_negative_crop=False), + dict( + type='Resize', + img_scale=[(480, 1333), (512, 1333), (544, 1333), + (576, 1333), (608, 1333), (640, 1333), + (672, 1333), (704, 1333), (736, 1333), + (768, 1333), (800, 1333)], + multiscale_mode='value', + override=True, + keep_ratio=True) + ] + ]), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size_divisor=32), + dict(type='DefaultFormatBundle'), + dict(type='Collect', keys=['img', 'gt_bboxes', 'gt_labels']) +] +# By default, models are trained on 8 GPUs with 2 images per GPU +data = dict( + samples_per_gpu=2, + train=dict(pipeline=train_pipeline)) +# optimizer +optimizer = dict( + _delete_=True, type='AdamW', lr=0.0001, weight_decay=0.05, + paramwise_cfg=dict( + custom_keys={ + 'backbone': dict(lr_mult=0.1), +})) +optimizer_config = dict(_delete_=True, grad_clip=dict(max_norm=0.1, norm_type=2)) +# learning policy +lr_config = dict( + policy='step', + warmup='linear', + warmup_iters=500, + warmup_ratio=0.001, + step=[11]) +evaluation = dict(save_best='auto') +checkpoint_config = dict( + interval=1, + max_keep_ckpts=1, + save_last=True, +) +# fp16 = dict(loss_scale=512.) \ No newline at end of file diff --git a/detection/configs/coco/dino_4scale_flash_internimage_s_1x_coco.py b/detection/configs/coco/dino_4scale_flash_internimage_s_1x_coco.py new file mode 100644 index 0000000..63e8119 --- /dev/null +++ b/detection/configs/coco/dino_4scale_flash_internimage_s_1x_coco.py @@ -0,0 +1,180 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2023 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- +_base_ = [ + '../_base_/datasets/coco_detection.py', + '../_base_/default_runtime.py', + '../_base_/schedules/schedule_1x.py', +] +pretrained = 'https://huggingface.co/OpenGVLab/DCNv4/resolve/main/flash_intern_image_s_1k_224.pth' +model = dict( + type='DINO', + backbone=dict( + type='FlashInternImage', + core_op='DCNv4', + channels=80, + depths=[4, 4, 21, 4], + groups=[5, 10, 20, 40], + mlp_ratio=4., + drop_path_rate=0.3, + norm_layer='LN', + layer_scale=1.0, + offset_scale=1.0, + post_norm=True, + with_cp=True, + dw_kernel_size=3, + out_indices=(1, 2, 3), + init_cfg=dict(type='Pretrained', checkpoint=pretrained)), + neck=dict( + type='ChannelMapper', + in_channels=[160, 320, 640], + kernel_size=1, + out_channels=256, + act_cfg=None, + norm_cfg=dict(type='GN', num_groups=32), + num_outs=4), + bbox_head=dict( + type='DINOHead', + num_query=900, + num_classes=80, + in_channels=2048, + sync_cls_avg_factor=True, + as_two_stage=True, + with_box_refine=True, + dn_cfg=dict( + type='CdnQueryGenerator', + noise_scale=dict(label=0.5, box=1.0), + group_cfg=dict(dynamic=True, num_groups=None, num_dn_queries=100)), + transformer=dict( + type='DinoTransformer', + two_stage_num_proposals=900, + encoder=dict( + type='DetrTransformerEncoder', + num_layers=6, + transformerlayers=dict( + type='BaseTransformerLayer', + attn_cfgs=dict( + type='MultiScaleDeformableAttention', + embed_dims=256, + dropout=0.0), + feedforward_channels=2048, + ffn_dropout=0.0, # 0.1 for DeformDETR + + operation_order=('self_attn', 'norm', 'ffn', 'norm'))), + decoder=dict( + type='DinoTransformerDecoder', + num_layers=6, + return_intermediate=True, + transformerlayers=dict( + type='DetrTransformerDecoderLayer', + attn_cfgs=[ + dict( + type='MultiheadAttention', + embed_dims=256, + num_heads=8, + dropout=0.0), + dict( + type='MultiScaleDeformableAttention', + embed_dims=256, + dropout=0.0), + ], + feedforward_channels=2048, + ffn_dropout=0.0, + operation_order=('self_attn', 'norm', 'cross_attn', 'norm', + 'ffn', 'norm')))), + positional_encoding=dict( + type='SinePositionalEncoding', + num_feats=128, + temperature=20, + normalize=True), + loss_cls=dict( + type='FocalLoss', + use_sigmoid=True, + gamma=2.0, + alpha=0.25, + loss_weight=1.0), + loss_bbox=dict(type='L1Loss', loss_weight=5.0), + loss_iou=dict(type='GIoULoss', loss_weight=2.0)), + # training and testing settings + train_cfg=dict( + assigner=dict( + type='HungarianAssigner', + cls_cost=dict(type='FocalLossCost', weight=2.0), + reg_cost=dict(type='BBoxL1Cost', weight=5.0, box_format='xywh'), + iou_cost=dict(type='IoUCost', iou_mode='giou', weight=2.0))), + test_cfg=dict(max_per_img=300)) +img_norm_cfg = dict( + mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) +# train_pipeline, NOTE the img_scale and the Pad's size_divisor is different +# from the default setting in mmdet. +train_pipeline = [ + dict(type='LoadImageFromFile'), + dict(type='LoadAnnotations', with_bbox=True), + dict(type='RandomFlip', flip_ratio=0.5), + dict( + type='AutoAugment', + policies=[ + [ + dict( + type='Resize', + img_scale=[(480, 1333), (512, 1333), (544, 1333), + (576, 1333), (608, 1333), (640, 1333), + (672, 1333), (704, 1333), (736, 1333), + (768, 1333), (800, 1333)], + multiscale_mode='value', + keep_ratio=True) + ], + [ + dict( + type='Resize', + img_scale=[(400, 4200), (500, 4200), (600, 4200)], + multiscale_mode='value', + keep_ratio=True), + dict( + type='RandomCrop', + crop_type='absolute_range', + crop_size=(384, 600), + allow_negative_crop=False), + dict( + type='Resize', + img_scale=[(480, 1333), (512, 1333), (544, 1333), + (576, 1333), (608, 1333), (640, 1333), + (672, 1333), (704, 1333), (736, 1333), + (768, 1333), (800, 1333)], + multiscale_mode='value', + override=True, + keep_ratio=True) + ] + ]), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size_divisor=32), + dict(type='DefaultFormatBundle'), + dict(type='Collect', keys=['img', 'gt_bboxes', 'gt_labels']) +] +# By default, models are trained on 8 GPUs with 4 images per GPU +data = dict( + samples_per_gpu=4, + train=dict(pipeline=train_pipeline)) +# optimizer +optimizer = dict( + _delete_=True, type='AdamW', lr=0.0002, weight_decay=0.0001, + constructor='CustomLayerDecayOptimizerConstructor', + paramwise_cfg=dict(num_layers=33, layer_decay_rate=0.9, + depths=[4, 4, 21, 4])) +optimizer_config = dict(_delete_=True, grad_clip=dict(max_norm=0.1, norm_type=2)) +# learning policy +lr_config = dict( + policy='step', + warmup='linear', + warmup_iters=500, + warmup_ratio=0.001, + step=[11]) +evaluation = dict(save_best='auto') +checkpoint_config = dict( + interval=1, + max_keep_ckpts=1, + save_last=True, +) +# fp16 = dict(loss_scale=512.) \ No newline at end of file diff --git a/detection/configs/coco/dino_4scale_flash_internimage_t_1x_coco.py b/detection/configs/coco/dino_4scale_flash_internimage_t_1x_coco.py new file mode 100644 index 0000000..ebb52f4 --- /dev/null +++ b/detection/configs/coco/dino_4scale_flash_internimage_t_1x_coco.py @@ -0,0 +1,179 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2023 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- +_base_ = [ + '../_base_/datasets/coco_detection.py', + '../_base_/default_runtime.py', + '../_base_/schedules/schedule_1x.py', +] +pretrained = 'https://huggingface.co/OpenGVLab/DCNv4/resolve/main/flash_intern_image_t_1k_224.pth' +model = dict( + type='DINO', + backbone=dict( + type='FlashInternImage', + core_op='DCNv4', + channels=64, + depths=[4, 4, 18, 4], + groups=[4, 8, 16, 32], + mlp_ratio=4., + drop_path_rate=0.2, + norm_layer='LN', + layer_scale=1.0, + offset_scale=1.0, + post_norm=False, + with_cp=True, + out_indices=(1, 2, 3), + init_cfg=dict(type='Pretrained', checkpoint=pretrained)), + neck=dict( + type='ChannelMapper', + in_channels=[128, 256, 512], + kernel_size=1, + out_channels=256, + act_cfg=None, + norm_cfg=dict(type='GN', num_groups=32), + num_outs=4), + bbox_head=dict( + type='DINOHead', + num_query=900, + num_classes=80, + in_channels=2048, + sync_cls_avg_factor=True, + as_two_stage=True, + with_box_refine=True, + dn_cfg=dict( + type='CdnQueryGenerator', + noise_scale=dict(label=0.5, box=1.0), + group_cfg=dict(dynamic=True, num_groups=None, num_dn_queries=100)), + transformer=dict( + type='DinoTransformer', + two_stage_num_proposals=900, + encoder=dict( + type='DetrTransformerEncoder', + num_layers=6, + transformerlayers=dict( + type='BaseTransformerLayer', + attn_cfgs=dict( + type='MultiScaleDeformableAttention', + embed_dims=256, + dropout=0.0), + feedforward_channels=2048, + ffn_dropout=0.0, # 0.1 for DeformDETR + + operation_order=('self_attn', 'norm', 'ffn', 'norm'))), + decoder=dict( + type='DinoTransformerDecoder', + num_layers=6, + return_intermediate=True, + transformerlayers=dict( + type='DetrTransformerDecoderLayer', + attn_cfgs=[ + dict( + type='MultiheadAttention', + embed_dims=256, + num_heads=8, + dropout=0.0), + dict( + type='MultiScaleDeformableAttention', + embed_dims=256, + dropout=0.0), + ], + feedforward_channels=2048, + ffn_dropout=0.0, + operation_order=('self_attn', 'norm', 'cross_attn', 'norm', + 'ffn', 'norm')))), + positional_encoding=dict( + type='SinePositionalEncoding', + num_feats=128, + temperature=20, + normalize=True), + loss_cls=dict( + type='FocalLoss', + use_sigmoid=True, + gamma=2.0, + alpha=0.25, + loss_weight=1.0), + loss_bbox=dict(type='L1Loss', loss_weight=5.0), + loss_iou=dict(type='GIoULoss', loss_weight=2.0)), + # training and testing settings + train_cfg=dict( + assigner=dict( + type='HungarianAssigner', + cls_cost=dict(type='FocalLossCost', weight=2.0), + reg_cost=dict(type='BBoxL1Cost', weight=5.0, box_format='xywh'), + iou_cost=dict(type='IoUCost', iou_mode='giou', weight=2.0))), + test_cfg=dict(max_per_img=300)) +img_norm_cfg = dict( + mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) +# train_pipeline, NOTE the img_scale and the Pad's size_divisor is different +# from the default setting in mmdet. +train_pipeline = [ + dict(type='LoadImageFromFile'), + dict(type='LoadAnnotations', with_bbox=True), + dict(type='RandomFlip', flip_ratio=0.5), + dict( + type='AutoAugment', + policies=[ + [ + dict( + type='Resize', + img_scale=[(480, 1333), (512, 1333), (544, 1333), + (576, 1333), (608, 1333), (640, 1333), + (672, 1333), (704, 1333), (736, 1333), + (768, 1333), (800, 1333)], + multiscale_mode='value', + keep_ratio=True) + ], + [ + dict( + type='Resize', + img_scale=[(400, 4200), (500, 4200), (600, 4200)], + multiscale_mode='value', + keep_ratio=True), + dict( + type='RandomCrop', + crop_type='absolute_range', + crop_size=(384, 600), + allow_negative_crop=False), + dict( + type='Resize', + img_scale=[(480, 1333), (512, 1333), (544, 1333), + (576, 1333), (608, 1333), (640, 1333), + (672, 1333), (704, 1333), (736, 1333), + (768, 1333), (800, 1333)], + multiscale_mode='value', + override=True, + keep_ratio=True) + ] + ]), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size_divisor=32), + dict(type='DefaultFormatBundle'), + dict(type='Collect', keys=['img', 'gt_bboxes', 'gt_labels']) +] +# By default, models are trained on 8 GPUs with 4 images per GPU +data = dict( + samples_per_gpu=4, + train=dict(pipeline=train_pipeline)) +# optimizer +optimizer = dict( + _delete_=True, type='AdamW', lr=0.0002, weight_decay=0.0001, + constructor='CustomLayerDecayOptimizerConstructor', + paramwise_cfg=dict(num_layers=30, layer_decay_rate=0.9, + depths=[4, 4, 18, 4])) +optimizer_config = dict(_delete_=True, grad_clip=dict(max_norm=0.1, norm_type=2)) +# learning policy +lr_config = dict( + policy='step', + warmup='linear', + warmup_iters=500, + warmup_ratio=0.001, + step=[11]) +evaluation = dict(save_best='auto') +checkpoint_config = dict( + interval=1, + max_keep_ckpts=1, + save_last=True, +) +# fp16 = dict(loss_scale=512.) \ No newline at end of file diff --git a/detection/configs/coco/mask_rcnn_flash_intern_image_b_fpn_1x_coco.py b/detection/configs/coco/mask_rcnn_flash_intern_image_b_fpn_1x_coco.py new file mode 100644 index 0000000..7f1d456 --- /dev/null +++ b/detection/configs/coco/mask_rcnn_flash_intern_image_b_fpn_1x_coco.py @@ -0,0 +1,83 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2023 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- +_base_ = [ + '../_base_/models/mask_rcnn_r50_fpn.py', + '../_base_/datasets/coco_instance.py', + '../_base_/schedules/schedule_1x.py', + '../_base_/default_runtime.py' +] +pretrained = 'https://huggingface.co/OpenGVLab/DCNv4/resolve/main/flash_intern_image_b_1k_224.pth' +model = dict( + backbone=dict( + _delete_=True, + type='FlashInternImage', + core_op='DCNv4', + channels=112, + depths=[4, 4, 21, 4], + groups=[7, 14, 28, 56], + mlp_ratio=4., + drop_path_rate=0.3, + norm_layer='LN', + layer_scale=1.0, + offset_scale=0.5, + post_norm=True, + with_cp=True, + dw_kernel_size=3, + out_indices=(0, 1, 2, 3), + init_cfg=dict(type='Pretrained', checkpoint=pretrained)), + # We leverage the FPN implemented in ViTDet for stable training, + # and we don't benefit from this FPN in terms of performance. + neck=dict( + type='FPN_vitdet', + in_channels=[112, 224, 448, 896], + out_channels=256, + norm_cfg=dict(type='LN', requires_grad=True), + use_residual=True, + num_outs=5), +) +# By default, models are trained on 8 GPUs with 2 images per GPU +data = dict(samples_per_gpu=2) +optimizer = dict( + _delete_=True, type='AdamW', lr=0.0001, weight_decay=0.05, + constructor='CustomLayerDecayOptimizerConstructor', + paramwise_cfg=dict(num_layers=33, layer_decay_rate=1.0, + depths=[4, 4, 21, 4])) +optimizer_config = dict(grad_clip=None) +# fp16 = dict(loss_scale=dict(init_scale=512)) +evaluation = dict(save_best='auto') +checkpoint_config = dict( + interval=1, + max_keep_ckpts=1, + save_last=True, +) + +# Bbox +# Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.5005 +# Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=1000 ] = 0.717 +# Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=1000 ] = 0.543 +# Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=1000 ] = 0.322 +# Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=1000 ] = 0.540 +# Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=1000 ] = 0.652 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.617 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=300 ] = 0.617 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=1000 ] = 0.617 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=1000 ] = 0.433 +# Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=1000 ] = 0.658 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=1000 ] = 0.774 + +# Segm +# Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.445 +# Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=1000 ] = 0.687 +# Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=1000 ] = 0.478 +# Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=1000 ] = 0.244 +# Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=1000 ] = 0.477 +# Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=1000 ] = 0.637 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.556 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=300 ] = 0.556 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=1000 ] = 0.556 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=1000 ] = 0.375 +# Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=1000 ] = 0.595 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=1000 ] = 0.720 \ No newline at end of file diff --git a/detection/configs/coco/mask_rcnn_flash_intern_image_b_fpn_3x_coco.py b/detection/configs/coco/mask_rcnn_flash_intern_image_b_fpn_3x_coco.py new file mode 100644 index 0000000..41b0d5f --- /dev/null +++ b/detection/configs/coco/mask_rcnn_flash_intern_image_b_fpn_3x_coco.py @@ -0,0 +1,125 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2023 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- +_base_ = [ + '../_base_/models/mask_rcnn_r50_fpn.py', + '../_base_/datasets/coco_instance.py', + '../_base_/schedules/schedule_3x.py', + '../_base_/default_runtime.py' +] +pretrained = 'https://huggingface.co/OpenGVLab/DCNv4/resolve/main/flash_intern_image_b_1k_224.pth' +model = dict( + backbone=dict( + _delete_=True, + type='FlashInternImage', + core_op='DCNv4', + channels=112, + depths=[4, 4, 21, 4], + groups=[7, 14, 28, 56], + mlp_ratio=4., + drop_path_rate=0.4, + norm_layer='LN', + layer_scale=1.0, + offset_scale=0.5, + post_norm=True, + with_cp=True, + dw_kernel_size=3, + out_indices=(0, 1, 2, 3), + init_cfg=dict(type='Pretrained', checkpoint=pretrained)), + # We leverage the FPN implemented in ViTDet for stable training, + # and we don't benefit from this FPN in terms of performance. + neck=dict( + type='FPN_vitdet', + in_channels=[112, 224, 448, 896], + out_channels=256, + norm_cfg=dict(type='LN', requires_grad=True), + use_residual=True, + num_outs=5)) +# By default, models are trained on 8 GPUs with 2 images per GPU +img_norm_cfg = dict( + mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) +train_pipeline = [ + dict(type='LoadImageFromFile'), + dict(type='LoadAnnotations', with_bbox=True, with_mask=True), + dict(type='RandomFlip', flip_ratio=0.5), + dict(type='AutoAugment', + policies=[ + [ + dict(type='Resize', + img_scale=[(480, 1333), (512, 1333), (544, 1333), (576, 1333), + (608, 1333), (640, 1333), (672, 1333), (704, 1333), + (736, 1333), (768, 1333), (800, 1333)], + multiscale_mode='value', + keep_ratio=True) + ], + [ + dict(type='Resize', + img_scale=[(400, 1333), (500, 1333), (600, 1333)], + multiscale_mode='value', + keep_ratio=True), + dict(type='RandomCrop', + crop_type='absolute_range', + crop_size=(384, 600), + allow_negative_crop=True), + dict(type='Resize', + img_scale=[(480, 1333), (512, 1333), (544, 1333), + (576, 1333), (608, 1333), (640, 1333), + (672, 1333), (704, 1333), (736, 1333), + (768, 1333), (800, 1333)], + multiscale_mode='value', + override=True, + keep_ratio=True) + ] + ]), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size_divisor=32), + dict(type='DefaultFormatBundle'), + dict(type='Collect', keys=['img', 'gt_bboxes', 'gt_labels', 'gt_masks']), +] +# we use 4 nodes to train this model, with a total batch size of 64 +data = dict( + samples_per_gpu=4, + train=dict(pipeline=train_pipeline)) +optimizer = dict( + _delete_=True, type='AdamW', lr=0.0001 * 2, weight_decay=0.05, + constructor='CustomLayerDecayOptimizerConstructor', + paramwise_cfg=dict(num_layers=33, layer_decay_rate=0.9, + depths=[4, 4, 21, 4], offset_lr_scale=0.01)) +optimizer_config = dict(grad_clip=None) +# fp16 = dict(loss_scale=dict(init_scale=512)) +evaluation = dict(save_best='auto') +checkpoint_config = dict( + interval=1, + max_keep_ckpts=1, + save_last=True, +) + +# Bbox +# Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.506 +# Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=1000 ] = 0.726 +# Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=1000 ] = 0.554 +# Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=1000 ] = 0.361 +# Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=1000 ] = 0.549 +# Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=1000 ] = 0.651 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.622 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=300 ] = 0.622 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=1000 ] = 0.622 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=1000 ] = 0.476 +# Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=1000 ] = 0.661 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=1000 ] = 0.764 + +# Segm +# Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.454 +# Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=1000 ] = 0.700 +# Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=1000 ] = 0.492 +# Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=1000 ] = 0.270 +# Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=1000 ] = 0.492 +# Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=1000 ] = 0.638 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.565 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=300 ] = 0.565 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=1000 ] = 0.565 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=1000 ] = 0.408 +# Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=1000 ] = 0.607 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=1000 ] = 0.717 diff --git a/detection/configs/coco/mask_rcnn_flash_intern_image_s_fpn_1x_coco.py b/detection/configs/coco/mask_rcnn_flash_intern_image_s_fpn_1x_coco.py new file mode 100644 index 0000000..7c92e8b --- /dev/null +++ b/detection/configs/coco/mask_rcnn_flash_intern_image_s_fpn_1x_coco.py @@ -0,0 +1,84 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2023 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- +_base_ = [ + '../_base_/models/mask_rcnn_r50_fpn.py', + '../_base_/datasets/coco_instance.py', + '../_base_/schedules/schedule_1x.py', + '../_base_/default_runtime.py' +] +pretrained = 'https://huggingface.co/OpenGVLab/DCNv4/resolve/main/flash_intern_image_s_1k_224.pth' +model = dict( + backbone=dict( + _delete_=True, + type='FlashInternImage', + core_op='DCNv4', + channels=80, + depths=[4, 4, 21, 4], + groups=[5, 10, 20, 40], + mlp_ratio=4., + drop_path_rate=0.3, + norm_layer='LN', + layer_scale=1.0, + offset_scale=1.0, + post_norm=True, + with_cp=True, + dw_kernel_size=3, + out_indices=(0, 1, 2, 3), + init_cfg=dict(type='Pretrained', checkpoint=pretrained)), + # We leverage the FPN implemented in ViTDet for stable training, + # and we don't benefit from this FPN in terms of performance. + neck=dict( + type='FPN_vitdet', + in_channels=[80, 160, 320, 640], + out_channels=256, + norm_cfg=dict(type='LN', requires_grad=True), + use_residual=True, + num_outs=5), + ) +# By default, models are trained on 8 GPUs with 2 images per GPU +data = dict(samples_per_gpu=2) +optimizer = dict( + _delete_=True, type='AdamW', lr=0.0001, weight_decay=0.05, + constructor='CustomLayerDecayOptimizerConstructor', + paramwise_cfg=dict(num_layers=33, layer_decay_rate=1.0, + depths=[4, 4, 21, 4])) +optimizer_config = dict(grad_clip=None) +# fp16 = dict(loss_scale=dict(init_scale=512)) +evaluation = dict(save_best='auto') +checkpoint_config = dict( + interval=1, + max_keep_ckpts=1, + save_last=True, +) + +# BBox +# Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.492 +# Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=1000 ] = 0.707 +# Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=1000 ] = 0.539 +# Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=1000 ] = 0.328 +# Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=1000 ] = 0.531 +# Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=1000 ] = 0.647 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.609 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=300 ] = 0.609 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=1000 ] = 0.609 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=1000 ] = 0.431 +# Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=1000 ] = 0.650 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=1000 ] = 0.768 + + +# Segm +# Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.440 +# Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=1000 ] = 0.678 +# Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=1000 ] = 0.476 +# Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=1000 ] = 0.245 +# Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=1000 ] = 0.470 +# Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=1000 ] = 0.633 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.551 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=300 ] = 0.551 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=1000 ] = 0.551 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=1000 ] = 0.372 +# Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=1000 ] = 0.591 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=1000 ] = 0.714 \ No newline at end of file diff --git a/detection/configs/coco/mask_rcnn_flash_intern_image_s_fpn_3x_coco.py b/detection/configs/coco/mask_rcnn_flash_intern_image_s_fpn_3x_coco.py new file mode 100644 index 0000000..f68dc78 --- /dev/null +++ b/detection/configs/coco/mask_rcnn_flash_intern_image_s_fpn_3x_coco.py @@ -0,0 +1,126 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2023 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- +_base_ = [ + '../_base_/models/mask_rcnn_r50_fpn.py', + '../_base_/datasets/coco_instance.py', + '../_base_/schedules/schedule_3x.py', + '../_base_/default_runtime.py' +] +pretrained = 'https://huggingface.co/OpenGVLab/DCNv4/resolve/main/flash_intern_image_s_1k_224.pth' +model = dict( + backbone=dict( + _delete_=True, + type='FlashInternImage', + core_op='DCNv4', + channels=80, + depths=[4, 4, 21, 4], + groups=[5, 10, 20, 40], + mlp_ratio=4., + drop_path_rate=0.4, + norm_layer='LN', + layer_scale=1.0, + offset_scale=1.0, + post_norm=True, + with_cp=True, + dw_kernel_size=3, + out_indices=(0, 1, 2, 3), + init_cfg=dict(type='Pretrained', checkpoint=pretrained)), + # We leverage the FPN implemented in ViTDet for stable training, + # and we don't benefit from this FPN in terms of performance. + neck=dict( + type='FPN_vitdet', + in_channels=[80, 160, 320, 640], + out_channels=256, + norm_cfg=dict(type='LN', requires_grad=True), + use_residual=True, + num_outs=5)) +# By default, models are trained on 8 GPUs with 2 images per GPU +img_norm_cfg = dict( + mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) +train_pipeline = [ + dict(type='LoadImageFromFile'), + dict(type='LoadAnnotations', with_bbox=True, with_mask=True), + dict(type='RandomFlip', flip_ratio=0.5), + dict(type='AutoAugment', + policies=[ + [ + dict(type='Resize', + img_scale=[(480, 1333), (512, 1333), (544, 1333), (576, 1333), + (608, 1333), (640, 1333), (672, 1333), (704, 1333), + (736, 1333), (768, 1333), (800, 1333)], + multiscale_mode='value', + keep_ratio=True) + ], + [ + dict(type='Resize', + img_scale=[(400, 1333), (500, 1333), (600, 1333)], + multiscale_mode='value', + keep_ratio=True), + dict(type='RandomCrop', + crop_type='absolute_range', + crop_size=(384, 600), + allow_negative_crop=True), + dict(type='Resize', + img_scale=[(480, 1333), (512, 1333), (544, 1333), + (576, 1333), (608, 1333), (640, 1333), + (672, 1333), (704, 1333), (736, 1333), + (768, 1333), (800, 1333)], + multiscale_mode='value', + override=True, + keep_ratio=True) + ] + ]), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size_divisor=32), + dict(type='DefaultFormatBundle'), + dict(type='Collect', keys=['img', 'gt_bboxes', 'gt_labels', 'gt_masks']), +] +# we use 4 nodes to train this model, with a total batch size of 64 +data = dict( + samples_per_gpu=8, + train=dict(pipeline=train_pipeline)) +optimizer = dict( + _delete_=True, type='AdamW', lr=0.0001 * 2, weight_decay=0.05, + constructor='CustomLayerDecayOptimizerConstructor', + paramwise_cfg=dict(num_layers=33, layer_decay_rate=1.0, + depths=[4, 4, 21, 4])) +optimizer_config = dict(grad_clip=None) +# fp16 = dict(loss_scale=dict(init_scale=512)) +evaluation = dict(save_best='auto') +checkpoint_config = dict( + interval=1, + max_keep_ckpts=1, + save_last=True, +) + + +# BBox +# Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.505 +# Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=1000 ] = 0.720 +# Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=1000 ] = 0.552 +# Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=1000 ] = 0.341 +# Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=1000 ] = 0.545 +# Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=1000 ] = 0.647 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.623 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=300 ] = 0.623 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=1000 ] = 0.623 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=1000 ] = 0.461 +# Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=1000 ] = 0.657 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=1000 ] = 0.769 + +# Segm +# Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.449 +# Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=1000 ] = 0.690 +# Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=1000 ] = 0.484 +# Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=1000 ] = 0.252 +# Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=1000 ] = 0.480 +# Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=1000 ] = 0.635 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.560 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=300 ] = 0.560 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=1000 ] = 0.560 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=1000 ] = 0.396 +# Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=1000 ] = 0.596 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=1000 ] = 0.715 \ No newline at end of file diff --git a/detection/configs/coco/mask_rcnn_flash_intern_image_t_fpn_1x_coco.py b/detection/configs/coco/mask_rcnn_flash_intern_image_t_fpn_1x_coco.py new file mode 100644 index 0000000..4ec5096 --- /dev/null +++ b/detection/configs/coco/mask_rcnn_flash_intern_image_t_fpn_1x_coco.py @@ -0,0 +1,81 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2023 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- +_base_ = [ + '../_base_/models/mask_rcnn_r50_fpn.py', + '../_base_/datasets/coco_instance.py', + '../_base_/schedules/schedule_1x.py', + '../_base_/default_runtime.py' +] +pretrained = 'https://huggingface.co/OpenGVLab/DCNv4/resolve/main/flash_intern_image_t_1k_224.pth' +model = dict( + backbone=dict( + _delete_=True, + type='FlashInternImage', + core_op='DCNv4', + channels=64, + depths=[4, 4, 18, 4], + groups=[4, 8, 16, 32], + mlp_ratio=4., + drop_path_rate=0.2, + norm_layer='LN', + layer_scale=1.0, + offset_scale=1.0, + post_norm=False, + with_cp=True, + out_indices=(0, 1, 2, 3), + init_cfg=dict(type='Pretrained', checkpoint=pretrained)), + # We leverage the FPN implemented in ViTDet for stable training, + # and we don't benefit from this FPN in terms of performance. + neck=dict( + type='FPN_vitdet', + in_channels=[64, 128, 256, 512], + out_channels=256, + norm_cfg=dict(type='LN', requires_grad=True), + use_residual=True, + num_outs=5) + ) + +# By default, models are trained on 8 GPUs with 2 images per GPU +data = dict(samples_per_gpu=2) +optimizer = dict( + _delete_=True, type='AdamW', lr=0.0001, weight_decay=0.05, + constructor='CustomLayerDecayOptimizerConstructor', + paramwise_cfg=dict(num_layers=30, layer_decay_rate=1.0, + depths=[4, 4, 18, 4])) +optimizer_config = dict(grad_clip=None) +# fp16 = dict(loss_scale=dict(init_scale=512)) +evaluation = dict(save_best='auto') +checkpoint_config = dict( + interval=1, + max_keep_ckpts=1, + save_last=True, +) + +# BBox +# Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.480 +# Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=1000 ] = 0.695 +# Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=1000 ] = 0.528 +# Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=1000 ] = 0.303 +# Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=1000 ] = 0.515 +# Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=1000 ] = 0.629 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.599 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=300 ] = 0.599 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=1000 ] = 0.599 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=1000 ] = 0.408 +# Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=1000 ] = 0.637 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=1000 ] = 0.750 + +# Segm +# Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.431 +# Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=1000 ] = 0.667 Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=1000 ] = 0.463 Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=1000 ] = 0.225 +# Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=1000 ] = 0.461 +# Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=1000 ] = 0.622 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.543 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=300 ] = 0.543 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=1000 ] = 0.543 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=1000 ] = 0.352 +# Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=1000 ] = 0.581 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=1000 ] = 0.705 \ No newline at end of file diff --git a/detection/configs/coco/mask_rcnn_flash_intern_image_t_fpn_3x_coco.py b/detection/configs/coco/mask_rcnn_flash_intern_image_t_fpn_3x_coco.py new file mode 100644 index 0000000..e17ac6b --- /dev/null +++ b/detection/configs/coco/mask_rcnn_flash_intern_image_t_fpn_3x_coco.py @@ -0,0 +1,124 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2023 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- +_base_ = [ + '../_base_/models/mask_rcnn_r50_fpn.py', + '../_base_/datasets/coco_instance.py', + '../_base_/schedules/schedule_3x.py', + '../_base_/default_runtime.py' +] +pretrained = 'https://huggingface.co/OpenGVLab/DCNv4/resolve/main/flash_intern_image_t_1k_224.pth' +model = dict( + backbone=dict( + _delete_=True, + type='FlashInternImage', + core_op='DCNv4', + channels=64, + depths=[4, 4, 18, 4], + groups=[4, 8, 16, 32], + mlp_ratio=4., + drop_path_rate=0.2, + norm_layer='LN', + layer_scale=1.0, + offset_scale=1.0, + post_norm=False, + with_cp=True, + out_indices=(0, 1, 2, 3), + init_cfg=dict(type='Pretrained', checkpoint=pretrained)), + # We leverage the FPN implemented in ViTDet for stable training, + # and we don't benefit from this FPN in terms of performance. + neck=dict( + type='FPN_vitdet', + in_channels=[64, 128, 256, 512], + out_channels=256, + norm_cfg=dict(type='LN', requires_grad=True), + use_residual=True, + num_outs=5),) +# By default, models are trained on 8 GPUs with 2 images per GPU +img_norm_cfg = dict( + mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) +train_pipeline = [ + dict(type='LoadImageFromFile'), + dict(type='LoadAnnotations', with_bbox=True, with_mask=True), + dict(type='RandomFlip', flip_ratio=0.5), + dict(type='AutoAugment', + policies=[ + [ + dict(type='Resize', + img_scale=[(480, 1333), (512, 1333), (544, 1333), (576, 1333), + (608, 1333), (640, 1333), (672, 1333), (704, 1333), + (736, 1333), (768, 1333), (800, 1333)], + multiscale_mode='value', + keep_ratio=True) + ], + [ + dict(type='Resize', + img_scale=[(400, 1333), (500, 1333), (600, 1333)], + multiscale_mode='value', + keep_ratio=True), + dict(type='RandomCrop', + crop_type='absolute_range', + crop_size=(384, 600), + allow_negative_crop=True), + dict(type='Resize', + img_scale=[(480, 1333), (512, 1333), (544, 1333), + (576, 1333), (608, 1333), (640, 1333), + (672, 1333), (704, 1333), (736, 1333), + (768, 1333), (800, 1333)], + multiscale_mode='value', + override=True, + keep_ratio=True) + ] + ]), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size_divisor=32), + dict(type='DefaultFormatBundle'), + dict(type='Collect', keys=['img', 'gt_bboxes', 'gt_labels', 'gt_masks']), +] +# we use 4 nodes to train this model, with a total batch size of 64 +data = dict( + samples_per_gpu=2, + train=dict(pipeline=train_pipeline)) +optimizer = dict( + _delete_=True, type='AdamW', lr=0.0001, weight_decay=0.05, + constructor='CustomLayerDecayOptimizerConstructor', + paramwise_cfg=dict(num_layers=30, layer_decay_rate=1.0, + depths=[4, 4, 18, 4])) +optimizer_config = dict(grad_clip=None) +# fp16 = dict(loss_scale=dict(init_scale=512)) +evaluation = dict(save_best='auto') +checkpoint_config = dict( + interval=1, + max_keep_ckpts=1, + save_last=True, +) + +# BBox +# Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.495 +# Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=1000 ] = 0.707 +# Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=1000 ] = 0.543 +# Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=1000 ] = 0.339 +# Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=1000 ] = 0.532 +# Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=1000 ] = 0.641 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.607 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=300 ] = 0.607 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=1000 ] = 0.607 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=1000 ] = 0.443 +# Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=1000 ] = 0.643 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=1000 ] = 0.752 + +# Segm +# Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.440 +# Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=1000 ] = 0.677 +# Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=1000 ] = 0.474 +# Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=1000 ] = 0.255 +# Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=1000 ] = 0.473 +# Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=1000 ] = 0.624 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.545 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=300 ] = 0.545 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=1000 ] = 0.545 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=1000 ] = 0.380 +# Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=1000 ] = 0.582 +# Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=1000 ] = 0.704 \ No newline at end of file diff --git a/detection/dist_test.sh b/detection/dist_test.sh new file mode 100755 index 0000000..5f6be76 --- /dev/null +++ b/detection/dist_test.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +CONFIG=$1 +CHECKPOINT=$2 +GPUS=$3 +PORT=${PORT:-29511} +PYTHONPATH="$(dirname $0)/..":$PYTHONPATH \ +python -m torch.distributed.launch --nproc_per_node=$GPUS --master_port=$PORT \ + $(dirname "$0")/test.py $CONFIG $CHECKPOINT --launcher pytorch ${@:4} diff --git a/detection/dist_train.sh b/detection/dist_train.sh new file mode 100755 index 0000000..263a00e --- /dev/null +++ b/detection/dist_train.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +CONFIG=$1 +GPUS=$2 +PORT=${PORT:-29500} +# cat /proc/193481/cmdline +PYTHONPATH="$(dirname $0)/..":$PYTHONPATH \ +python -m torch.distributed.launch --nproc_per_node=$GPUS --master_port=63667 \ + $(dirname "$0")/train.py $CONFIG --launcher pytorch ${@:3} \ No newline at end of file diff --git a/detection/get_flops.py b/detection/get_flops.py new file mode 100644 index 0000000..a0073f7 --- /dev/null +++ b/detection/get_flops.py @@ -0,0 +1,120 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import argparse + +import numpy as np +import torch +from mmcv import Config, DictAction + +from mmdet.models import build_detector +import mmcv_custom # noqa: F401,F403 +import mmdet_custom # noqa: F401,F403 + +try: + from mmcv.cnn.utils.flops_counter import flops_to_string, params_to_string + from mmcv.cnn import get_model_complexity_info +except ImportError: + raise ImportError('Please upgrade mmcv to >0.6.2') + + +def parse_args(): + parser = argparse.ArgumentParser(description='Train a detector') + parser.add_argument('config', help='train config file path') + parser.add_argument( + '--shape', + type=int, + nargs='+', + default=[800, 1280], + help='input image size') + parser.add_argument( + '--cfg-options', + nargs='+', + action=DictAction, + help='override some settings in the used config, the key-value pair ' + 'in xxx=yyy format will be merged into config file. If the value to ' + 'be overwritten is a list, it should be like key="[a,b]" or key=a,b ' + 'It also allows nested list/tuple values, e.g. key="[(a,b),(c,d)]" ' + 'Note that the quotation marks are necessary and that no white space ' + 'is allowed.') + parser.add_argument( + '--size-divisor', + type=int, + default=32, + help='Pad the input image, the minimum size that is divisible ' + 'by size_divisor, -1 means do not pad the image.') + args = parser.parse_args() + return args + + +def dcnv3_flops(n, k, c): + return 5 * n * k * c + + +def get_flops(model, input_shape): + flops, params = get_model_complexity_info(model, input_shape, as_strings=False) + + backbone = model.backbone + backbone_name = type(backbone).__name__ + _, H, W = input_shape + + temp = 0 + if 'InternImage' in backbone_name: + depths = backbone.depths # [4, 4, 18, 4] + for idx, depth in enumerate(depths): + channels = backbone.channels * (2 ** idx) + h = H / (4 * (2 ** idx)) + w = W / (4 * (2 ** idx)) + temp += depth * dcnv3_flops(n=h * w, k=3 * 3, c=channels) + + flops = flops + temp + return flops_to_string(flops), params_to_string(params) + + +if __name__ == '__main__': + + args = parse_args() + + if len(args.shape) == 1: + h = w = args.shape[0] + elif len(args.shape) == 2: + h, w = args.shape + else: + raise ValueError('invalid input shape') + orig_shape = (3, h, w) + divisor = args.size_divisor + if divisor > 0: + h = int(np.ceil(h / divisor)) * divisor + w = int(np.ceil(w / divisor)) * divisor + + input_shape = (3, h, w) + + cfg = Config.fromfile(args.config) + if args.cfg_options is not None: + cfg.merge_from_dict(args.cfg_options) + + model = build_detector( + cfg.model, + train_cfg=cfg.get('train_cfg'), + test_cfg=cfg.get('test_cfg')) + + if torch.cuda.is_available(): + model.cuda() + model.eval() + if hasattr(model, 'forward_dummy'): + model.forward = model.forward_dummy + else: + raise NotImplementedError( + 'FLOPs counter is currently not currently supported with {}'. + format(model.__class__.__name__)) + + flops, params = get_flops(model, input_shape) + split_line = '=' * 30 + + if divisor > 0 and \ + input_shape != orig_shape: + print(f'{split_line}\nUse size divisor set input shape ' + f'from {orig_shape} to {input_shape}\n') + print(f'{split_line}\nInput shape: {input_shape}\n' + f'Flops: {flops}\nParams: {params}\n{split_line}') + print('!!!Please be cautious if you use the results in papers. ' + 'You may need to check if all ops are supported and verify that the ' + 'flops computation is correct.') diff --git a/detection/image_demo.py b/detection/image_demo.py new file mode 100644 index 0000000..303df31 --- /dev/null +++ b/detection/image_demo.py @@ -0,0 +1,61 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import asyncio +from argparse import ArgumentParser + +from mmdet.apis import (async_inference_detector, inference_detector, + init_detector, show_result_pyplot) +import mmcv +import mmcv_custom # noqa: F401,F403 +import mmdet_custom # noqa: F401,F403 +import os.path as osp + + +def parse_args(): + parser = ArgumentParser() + parser.add_argument('img', help='Image file') + parser.add_argument('config', help='Config file') + parser.add_argument('checkpoint', help='Checkpoint file') + parser.add_argument('--out', type=str, default="demo", help='out dir') + parser.add_argument( + '--device', default='cuda:0', help='Device used for inference') + parser.add_argument( + '--palette', + default='coco', + choices=['coco', 'voc', 'citys', 'random'], + help='Color palette used for visualization') + parser.add_argument( + '--score-thr', type=float, default=0.3, help='bbox score threshold') + parser.add_argument( + '--async-test', + action='store_true', + help='whether to set async options for async inference.') + args = parser.parse_args() + return args + + +def main(args): + # build the model from a config file and a checkpoint file + model = init_detector(args.config, args.checkpoint, device=args.device) + # test a single image + result = inference_detector(model, args.img) + + mmcv.mkdir_or_exist(args.out) + out_file = osp.join(args.out, osp.basename(args.img)) + # show the results + model.show_result( + args.img, + result, + score_thr=args.score_thr, + show=False, + bbox_color=args.palette, + text_color=(200, 200, 200), + mask_color=args.palette, + out_file=out_file + ) + print(f"Result is save at {out_file}") + + + +if __name__ == '__main__': + args = parse_args() + main(args) \ No newline at end of file diff --git a/detection/mmcv_custom/__init__.py b/detection/mmcv_custom/__init__.py new file mode 100644 index 0000000..8a2c005 --- /dev/null +++ b/detection/mmcv_custom/__init__.py @@ -0,0 +1,9 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + +# -*- coding: utf-8 -*- +from .custom_layer_decay_optimizer_constructor import CustomLayerDecayOptimizerConstructor +__all__ = ['CustomLayerDecayOptimizerConstructor'] diff --git a/detection/mmcv_custom/custom_layer_decay_optimizer_constructor.py b/detection/mmcv_custom/custom_layer_decay_optimizer_constructor.py new file mode 100644 index 0000000..f632b80 --- /dev/null +++ b/detection/mmcv_custom/custom_layer_decay_optimizer_constructor.py @@ -0,0 +1,161 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + +""" +Mostly copy-paste from BEiT library: +https://github.com/microsoft/unilm/blob/master/beit/semantic_segmentation/mmcv_custom/layer_decay_optimizer_constructor.py +""" + +import json + +from mmcv.runner import OPTIMIZER_BUILDERS, DefaultOptimizerConstructor +from mmcv.runner import get_dist_info +from mmdet.utils import get_root_logger + + +def get_num_layer_for_swin(var_name, num_max_layer, depths): + if var_name.startswith("backbone.patch_embed"): + return 0 + elif "level_embeds" in var_name: + return 0 + elif var_name.startswith("backbone.layers") or var_name.startswith( + "backbone.levels"): + if var_name.split('.')[3] not in ['downsample', 'norm']: + stage_id = int(var_name.split('.')[2]) + layer_id = int(var_name.split('.')[4]) + # layers for Swin-Large: [2, 2, 18, 2] + if stage_id == 0: + return layer_id + 1 + elif stage_id == 1: + return layer_id + 1 + depths[0] + elif stage_id == 2: + return layer_id + 1 + depths[0] + depths[1] + else: + return layer_id + 1 + depths[0] + depths[1] + depths[2] + else: + stage_id = int(var_name.split('.')[2]) + if stage_id == 0: + return 1 + depths[0] + elif stage_id == 1: + return 1 + depths[0] + depths[1] + elif stage_id == 2: + return 1 + depths[0] + depths[1] + depths[2] + else: + return 1 + depths[0] + depths[1] + depths[2] + else: + return num_max_layer - 1 + + +@OPTIMIZER_BUILDERS.register_module() +class CustomLayerDecayOptimizerConstructor(DefaultOptimizerConstructor): + + def add_params(self, params, module, prefix='', is_dcn_module=None): + """Add all parameters of module to the params list. + The parameters of the given module will be added to the list of param + groups, with specific rules defined by paramwise_cfg. + Args: + params (list[dict]): A list of param groups, it will be modified + in place. + module (nn.Module): The module to be added. + prefix (str): The prefix of the module + is_dcn_module (int|float|None): If the current module is a + submodule of DCN, `is_dcn_module` will be passed to + control conv_offset layer's learning rate. Defaults to None. + """ + parameter_groups = {} + logger = get_root_logger() + logger.info(self.paramwise_cfg) + backbone_small_lr = self.paramwise_cfg.get('backbone_small_lr', False) + dino_head = self.paramwise_cfg.get('dino_head', False) + num_layers = self.paramwise_cfg.get('num_layers') + 2 + layer_decay_rate = self.paramwise_cfg.get('layer_decay_rate') + depths = self.paramwise_cfg.get('depths') + offset_lr_scale = self.paramwise_cfg.get('offset_lr_scale', 1.0) + + logger.info("Build CustomLayerDecayOptimizerConstructor %f - %d" % + (layer_decay_rate, num_layers)) + weight_decay = self.base_wd + + custom_keys = self.paramwise_cfg.get('custom_keys', {}) + # first sort with alphabet order and then sort with reversed len of str + sorted_keys = sorted(custom_keys.keys()) + for name, param in module.named_parameters(): + if not param.requires_grad: + continue # frozen weights + if len(param.shape) == 1 or name.endswith(".bias") or \ + "relative_position" in name or \ + "norm" in name or\ + "sampling_offsets" in name: + group_name = "no_decay" + this_weight_decay = 0. + else: + group_name = "decay" + this_weight_decay = weight_decay + + layer_id = get_num_layer_for_swin(name, num_layers, depths) + if layer_id == num_layers - 1 and dino_head and \ + ("sampling_offsets" in name or "reference_points" in name): + group_name = "layer_%d_%s_0.1x" % (layer_id, group_name) + elif "sampling_offsets" in name or "reference_points" in name: + group_name = "layer_%d_%s_offset_lr_scale" % (layer_id, + group_name) + elif "offset_mask" in name and "offset_mask_dw" not in name: + group_name = "layer_%d_%s_offset_lr_scale" % (layer_id, + group_name) + elif name.endswith('offset'): + group_name = "layer_%d_%s_offset_lr_scale" % (layer_id, + group_name) + else: + group_name = "layer_%d_%s" % (layer_id, group_name) + + # if the parameter match one of the custom keys, ignore other rules + this_lr_multi = 1. + for key in sorted_keys: + if key in f'{name}': + logger.info(custom_keys[key]) + lr_mult = custom_keys[key].get('lr_mult', 1.) + this_lr_multi = lr_mult + group_name = "%s_%s" % (group_name, key) + break + if group_name not in parameter_groups: + scale = layer_decay_rate ** (num_layers - layer_id - 1) + if scale < 1 and backbone_small_lr == True: + scale = scale * 0.1 + if "0.1x" in group_name: + scale = scale * 0.1 + if "offset_lr_scale" in group_name: + scale = scale * offset_lr_scale + + parameter_groups[group_name] = { + "weight_decay": this_weight_decay, + "params": [], + "param_names": [], + "lr_scale": scale, + "group_name": group_name, + "lr": scale * self.base_lr * this_lr_multi, + } + + parameter_groups[group_name]["params"].append(param) + parameter_groups[group_name]["param_names"].append(name) + rank, _ = get_dist_info() + if rank == 0: + to_display = {} + for key in parameter_groups: + to_display[key] = { + "param_names": parameter_groups[key]["param_names"], + "lr_scale": parameter_groups[key]["lr_scale"], + "lr": parameter_groups[key]["lr"], + "weight_decay": parameter_groups[key]["weight_decay"], + } + logger.info("Param groups = %s" % json.dumps(to_display, indent=2)) + + # state_dict = module.state_dict() + # for group_name in parameter_groups: + # group = parameter_groups[group_name] + # for name in group["param_names"]: + # group["params"].append(state_dict[name]) + + params.extend(parameter_groups.values()) \ No newline at end of file diff --git a/detection/mmdet_custom/__init__.py b/detection/mmdet_custom/__init__.py new file mode 100644 index 0000000..790f5ee --- /dev/null +++ b/detection/mmdet_custom/__init__.py @@ -0,0 +1,8 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + +from .models import * # noqa: F401,F403 +from .datasets import * \ No newline at end of file diff --git a/detection/mmdet_custom/datasets/__init__.py b/detection/mmdet_custom/datasets/__init__.py new file mode 100644 index 0000000..d6978b0 --- /dev/null +++ b/detection/mmdet_custom/datasets/__init__.py @@ -0,0 +1,7 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + +from .crowd_human import CrowdHumanDataset \ No newline at end of file diff --git a/detection/mmdet_custom/datasets/crowd_human.py b/detection/mmdet_custom/datasets/crowd_human.py new file mode 100644 index 0000000..d7821fd --- /dev/null +++ b/detection/mmdet_custom/datasets/crowd_human.py @@ -0,0 +1,529 @@ +import itertools +import logging +import os.path as osp +import tempfile +import warnings +from collections import OrderedDict + +import mmcv +import numpy as np +from mmcv.utils import print_log +from terminaltables import AsciiTable + +from mmdet.core import eval_recalls +from mmdet.datasets.api_wrappers import COCO, COCOeval + +from mmdet.datasets.custom import CustomDataset +from mmdet.datasets.builder import DATASETS + + +@DATASETS.register_module() +class CrowdHumanDataset(CustomDataset): + + CLASSES = ('person', ) + + def load_annotations(self, ann_file): + """Load annotation from COCO style annotation file. + Args: + ann_file (str): Path of annotation file. + Returns: + list[dict]: Annotation info from COCO api. + """ + + self.coco = COCO(ann_file) + # The order of returned `cat_ids` will not + # change with the order of the CLASSES + self.cat_ids = self.coco.get_cat_ids(cat_names=self.CLASSES) + + self.cat2label = {cat_id: i for i, cat_id in enumerate(self.cat_ids)} + self.img_ids = self.coco.get_img_ids() + data_infos = [] + total_ann_ids = [] + for i in self.img_ids: + info = self.coco.load_imgs([i])[0] + info['filename'] = info['file_name'] + data_infos.append(info) + ann_ids = self.coco.get_ann_ids(img_ids=[i]) + total_ann_ids.extend(ann_ids) + assert len(set(total_ann_ids)) == len( + total_ann_ids), f"Annotation ids in '{ann_file}' are not unique!" + return data_infos + + def get_ann_info(self, idx): + """Get COCO annotation by index. + Args: + idx (int): Index of data. + Returns: + dict: Annotation info of specified index. + """ + + img_id = self.data_infos[idx]['id'] + ann_ids = self.coco.get_ann_ids(img_ids=[img_id]) + ann_info = self.coco.load_anns(ann_ids) + return self._parse_ann_info(self.data_infos[idx], ann_info) + + def get_cat_ids(self, idx): + """Get COCO category ids by index. + Args: + idx (int): Index of data. + Returns: + list[int]: All categories in the image of specified index. + """ + + img_id = self.data_infos[idx]['id'] + ann_ids = self.coco.get_ann_ids(img_ids=[img_id]) + ann_info = self.coco.load_anns(ann_ids) + return [ann['category_id'] for ann in ann_info] + + def _filter_imgs(self, min_size=32): + """Filter images too small or without ground truths.""" + valid_inds = [] + # obtain images that contain annotation + ids_with_ann = set(_['image_id'] for _ in self.coco.anns.values()) + # obtain images that contain annotations of the required categories + ids_in_cat = set() + for i, class_id in enumerate(self.cat_ids): + ids_in_cat |= set(self.coco.cat_img_map[class_id]) + # merge the image id sets of the two conditions and use the merged set + # to filter out images if self.filter_empty_gt=True + ids_in_cat &= ids_with_ann + + valid_img_ids = [] + for i, img_info in enumerate(self.data_infos): + img_id = self.img_ids[i] + if self.filter_empty_gt and img_id not in ids_in_cat: + continue + if min(img_info['width'], img_info['height']) >= min_size: + valid_inds.append(i) + valid_img_ids.append(img_id) + self.img_ids = valid_img_ids + return valid_inds + + def _parse_ann_info(self, img_info, ann_info): + """Parse bbox and mask annotation. + Args: + ann_info (list[dict]): Annotation info of an image. + with_mask (bool): Whether to parse mask annotations. + Returns: + dict: A dict containing the following keys: bboxes, bboxes_ignore,\ + labels, masks, seg_map. "masks" are raw annotations and not \ + decoded into binary masks. + """ + gt_bboxes = [] + gt_labels = [] + gt_bboxes_ignore = [] + gt_masks_ann = [] + for i, ann in enumerate(ann_info): + if ann.get('ignore', False): + continue + x1, y1, w, h = ann['bbox'] + inter_w = max(0, min(x1 + w, img_info['width']) - max(x1, 0)) + inter_h = max(0, min(y1 + h, img_info['height']) - max(y1, 0)) + if inter_w * inter_h == 0: + continue + if ann['area'] <= 0 or w < 1 or h < 1: + continue + if ann['category_id'] not in self.cat_ids: + continue + bbox = [x1, y1, x1 + w, y1 + h] + if ann.get('iscrowd', False): + gt_bboxes_ignore.append(bbox) + else: + gt_bboxes.append(bbox) + gt_labels.append(self.cat2label[ann['category_id']]) + gt_masks_ann.append(ann.get('segmentation', None)) + + if gt_bboxes: + gt_bboxes = np.array(gt_bboxes, dtype=np.float32) + gt_labels = np.array(gt_labels, dtype=np.int64) + else: + gt_bboxes = np.zeros((0, 4), dtype=np.float32) + gt_labels = np.array([], dtype=np.int64) + + if gt_bboxes_ignore: + gt_bboxes_ignore = np.array(gt_bboxes_ignore, dtype=np.float32) + else: + gt_bboxes_ignore = np.zeros((0, 4), dtype=np.float32) + + seg_map = img_info['filename'].replace('jpg', 'png') + + ann = dict( + bboxes=gt_bboxes, + labels=gt_labels, + bboxes_ignore=gt_bboxes_ignore, + masks=gt_masks_ann, + seg_map=seg_map) + + return ann + + def xyxy2xywh(self, bbox): + """Convert ``xyxy`` style bounding boxes to ``xywh`` style for COCO + evaluation. + Args: + bbox (numpy.ndarray): The bounding boxes, shape (4, ), in + ``xyxy`` order. + Returns: + list[float]: The converted bounding boxes, in ``xywh`` order. + """ + + _bbox = bbox.tolist() + return [ + _bbox[0], + _bbox[1], + _bbox[2] - _bbox[0], + _bbox[3] - _bbox[1], + ] + + def _proposal2json(self, results): + """Convert proposal results to COCO json style.""" + json_results = [] + for idx in range(len(self)): + img_id = self.img_ids[idx] + bboxes = results[idx] + for i in range(bboxes.shape[0]): + data = dict() + data['image_id'] = img_id + data['bbox'] = self.xyxy2xywh(bboxes[i]) + data['score'] = float(bboxes[i][4]) + data['category_id'] = 1 + json_results.append(data) + return json_results + + def _det2json(self, results): + """Convert detection results to COCO json style.""" + json_results = [] + for idx in range(len(self)): + img_id = self.img_ids[idx] + result = results[idx] + for label in range(len(result)): + bboxes = result[label] + for i in range(bboxes.shape[0]): + data = dict() + data['image_id'] = img_id + data['bbox'] = self.xyxy2xywh(bboxes[i]) + data['score'] = float(bboxes[i][4]) + data['category_id'] = self.cat_ids[label] + json_results.append(data) + return json_results + + def _segm2json(self, results): + """Convert instance segmentation results to COCO json style.""" + bbox_json_results = [] + segm_json_results = [] + for idx in range(len(self)): + img_id = self.img_ids[idx] + det, seg = results[idx] + for label in range(len(det)): + # bbox results + bboxes = det[label] + for i in range(bboxes.shape[0]): + data = dict() + data['image_id'] = img_id + data['bbox'] = self.xyxy2xywh(bboxes[i]) + data['score'] = float(bboxes[i][4]) + data['category_id'] = self.cat_ids[label] + bbox_json_results.append(data) + + # segm results + # some detectors use different scores for bbox and mask + if isinstance(seg, tuple): + segms = seg[0][label] + mask_score = seg[1][label] + else: + segms = seg[label] + mask_score = [bbox[4] for bbox in bboxes] + for i in range(bboxes.shape[0]): + data = dict() + data['image_id'] = img_id + data['bbox'] = self.xyxy2xywh(bboxes[i]) + data['score'] = float(mask_score[i]) + data['category_id'] = self.cat_ids[label] + if isinstance(segms[i]['counts'], bytes): + segms[i]['counts'] = segms[i]['counts'].decode() + data['segmentation'] = segms[i] + segm_json_results.append(data) + return bbox_json_results, segm_json_results + + def results2json(self, results, outfile_prefix): + """Dump the detection results to a COCO style json file. + There are 3 types of results: proposals, bbox predictions, mask + predictions, and they have different data types. This method will + automatically recognize the type, and dump them to json files. + Args: + results (list[list | tuple | ndarray]): Testing results of the + dataset. + outfile_prefix (str): The filename prefix of the json files. If the + prefix is "somepath/xxx", the json files will be named + "somepath/xxx.bbox.json", "somepath/xxx.segm.json", + "somepath/xxx.proposal.json". + Returns: + dict[str: str]: Possible keys are "bbox", "segm", "proposal", and \ + values are corresponding filenames. + """ + result_files = dict() + if isinstance(results[0], list): + json_results = self._det2json(results) + result_files['bbox'] = f'{outfile_prefix}.bbox.json' + result_files['proposal'] = f'{outfile_prefix}.bbox.json' + mmcv.dump(json_results, result_files['bbox']) + elif isinstance(results[0], tuple): + json_results = self._segm2json(results) + result_files['bbox'] = f'{outfile_prefix}.bbox.json' + result_files['proposal'] = f'{outfile_prefix}.bbox.json' + result_files['segm'] = f'{outfile_prefix}.segm.json' + mmcv.dump(json_results[0], result_files['bbox']) + mmcv.dump(json_results[1], result_files['segm']) + elif isinstance(results[0], np.ndarray): + json_results = self._proposal2json(results) + result_files['proposal'] = f'{outfile_prefix}.proposal.json' + mmcv.dump(json_results, result_files['proposal']) + else: + raise TypeError('invalid type of results') + return result_files + + def fast_eval_recall(self, results, proposal_nums, iou_thrs, logger=None): + gt_bboxes = [] + for i in range(len(self.img_ids)): + ann_ids = self.coco.get_ann_ids(img_ids=self.img_ids[i]) + ann_info = self.coco.load_anns(ann_ids) + if len(ann_info) == 0: + gt_bboxes.append(np.zeros((0, 4))) + continue + bboxes = [] + for ann in ann_info: + if ann.get('ignore', False) or ann['iscrowd']: + continue + x1, y1, w, h = ann['bbox'] + bboxes.append([x1, y1, x1 + w, y1 + h]) + bboxes = np.array(bboxes, dtype=np.float32) + if bboxes.shape[0] == 0: + bboxes = np.zeros((0, 4)) + gt_bboxes.append(bboxes) + + recalls = eval_recalls( + gt_bboxes, results, proposal_nums, iou_thrs, logger=logger) + ar = recalls.mean(axis=1) + return ar + + def format_results(self, results, jsonfile_prefix=None, **kwargs): + """Format the results to json (standard format for COCO evaluation). + Args: + results (list[tuple | numpy.ndarray]): Testing results of the + dataset. + jsonfile_prefix (str | None): The prefix of json files. It includes + the file path and the prefix of filename, e.g., "a/b/prefix". + If not specified, a temp file will be created. Default: None. + Returns: + tuple: (result_files, tmp_dir), result_files is a dict containing \ + the json filepaths, tmp_dir is the temporal directory created \ + for saving json files when jsonfile_prefix is not specified. + """ + assert isinstance(results, list), 'results must be a list' + assert len(results) == len(self), ( + 'The length of results is not equal to the dataset len: {} != {}'. + format(len(results), len(self))) + + if jsonfile_prefix is None: + tmp_dir = tempfile.TemporaryDirectory() + jsonfile_prefix = osp.join(tmp_dir.name, 'results') + else: + tmp_dir = None + result_files = self.results2json(results, jsonfile_prefix) + return result_files, tmp_dir + + def evaluate(self, + results, + metric='bbox', + logger=None, + jsonfile_prefix=None, + classwise=False, + proposal_nums=(100, 300, 1000), + iou_thrs=None, + metric_items=None): + """Evaluation in COCO protocol. + Args: + results (list[list | tuple]): Testing results of the dataset. + metric (str | list[str]): Metrics to be evaluated. Options are + 'bbox', 'segm', 'proposal', 'proposal_fast'. + logger (logging.Logger | str | None): Logger used for printing + related information during evaluation. Default: None. + jsonfile_prefix (str | None): The prefix of json files. It includes + the file path and the prefix of filename, e.g., "a/b/prefix". + If not specified, a temp file will be created. Default: None. + classwise (bool): Whether to evaluating the AP for each class. + proposal_nums (Sequence[int]): Proposal number used for evaluating + recalls, such as recall@100, recall@1000. + Default: (100, 300, 1000). + iou_thrs (Sequence[float], optional): IoU threshold used for + evaluating recalls/mAPs. If set to a list, the average of all + IoUs will also be computed. If not specified, [0.50, 0.55, + 0.60, 0.65, 0.70, 0.75, 0.80, 0.85, 0.90, 0.95] will be used. + Default: None. + metric_items (list[str] | str, optional): Metric items that will + be returned. If not specified, ``['AR@100', 'AR@300', + 'AR@1000', 'AR_s@1000', 'AR_m@1000', 'AR_l@1000' ]`` will be + used when ``metric=='proposal'``, ``['mAP', 'mAP_50', 'mAP_75', + 'mAP_s', 'mAP_m', 'mAP_l']`` will be used when + ``metric=='bbox' or metric=='segm'``. + Returns: + dict[str, float]: COCO style evaluation metric. + """ + + metrics = metric if isinstance(metric, list) else [metric] + allowed_metrics = ['bbox', 'segm', 'proposal', 'proposal_fast'] + for metric in metrics: + if metric not in allowed_metrics: + raise KeyError(f'metric {metric} is not supported') + if iou_thrs is None: + iou_thrs = np.linspace( + .5, 0.95, int(np.round((0.95 - .5) / .05)) + 1, endpoint=True) + if metric_items is not None: + if not isinstance(metric_items, list): + metric_items = [metric_items] + + result_files, tmp_dir = self.format_results(results, jsonfile_prefix) + + eval_results = OrderedDict() + cocoGt = self.coco + for metric in metrics: + msg = f'Evaluating {metric}...' + if logger is None: + msg = '\n' + msg + print_log(msg, logger=logger) + + if metric == 'proposal_fast': + ar = self.fast_eval_recall( + results, proposal_nums, iou_thrs, logger='silent') + log_msg = [] + for i, num in enumerate(proposal_nums): + eval_results[f'AR@{num}'] = ar[i] + log_msg.append(f'\nAR@{num}\t{ar[i]:.4f}') + log_msg = ''.join(log_msg) + print_log(log_msg, logger=logger) + continue + + iou_type = 'bbox' if metric == 'proposal' else metric + if metric not in result_files: + raise KeyError(f'{metric} is not in results') + try: + predictions = mmcv.load(result_files[metric]) + if iou_type == 'segm': + # Refer to https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocotools/coco.py#L331 # noqa + # When evaluating mask AP, if the results contain bbox, + # cocoapi will use the box area instead of the mask area + # for calculating the instance area. Though the overall AP + # is not affected, this leads to different + # small/medium/large mask AP results. + for x in predictions: + x.pop('bbox') + warnings.simplefilter('once') + warnings.warn( + 'The key "bbox" is deleted for more accurate mask AP ' + 'of small/medium/large instances since v2.12.0. This ' + 'does not change the overall mAP calculation.', + UserWarning) + cocoDt = cocoGt.loadRes(predictions) + except IndexError: + print_log( + 'The testing results of the whole dataset is empty.', + logger=logger, + level=logging.ERROR) + break + + cocoEval = COCOeval(cocoGt, cocoDt, iou_type) + cocoEval.params.catIds = self.cat_ids + cocoEval.params.imgIds = self.img_ids + cocoEval.params.maxDets = list(proposal_nums) + cocoEval.params.iouThrs = iou_thrs + # mapping of cocoEval.stats + coco_metric_names = { + 'mAP': 0, + 'mAP_50': 1, + 'mAP_75': 2, + 'mAP_s': 3, + 'mAP_m': 4, + 'mAP_l': 5, + 'AR@100': 6, + 'AR@300': 7, + 'AR@1000': 8, + 'AR_s@1000': 9, + 'AR_m@1000': 10, + 'AR_l@1000': 11 + } + if metric_items is not None: + for metric_item in metric_items: + if metric_item not in coco_metric_names: + raise KeyError( + f'metric item {metric_item} is not supported') + + if metric == 'proposal': + cocoEval.params.useCats = 0 + cocoEval.evaluate() + cocoEval.accumulate() + cocoEval.summarize() + if metric_items is None: + metric_items = [ + 'AR@100', 'AR@300', 'AR@1000', 'AR_s@1000', + 'AR_m@1000', 'AR_l@1000' + ] + + for item in metric_items: + val = float( + f'{cocoEval.stats[coco_metric_names[item]]:.3f}') + eval_results[item] = val + else: + cocoEval.evaluate() + cocoEval.accumulate() + cocoEval.summarize() + if classwise: # Compute per-category AP + # Compute per-category AP + # from https://github.com/facebookresearch/detectron2/ + precisions = cocoEval.eval['precision'] + # precision: (iou, recall, cls, area range, max dets) + assert len(self.cat_ids) == precisions.shape[2] + + results_per_category = [] + for idx, catId in enumerate(self.cat_ids): + # area range index 0: all area ranges + # max dets index -1: typically 100 per image + nm = self.coco.loadCats(catId)[0] + precision = precisions[:, :, idx, 0, -1] + precision = precision[precision > -1] + if precision.size: + ap = np.mean(precision) + else: + ap = float('nan') + results_per_category.append( + (f'{nm["name"]}', f'{float(ap):0.3f}')) + + num_columns = min(6, len(results_per_category) * 2) + results_flatten = list( + itertools.chain(*results_per_category)) + headers = ['category', 'AP'] * (num_columns // 2) + results_2d = itertools.zip_longest(*[ + results_flatten[i::num_columns] + for i in range(num_columns) + ]) + table_data = [headers] + table_data += [result for result in results_2d] + table = AsciiTable(table_data) + print_log('\n' + table.table, logger=logger) + + if metric_items is None: + metric_items = [ + 'mAP', 'mAP_50', 'mAP_75', 'mAP_s', 'mAP_m', 'mAP_l' + ] + + for metric_item in metric_items: + key = f'{metric}_{metric_item}' + val = float( + f'{cocoEval.stats[coco_metric_names[metric_item]]:.3f}' + ) + eval_results[key] = val + ap = cocoEval.stats[:6] + eval_results[f'{metric}_mAP_copypaste'] = ( + f'{ap[0]:.3f} {ap[1]:.3f} {ap[2]:.3f} {ap[3]:.3f} ' + f'{ap[4]:.3f} {ap[5]:.3f}') + if tmp_dir is not None: + tmp_dir.cleanup() + return \ No newline at end of file diff --git a/detection/mmdet_custom/models/__init__.py b/detection/mmdet_custom/models/__init__.py new file mode 100644 index 0000000..1414a32 --- /dev/null +++ b/detection/mmdet_custom/models/__init__.py @@ -0,0 +1,11 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + +from .backbones import * # noqa: F401,F403 +from .dense_heads import * # noqa: F401,F403 +from .detectors import * # noqa: F401,F403 +from .utils import * # noqa: F401,F403 +from .necks.fpn import * \ No newline at end of file diff --git a/detection/mmdet_custom/models/backbones/__init__.py b/detection/mmdet_custom/models/backbones/__init__.py new file mode 100644 index 0000000..e744d45 --- /dev/null +++ b/detection/mmdet_custom/models/backbones/__init__.py @@ -0,0 +1,8 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2023 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- +from .flash_intern_image import FlashInternImage + +__all__ = ['FlashInternImage'] diff --git a/detection/mmdet_custom/models/backbones/flash_intern_image.py b/detection/mmdet_custom/models/backbones/flash_intern_image.py new file mode 100644 index 0000000..6fd5ef7 --- /dev/null +++ b/detection/mmdet_custom/models/backbones/flash_intern_image.py @@ -0,0 +1,763 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + +import torch +import torch.nn as nn +from collections import OrderedDict +import torch.utils.checkpoint as checkpoint +from timm.models.layers import trunc_normal_, DropPath +from mmcv.runner import _load_checkpoint +from mmcv.cnn import constant_init, trunc_normal_init +from mmdet.utils import get_root_logger +from mmdet.models.builder import BACKBONES +import torch.nn.functional as F +import DCNv4 + + +class to_channels_first(nn.Module): + + def __init__(self): + super().__init__() + + def forward(self, x): + return x.permute(0, 3, 1, 2) + + +class to_channels_last(nn.Module): + + def __init__(self): + super().__init__() + + def forward(self, x): + return x.permute(0, 2, 3, 1) + + +def build_norm_layer(dim, + norm_layer, + in_format='channels_last', + out_format='channels_last', + eps=1e-6): + layers = [] + if norm_layer == 'BN': + if in_format == 'channels_last': + layers.append(to_channels_first()) + layers.append(nn.BatchNorm2d(dim)) + if out_format == 'channels_last': + layers.append(to_channels_last()) + elif norm_layer == 'LN': + if in_format == 'channels_first': + layers.append(to_channels_last()) + layers.append(nn.LayerNorm(dim, eps=eps)) + if out_format == 'channels_first': + layers.append(to_channels_first()) + else: + raise NotImplementedError( + f'build_norm_layer does not support {norm_layer}') + return nn.Sequential(*layers) + + +def build_act_layer(act_layer): + if act_layer == 'ReLU': + return nn.ReLU(inplace=True) + elif act_layer == 'SiLU': + return nn.SiLU(inplace=True) + elif act_layer == 'GELU': + return nn.GELU() + + raise NotImplementedError(f'build_act_layer does not support {act_layer}') + + +class CrossAttention(nn.Module): + r""" Cross Attention Module + Args: + dim (int): Number of input channels. + num_heads (int): Number of attention heads. Default: 8 + qkv_bias (bool, optional): If True, add a learnable bias to q, k, v. + Default: False. + qk_scale (float | None, optional): Override default qk scale of + head_dim ** -0.5 if set. Default: None. + attn_drop (float, optional): Dropout ratio of attention weight. + Default: 0.0 + proj_drop (float, optional): Dropout ratio of output. Default: 0.0 + attn_head_dim (int, optional): Dimension of attention head. + out_dim (int, optional): Dimension of output. + """ + + def __init__(self, + dim, + num_heads=8, + qkv_bias=False, + qk_scale=None, + attn_drop=0., + proj_drop=0., + attn_head_dim=None, + out_dim=None): + super().__init__() + if out_dim is None: + out_dim = dim + self.num_heads = num_heads + head_dim = dim // num_heads + if attn_head_dim is not None: + head_dim = attn_head_dim + all_head_dim = head_dim * self.num_heads + self.scale = qk_scale or head_dim ** -0.5 + assert all_head_dim == dim + + self.q = nn.Linear(dim, all_head_dim, bias=False) + self.k = nn.Linear(dim, all_head_dim, bias=False) + self.v = nn.Linear(dim, all_head_dim, bias=False) + + if qkv_bias: + self.q_bias = nn.Parameter(torch.zeros(all_head_dim)) + self.k_bias = nn.Parameter(torch.zeros(all_head_dim)) + self.v_bias = nn.Parameter(torch.zeros(all_head_dim)) + else: + self.q_bias = None + self.k_bias = None + self.v_bias = None + + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(all_head_dim, out_dim) + self.proj_drop = nn.Dropout(proj_drop) + + def forward(self, x, k=None, v=None): + B, N, C = x.shape + N_k = k.shape[1] + N_v = v.shape[1] + + q_bias, k_bias, v_bias = None, None, None + if self.q_bias is not None: + q_bias = self.q_bias + k_bias = self.k_bias + v_bias = self.v_bias + + q = F.linear(input=x, weight=self.q.weight, bias=q_bias) + q = q.reshape(B, N, 1, self.num_heads, + -1).permute(2, 0, 3, 1, + 4).squeeze(0) # (B, N_head, N_q, dim) + + k = F.linear(input=k, weight=self.k.weight, bias=k_bias) + k = k.reshape(B, N_k, 1, self.num_heads, -1).permute(2, 0, 3, 1, + 4).squeeze(0) + + v = F.linear(input=v, weight=self.v.weight, bias=v_bias) + v = v.reshape(B, N_v, 1, self.num_heads, -1).permute(2, 0, 3, 1, + 4).squeeze(0) + + q = q * self.scale + attn = (q @ k.transpose(-2, -1)) # (B, N_head, N_q, N_k) + + attn = attn.softmax(dim=-1) + attn = self.attn_drop(attn) + + x = (attn @ v).transpose(1, 2).reshape(B, N, -1) + x = self.proj(x) + x = self.proj_drop(x) + + return x + + +class AttentiveBlock(nn.Module): + r"""Attentive Block + Args: + dim (int): Number of input channels. + num_heads (int): Number of attention heads. Default: 8 + qkv_bias (bool, optional): If True, add a learnable bias to q, k, v. + Default: False. + qk_scale (float | None, optional): Override default qk scale of + head_dim ** -0.5 if set. Default: None. + drop (float, optional): Dropout rate. Default: 0.0. + attn_drop (float, optional): Attention dropout rate. Default: 0.0. + drop_path (float | tuple[float], optional): Stochastic depth rate. + Default: 0.0. + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm. + attn_head_dim (int, optional): Dimension of attention head. Default: None. + out_dim (int, optional): Dimension of output. Default: None. + """ + + def __init__(self, + dim, + num_heads, + qkv_bias=False, + qk_scale=None, + drop=0., + attn_drop=0., + drop_path=0., + norm_layer="LN", + attn_head_dim=None, + out_dim=None): + super().__init__() + + self.norm1_q = build_norm_layer(dim, norm_layer, eps=1e-6) + self.norm1_k = build_norm_layer(dim, norm_layer, eps=1e-6) + self.norm1_v = build_norm_layer(dim, norm_layer, eps=1e-6) + self.cross_dcn = CrossAttention(dim, + num_heads=num_heads, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + attn_drop=attn_drop, + proj_drop=drop, + attn_head_dim=attn_head_dim, + out_dim=out_dim) + + self.drop_path = DropPath( + drop_path) if drop_path > 0. else nn.Identity() + + def forward(self, + x_q, + x_kv, + pos_q, + pos_k, + bool_masked_pos, + rel_pos_bias=None): + x_q = self.norm1_q(x_q + pos_q) + x_k = self.norm1_k(x_kv + pos_k) + x_v = self.norm1_v(x_kv) + + x = self.cross_dcn(x_q, k=x_k, v=x_v) + + return x + + +class AttentionPoolingBlock(AttentiveBlock): + + def forward(self, x): + x_q = x.mean(1, keepdim=True) + x_kv = x + pos_q, pos_k = 0, 0 + x = super().forward(x_q, x_kv, pos_q, pos_k, + bool_masked_pos=None, + rel_pos_bias=None) + x = x.squeeze(1) + return x + + +class StemLayer(nn.Module): + r""" Stem layer of InternImage + Args: + in_chans (int): number of input channels + out_chans (int): number of output channels + act_layer (str): activation layer + norm_layer (str): normalization layer + """ + + def __init__(self, + in_chans=3, + out_chans=96, + act_layer='GELU', + norm_layer='BN'): + super().__init__() + self.conv1 = nn.Conv2d(in_chans, + out_chans // 2, + kernel_size=3, + stride=2, + padding=1) + self.norm1 = build_norm_layer(out_chans // 2, norm_layer, + 'channels_first', 'channels_first') + self.act = build_act_layer(act_layer) + self.conv2 = nn.Conv2d(out_chans // 2, + out_chans, + kernel_size=3, + stride=2, + padding=1) + self.norm2 = build_norm_layer(out_chans, norm_layer, 'channels_first', + 'channels_last') + + def forward(self, x): + x = self.conv1(x) + x = self.norm1(x) + x = self.act(x) + x = self.conv2(x) + x = self.norm2(x) + return x + + + +class DownsampleLayer(nn.Module): + r""" Downsample layer of InternImage + Args: + channels (int): number of input channels + norm_layer (str): normalization layer + """ + + def __init__(self, channels, norm_layer='LN'): + super().__init__() + self.conv = nn.Conv2d(channels, + 2 * channels, + kernel_size=3, + stride=2, + padding=1, + bias=False) + self.norm = build_norm_layer(2 * channels, norm_layer, + 'channels_first', 'channels_first') + + + def forward(self, x, shape=None): + H, W = shape + N, HW, C = x.shape + + x = x.view(N, H, W, C) + x = self.conv(x.permute(0, 3, 1, 2)) + x = self.norm(x) # B C H W + H, W = x.size(2), x.size(3) + x = x.flatten(2).permute(0, 2, 1) + + return x, (H, W) + + + +class MLPLayer(nn.Module): + r""" MLP layer of InternImage + Args: + in_features (int): number of input features + hidden_features (int): number of hidden features + out_features (int): number of output features + act_layer (str): activation layer + drop (float): dropout rate + """ + + def __init__(self, + in_features, + hidden_features=None, + out_features=None, + act_layer='GELU', + mlp_fc2_bias=False, + drop=0.): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = nn.Linear(in_features, hidden_features, bias=True) + self.act = build_act_layer(act_layer) + self.fc2 = nn.Linear(hidden_features, out_features, bias=mlp_fc2_bias) + self.drop = nn.Dropout(drop) + + + def forward(self, x, shape, level_idx=0): + x = self.fc1(x) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + return x + + +class InternImageLayer(nn.Module): + r""" Basic layer of InternImage + Args: + core_op (nn.Module): core operation of InternImage + channels (int): number of input channels + groups (list): Groups of each block. + mlp_ratio (float): ratio of mlp hidden features to input channels + drop (float): dropout rate + drop_path (float): drop path rate + act_layer (str): activation layer + norm_layer (str): normalization layer + post_norm (bool): whether to use post normalization + layer_scale (float): layer scale + offset_scale (float): offset scale + with_cp (bool): whether to use checkpoint + """ + + def __init__(self, + core_op, + channels, + groups, + mlp_ratio=4., + drop=0., + drop_path=0., + act_layer='GELU', + norm_layer='LN', + post_norm=False, + layer_scale=None, + offset_scale=1.0, + with_cp=False, + dcn_output_bias=False, + mlp_fc2_bias=False, + dw_kernel_size=None, # for InternImage-H/G + res_post_norm=False, # for InternImage-H/G + center_feature_scale=False): # for InternImage-H/G + super().__init__() + self.channels = channels + self.groups = groups + self.mlp_ratio = mlp_ratio + self.with_cp = with_cp + + self.norm1 = build_norm_layer(channels, 'LN') + self.post_norm = post_norm + self.dcn = core_op( + channels=channels, + group=groups, + offset_scale=offset_scale, + dw_kernel_size=dw_kernel_size, + output_bias=dcn_output_bias, + ) + self.drop_path = DropPath(drop_path) if drop_path > 0. \ + else nn.Identity() + self.norm2 = build_norm_layer(channels, 'LN') + self.mlp = MLPLayer(in_features=channels, + hidden_features=int(channels * mlp_ratio), + act_layer=act_layer, + drop=drop, + mlp_fc2_bias=mlp_fc2_bias + ) + self.layer_scale = layer_scale is not None + if self.layer_scale: + self.gamma1 = nn.Parameter(layer_scale * torch.ones(channels), + requires_grad=True) + self.gamma2 = nn.Parameter(layer_scale * torch.ones(channels), + requires_grad=True) + self.res_post_norm = res_post_norm + if res_post_norm: + self.res_post_norm1 = build_norm_layer(channels, 'LN') + self.res_post_norm2 = build_norm_layer(channels, 'LN') + def forward(self, x, shape, level_idx=0): + + def _inner_forward(x, shape, level_idx): + if not self.layer_scale: + if self.post_norm: + x = x + self.drop_path(self.norm1(self.dcn(x, shape, level_idx))) + x = x + self.drop_path(self.norm2(self.mlp(x, shape, level_idx))) + elif self.res_post_norm: # for InternImage-H/G + x = x + self.drop_path(self.res_post_norm1(self.dcn(self.norm1(x), shape, level_idx))) + x = x + self.drop_path(self.res_post_norm2(self.mlp(self.norm2(x), shape, level_idx))) + + else: + x = x + self.drop_path(self.dcn(self.norm1(x), shape, level_idx)) + x = x + self.drop_path(self.mlp(self.norm2(x), shape, level_idx)) + return x + if self.post_norm: + x = x + self.drop_path(self.gamma1 * self.norm1(self.dcn(x, shape))) + x = x + self.drop_path(self.gamma2 * self.norm2(self.mlp(x, shape, level_idx))) + else: + x = x + self.drop_path(self.gamma1 * self.dcn(self.norm1(x), shape)) + x = x + self.drop_path(self.gamma2 * self.mlp(self.norm2(x), shape, level_idx)) + return x + + if self.with_cp and x.requires_grad: + x = checkpoint.checkpoint(_inner_forward, x, shape, level_idx) + else: + x = _inner_forward(x, shape, level_idx) + + return x + + +class InternImageBlock(nn.Module): + r""" Block of InternImage + Args: + core_op (nn.Module): core operation of InternImage + channels (int): number of input channels + depths (list): Depth of each block. + groups (list): Groups of each block. + mlp_ratio (float): ratio of mlp hidden features to input channels + drop (float): dropout rate + drop_path (float): drop path rate + act_layer (str): activation layer + norm_layer (str): normalization layer + post_norm (bool): whether to use post normalization + layer_scale (float): layer scale + offset_scale (float): offset scale + with_cp (bool): whether to use checkpoint + """ + + def __init__(self, + core_op, + channels, + depth, + groups, + downsample=True, + downsample_layer=DownsampleLayer, + mlp_ratio=4., + drop=0., + drop_path=0., + act_layer='GELU', + norm_layer='LN', + post_norm=False, + offset_scale=0.5, + layer_scale=None, + with_cp=False, + dcn_output_bias=False, + mlp_fc2_bias=False, + dw_kernel_size=None, # for InternImage-H/G + post_norm_block_ids=None, # for InternImage-H/G + res_post_norm=False, # for InternImage-H/G + center_feature_scale=False): # for InternImage-H/G + super().__init__() + self.channels = channels + self.depth = depth + self.post_norm = post_norm + self.center_feature_scale = center_feature_scale + + self.blocks = nn.ModuleList([ + InternImageLayer( + core_op=core_op, + channels=channels, + groups=groups, + mlp_ratio=mlp_ratio, + drop=drop, + drop_path=drop_path[i] if isinstance( + drop_path, list) else drop_path, + act_layer=act_layer, + norm_layer=norm_layer, + post_norm=post_norm, + layer_scale=layer_scale, + offset_scale=offset_scale, + with_cp=with_cp, + dcn_output_bias=dcn_output_bias, + mlp_fc2_bias=mlp_fc2_bias, + dw_kernel_size=dw_kernel_size, # for InternImage-H/G + res_post_norm=res_post_norm, # for InternImage-H/G + center_feature_scale=center_feature_scale # for InternImage-H/G + ) for i in range(depth) + ]) + if not self.post_norm or center_feature_scale: + self.norm = build_norm_layer(channels, 'LN') + self.post_norm_block_ids = post_norm_block_ids + if post_norm_block_ids is not None: # for InternImage-H/G + self.post_norms = nn.ModuleList( + [build_norm_layer(channels, 'LN', eps=1e-6) for _ in post_norm_block_ids] + ) + self.downsample = downsample_layer( + channels=channels, norm_layer=norm_layer) if downsample else None + + + def forward(self, x, return_wo_downsample=False, shape=None, level_idx=0 + ): + for i, blk in enumerate(self.blocks): + x = blk(x, shape=shape, level_idx=level_idx) + if (self.post_norm_block_ids is not None) and (i in self.post_norm_block_ids): + index = self.post_norm_block_ids.index(i) + x = self.post_norms[index](x) # for InternImage-H/G + if not self.post_norm or self.center_feature_scale: + x = self.norm(x) + if return_wo_downsample: + x_ = x.clone() + if self.downsample is not None: + x, shape = self.downsample(x, shape=shape) + + if return_wo_downsample: + return x, x_, shape + return x, shape + +@BACKBONES.register_module() +class FlashInternImage(nn.Module): + r""" FlashInternImage + A PyTorch impl based on : + `InternImage: Exploring Large-Scale Vision Foundation Models with Deformable Convolutions` - + https://arxiv.org/pdf/2103.14030 + 'DCNv4': TODO: add arxiv + Args: + core_op (str): Core operator. Default: 'DCNv4' + channels (int): Number of the first stage. Default: 64 + depths (list): Depth of each block. Default: [3, 4, 18, 5] + groups (list): Groups of each block. Default: [3, 6, 12, 24] + num_classes (int): Number of classes. Default: 1000 + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. Default: 4. + drop_rate (float): Probability of an element to be zeroed. Default: 0. + drop_path_rate (float): Stochastic depth rate. Default: 0.2 + act_layer (str): Activation layer. Default: 'GELU' + norm_layer (str): Normalization layer. Default: 'LN' + layer_scale (bool): Whether to use layer scale. Default: False + cls_scale (bool): Whether to use class scale. Default: False + with_cp (bool): Use checkpoint or not. Using checkpoint will save some + dw_kernel_size (int): Size of the dwconv. Default: None + use_clip_projector (bool): Whether to use clip projector. Default: False + level2_post_norm (bool): Whether to use level2 post norm. Default: False + level2_post_norm_block_ids (list): Indexes of post norm blocks. Default: None + res_post_norm (bool): Whether to use res post norm. Default: False + center_feature_scale (bool): Whether to use center feature scale. Default: False + """ + + def __init__(self, + core_op='DCNv4', + channels=64, + depths=[3, 4, 18, 5], + groups=[3, 6, 12, 24], + num_classes=1000, + mlp_ratio=4., + drop_rate=0., + drop_path_rate=0.2, + drop_path_type='linear', + act_layer='GELU', + norm_layer='LN', + layer_scale=None, + offset_scale=0.5, + post_norm=False, + with_cp=False, + mlp_fc2_bias=False, + dcn_output_bias=False, + dw_kernel_size=None, # for InternImage-H/G + level2_post_norm=False, # for InternImage-H/G + level2_post_norm_block_ids=None, # for InternImage-H/G + res_post_norm=False, # for InternImage-H/G + center_feature_scale=False, # for InternImage-H/G + out_indices=(0, 1, 2, 3), + init_cfg=None, + **kwargs): + super().__init__() + self.core_op = core_op + self.num_levels = len(depths) + self.depths = depths + self.channels = channels + self.num_features = int(channels * 2**(self.num_levels - 1)) + self.post_norm = post_norm + self.mlp_ratio = mlp_ratio + self.init_cfg = init_cfg + self.out_indices = out_indices + self.level2_post_norm_block_ids = level2_post_norm_block_ids + logger = get_root_logger() + logger.info(f'using core type: {core_op}') + logger.info(f'using activation layer: {act_layer}') + logger.info(f'using main norm layer: {norm_layer}') + logger.info(f'using dpr: {drop_path_type}, {drop_path_rate}') + logger.info(f"level2_post_norm: {level2_post_norm}") + logger.info(f"level2_post_norm_block_ids: {level2_post_norm_block_ids}") + logger.info(f"res_post_norm: {res_post_norm}") + + in_chans = 3 + self.patch_embed = StemLayer(in_chans=in_chans, + out_chans=channels, + act_layer=act_layer, + norm_layer=norm_layer) + self.pos_drop = nn.Dropout(p=drop_rate) + + dpr = [ + x.item() for x in torch.linspace(0, drop_path_rate, sum(depths)) + ] + if drop_path_type == 'uniform': + for i in range(len(dpr)): + dpr[i] = drop_path_rate + + self.levels = nn.ModuleList() + for i in range(self.num_levels): + post_norm_block_ids = level2_post_norm_block_ids if level2_post_norm and ( + i == 2) else None # for InternImage-H/G + + level = InternImageBlock( + core_op=getattr(DCNv4, core_op), + channels=int(channels * 2**i), + depth=depths[i], + groups=groups[i], + mlp_ratio=self.mlp_ratio, + drop=drop_rate, + drop_path=dpr[sum(depths[:i]):sum(depths[:i + 1])], + act_layer=act_layer, + norm_layer=norm_layer, + post_norm=post_norm, + downsample=(i < self.num_levels - 1), + downsample_layer = DownsampleLayer, + layer_scale=layer_scale, + offset_scale=offset_scale, + with_cp=with_cp, + mlp_fc2_bias=mlp_fc2_bias, + dcn_output_bias=dcn_output_bias, + dw_kernel_size=dw_kernel_size, # for InternImage-H/G + post_norm_block_ids=post_norm_block_ids, # for InternImage-H/G + res_post_norm=res_post_norm, # for InternImage-H/G + center_feature_scale=center_feature_scale # for InternImage-H/G + ) + self.levels.append(level) + + self.num_layers = len(depths) + self.apply(self._init_weights) + self.apply(self._init_deform_weights) + + def init_weights(self): + logger = get_root_logger() + if self.init_cfg is None: + logger.warn(f'No pre-trained weights for ' + f'{self.__class__.__name__}, ' + f'training start from scratch') + for m in self.modules(): + if isinstance(m, nn.Linear): + trunc_normal_init(m, std=.02, bias=0.) + elif isinstance(m, nn.LayerNorm): + constant_init(m, 1.0) + else: + assert 'checkpoint' in self.init_cfg, f'Only support ' \ + f'specify `Pretrained` in ' \ + f'`init_cfg` in ' \ + f'{self.__class__.__name__} ' + ckpt = _load_checkpoint(self.init_cfg.checkpoint, + logger=logger, + map_location='cpu') + if 'state_dict' in ckpt: + _state_dict = ckpt['state_dict'] + elif 'model_ema' in ckpt: + _state_dict = ckpt['model_ema'] + elif 'model' in ckpt: + _state_dict = ckpt['model'] + else: + _state_dict = ckpt + + state_dict = OrderedDict() + for k, v in _state_dict.items(): + if k.startswith('backbone.'): + state_dict[k[9:]] = v + else: + state_dict[k] = v + + # strip prefix of state_dict + if list(state_dict.keys())[0].startswith('module.'): + state_dict = {k[7:]: v for k, v in state_dict.items()} + + # load state_dict + meg = self.load_state_dict(state_dict, False) + logger.info(meg) + + def _init_weights(self, m): + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + + def _init_deform_weights(self, m): + if isinstance(m, getattr(DCNv4, self.core_op)): + m._reset_parameters() + + @torch.jit.ignore + def lr_decay_keywards(self, decay_ratio=0.87): + lr_ratios = {} + + # blocks + idx = 0 + for i in range(4): + layer_num = 3 - i # 3 2 1 0 + for j in range(self.depths[layer_num]): + block_num = self.depths[layer_num] - j - 1 + tag = 'levels.{}.blocks.{}.'.format(layer_num, block_num) + decay = 1.0 * (decay_ratio**idx) + lr_ratios[tag] = decay + idx += 1 + # patch_embed (before stage-1) + lr_ratios["patch_embed"] = lr_ratios['levels.0.blocks.0.'] + # levels.0.downsample (between stage-1 and stage-2) + lr_ratios["levels.0.downsample"] = lr_ratios['levels.1.blocks.0.'] + lr_ratios["levels.0.norm"] = lr_ratios['levels.1.blocks.0.'] + # levels.1.downsample (between stage-2 and stage-3) + lr_ratios["levels.1.downsample"] = lr_ratios['levels.2.blocks.0.'] + lr_ratios["levels.1.norm"] = lr_ratios['levels.2.blocks.0.'] + # levels.2.downsample (between stage-3 and stage-4) + lr_ratios["levels.2.downsample"] = lr_ratios['levels.3.blocks.0.'] + lr_ratios["levels.2.norm"] = lr_ratios['levels.3.blocks.0.'] + return lr_ratios + + def forward(self, x): + x = self.patch_embed(x) + N, H, W, C = x.shape + x = x.view(N, H*W, C) + + shape=(H, W) + seq_out = [] + for level_idx, level in enumerate(self.levels): + old_shape = shape + x, x_ , shape = level(x, return_wo_downsample=True, shape=shape, level_idx=level_idx) + if level_idx in self.out_indices: + h, w= old_shape + seq_out.append(x_.reshape(N, h, w, -1).permute(0, 3, 1, 2)) + return seq_out diff --git a/detection/mmdet_custom/models/dense_heads/__init__.py b/detection/mmdet_custom/models/dense_heads/__init__.py new file mode 100644 index 0000000..5266cd7 --- /dev/null +++ b/detection/mmdet_custom/models/dense_heads/__init__.py @@ -0,0 +1,13 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + +from .deformable_detr_head import DeformableDETRHead +from .detr_head import DETRHead +from .dino_head import DINOHead +from .msda import FlashMultiScaleDeformableAttention +from .bbox_head import DCNv4FCBBoxHead +from .mask_rcnn import MaskRCNN_ +__all__ = ['DeformableDETRHead', 'DETRHead', 'DINOHead'] \ No newline at end of file diff --git a/detection/mmdet_custom/models/dense_heads/bbox_head.py b/detection/mmdet_custom/models/dense_heads/bbox_head.py new file mode 100644 index 0000000..8236d69 --- /dev/null +++ b/detection/mmdet_custom/models/dense_heads/bbox_head.py @@ -0,0 +1,222 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch.nn as nn +from mmcv.cnn import ConvModule + +from mmdet.models.builder import HEADS +from mmdet.models.utils import build_linear_layer +from mmdet.models.roi_heads.bbox_heads.bbox_head import BBoxHead +import DCNv4 + + + + + + +@HEADS.register_module() +class DCNv4FCBBoxHead(BBoxHead): + r"""More general bbox head, with shared conv and fc layers and two optional + separated branches. + + .. code-block:: none + + /-> cls convs -> cls fcs -> cls + shared convs -> shared fcs + \-> reg convs -> reg fcs -> reg + """ # noqa: W605 + + def __init__(self, + num_shared_convs=0, + num_shared_fcs=0, + num_cls_convs=0, + num_cls_fcs=0, + num_reg_convs=0, + num_reg_fcs=0, + conv_out_channels=256, + fc_out_channels=1024, + conv_cfg=None, + norm_cfg=None, + init_cfg=None, + with_dcn=True, + short_cut=False, + *args, + **kwargs): + super(DCNv4FCBBoxHead, self).__init__( + *args, init_cfg=init_cfg, **kwargs) + assert (num_shared_convs + num_shared_fcs + num_cls_convs + + num_cls_fcs + num_reg_convs + num_reg_fcs > 0) + if num_cls_convs > 0 or num_reg_convs > 0: + assert num_shared_fcs == 0 + if not self.with_cls: + assert num_cls_convs == 0 and num_cls_fcs == 0 + if not self.with_reg: + assert num_reg_convs == 0 and num_reg_fcs == 0 + self.num_shared_convs = num_shared_convs + self.num_shared_fcs = num_shared_fcs + self.num_cls_convs = num_cls_convs + self.num_cls_fcs = num_cls_fcs + self.num_reg_convs = num_reg_convs + self.num_reg_fcs = num_reg_fcs + self.conv_out_channels = conv_out_channels + self.fc_out_channels = fc_out_channels + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.with_dcn = with_dcn + self.short_cut = short_cut + + # add shared convs and fcs + self.shared_convs, self.shared_fcs, last_layer_dim = \ + self._add_conv_fc_branch( + self.num_shared_convs, self.num_shared_fcs, self.in_channels, + True) + self.shared_out_channels = last_layer_dim + + # add cls specific branch + self.cls_convs, self.cls_fcs, self.cls_last_dim = \ + self._add_conv_fc_branch( + self.num_cls_convs, self.num_cls_fcs, self.shared_out_channels) + + # add reg specific branch + self.reg_convs, self.reg_fcs, self.reg_last_dim = \ + self._add_conv_fc_branch( + self.num_reg_convs, self.num_reg_fcs, self.shared_out_channels) + + if self.num_shared_fcs == 0 and not self.with_avg_pool: + if self.num_cls_fcs == 0: + self.cls_last_dim *= self.roi_feat_area + if self.num_reg_fcs == 0: + self.reg_last_dim *= self.roi_feat_area + + self.relu = nn.ReLU(inplace=True) + # reconstruct fc_cls and fc_reg since input channels are changed + if self.with_cls: + if self.custom_cls_channels: + cls_channels = self.loss_cls.get_cls_channels(self.num_classes) + else: + cls_channels = self.num_classes + 1 + self.fc_cls = build_linear_layer( + self.cls_predictor_cfg, + in_features=self.cls_last_dim, + out_features=cls_channels) + if self.with_reg: + out_dim_reg = (4 if self.reg_class_agnostic else 4 * + self.num_classes) + self.fc_reg = build_linear_layer( + self.reg_predictor_cfg, + in_features=self.reg_last_dim, + out_features=out_dim_reg) + + if init_cfg is None: + # when init_cfg is None, + # It has been set to + # [[dict(type='Normal', std=0.01, override=dict(name='fc_cls'))], + # [dict(type='Normal', std=0.001, override=dict(name='fc_reg'))] + # after `super(ConvFCBBoxHead, self).__init__()` + # we only need to append additional configuration + # for `shared_fcs`, `cls_fcs` and `reg_fcs` + self.init_cfg += [ + dict( + type='Xavier', + distribution='uniform', + override=[ + dict(name='shared_fcs'), + dict(name='cls_fcs'), + dict(name='reg_fcs') + ]) + ] + + def _add_conv_fc_branch(self, + num_branch_convs, + num_branch_fcs, + in_channels, + is_shared=False): + """Add shared or separable branch. + + convs -> avg pool (optional) -> fcs + """ + last_layer_dim = in_channels + # add branch specific conv layers + branch_convs = nn.ModuleList() + if num_branch_convs > 0: + for i in range(num_branch_convs): + conv_in_channels = ( + last_layer_dim if i == 0 else self.conv_out_channels) + if self.with_dcn: + assert False, 'TODO: support DCNv4 in the task head' + conv = DCNv4.DCNv4( + conv_in_channels, + self.conv_out_channels, + ) + else: + conv = ConvModule( + conv_in_channels, + self.conv_out_channels, + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg) + + branch_convs.append(conv) + last_layer_dim = self.conv_out_channels + # add branch specific fc layers + branch_fcs = nn.ModuleList() + if num_branch_fcs > 0: + # for shared branch, only consider self.with_avg_pool + # for separated branches, also consider self.num_shared_fcs + if (is_shared + or self.num_shared_fcs == 0) and not self.with_avg_pool: + last_layer_dim *= self.roi_feat_area + for i in range(num_branch_fcs): + fc_in_channels = ( + last_layer_dim if i == 0 else self.fc_out_channels) + branch_fcs.append( + nn.Linear(fc_in_channels, self.fc_out_channels)) + last_layer_dim = self.fc_out_channels + return branch_convs, branch_fcs, last_layer_dim + + def forward(self, x): + # shared part + if self.with_dcn: + N, C, H, W = x.shape + x = x.permute(0, 2, 3, 1).view(N, H*W, C) + if self.num_shared_convs > 0: + for conv in self.shared_convs: + if self.short_cut: + x = x + conv(x, shape=(H, W)) + else: + x = conv(x, shape=(H, W)) + else: + if self.num_shared_convs > 0: + for conv in self.shared_convs: + x = conv(x) + if self.num_shared_fcs > 0: + if self.with_avg_pool: + x = self.avg_pool(x) + x = x.flatten(1) + + for fc in self.shared_fcs: + x = self.relu(fc(x)) + # separate branches + x_cls = x + x_reg = x + + for conv in self.cls_convs: + x_cls = conv(x_cls) + if x_cls.dim() > 2: + if self.with_avg_pool: + x_cls = self.avg_pool(x_cls) + x_cls = x_cls.flatten(1) + for fc in self.cls_fcs: + x_cls = self.relu(fc(x_cls)) + + for conv in self.reg_convs: + x_reg = conv(x_reg) + if x_reg.dim() > 2: + if self.with_avg_pool: + x_reg = self.avg_pool(x_reg) + x_reg = x_reg.flatten(1) + for fc in self.reg_fcs: + x_reg = self.relu(fc(x_reg)) + + cls_score = self.fc_cls(x_cls) if self.with_cls else None + bbox_pred = self.fc_reg(x_reg) if self.with_reg else None + return cls_score, bbox_pred diff --git a/detection/mmdet_custom/models/dense_heads/deformable_detr_head.py b/detection/mmdet_custom/models/dense_heads/deformable_detr_head.py new file mode 100644 index 0000000..c26f1e3 --- /dev/null +++ b/detection/mmdet_custom/models/dense_heads/deformable_detr_head.py @@ -0,0 +1,332 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import copy + +import torch +import torch.nn as nn +import torch.nn.functional as F +from mmcv.cnn import Linear, bias_init_with_prob, constant_init +from mmcv.runner import force_fp32 + +from mmdet.core import multi_apply +from mmdet.models.utils.transformer import inverse_sigmoid +from mmdet.models.builder import HEADS +from .detr_head import DETRHead + + +@HEADS.register_module(force=True) +class DeformableDETRHead(DETRHead): + """Head of DeformDETR: Deformable DETR: Deformable Transformers for End-to- + End Object Detection. + + Code is modified from the `official github repo + `_. + + More details can be found in the `paper + `_ . + + Args: + with_box_refine (bool): Whether to refine the reference points + in the decoder. Defaults to False. + as_two_stage (bool) : Whether to generate the proposal from + the outputs of encoder. + transformer (obj:`ConfigDict`): ConfigDict is used for building + the Encoder and Decoder. + """ + + def __init__(self, + *args, + with_box_refine=False, + as_two_stage=False, + transformer=None, + use_2fc_cls_branch=False, + **kwargs): + self.with_box_refine = with_box_refine + self.as_two_stage = as_two_stage + self.use_2fc_cls_branch = use_2fc_cls_branch + if self.as_two_stage: + transformer['as_two_stage'] = self.as_two_stage + + super(DeformableDETRHead, self).__init__( + *args, transformer=transformer, **kwargs) + + def _init_layers(self): + """Initialize classification branch and regression branch of head.""" + + if not self.use_2fc_cls_branch: + fc_cls = Linear(self.embed_dims, self.cls_out_channels) + else: + fc_cls = nn.Sequential(*[ + Linear(self.embed_dims, int(self.embed_dims * 1.5)), + nn.LayerNorm(int(self.embed_dims * 1.5)), + nn.GELU(), + Linear(int(self.embed_dims * 1.5), self.cls_out_channels), + ]) + fc_cls.out_features = self.cls_out_channels + + reg_branch = [] + for _ in range(self.num_reg_fcs): + reg_branch.append(Linear(self.embed_dims, self.embed_dims)) + reg_branch.append(nn.ReLU()) + reg_branch.append(Linear(self.embed_dims, 4)) + reg_branch = nn.Sequential(*reg_branch) + + def _get_clones(module, N): + return nn.ModuleList([copy.deepcopy(module) for i in range(N)]) + + # last reg_branch is used to generate proposal from + # encode feature map when as_two_stage is True. + num_pred = (self.transformer.decoder.num_layers + 1) if \ + self.as_two_stage else self.transformer.decoder.num_layers + + if self.with_box_refine: + self.cls_branches = _get_clones(fc_cls, num_pred) + self.reg_branches = _get_clones(reg_branch, num_pred) + else: + + self.cls_branches = nn.ModuleList( + [fc_cls for _ in range(num_pred)]) + self.reg_branches = nn.ModuleList( + [reg_branch for _ in range(num_pred)]) + + if not self.as_two_stage: + self.query_embedding = nn.Embedding( + self.num_query, + self.embed_dims * 2) + + def init_weights(self): + """Initialize weights of the DeformDETR head.""" + self.transformer.init_weights() + if self.loss_cls.use_sigmoid: + bias_init = bias_init_with_prob(0.01) + if not self.use_2fc_cls_branch: + for m in self.cls_branches: + nn.init.constant_(m.bias, bias_init) + for m in self.reg_branches: + constant_init(m[-1], 0, bias=0) + nn.init.constant_(self.reg_branches[0][-1].bias.data[2:], -2.0) + if self.as_two_stage: + for m in self.reg_branches: + nn.init.constant_(m[-1].bias.data[2:], 0.0) + + def forward(self, mlvl_feats, img_metas): + """Forward function. + + Args: + mlvl_feats (tuple[Tensor]): Features from the upstream + network, each is a 4D-tensor with shape + (N, C, H, W). + img_metas (list[dict]): List of image information. + + Returns: + all_cls_scores (Tensor): Outputs from the classification head, \ + shape [nb_dec, bs, num_query, cls_out_channels]. Note \ + cls_out_channels should includes background. + all_bbox_preds (Tensor): Sigmoid outputs from the regression \ + head with normalized coordinate format (cx, cy, w, h). \ + Shape [nb_dec, bs, num_query, 4]. + enc_outputs_class (Tensor): The score of each point on encode \ + feature map, has shape (N, h*w, num_class). Only when \ + as_two_stage is True it would be returned, otherwise \ + `None` would be returned. + enc_outputs_coord (Tensor): The proposal generate from the \ + encode feature map, has shape (N, h*w, 4). Only when \ + as_two_stage is True it would be returned, otherwise \ + `None` would be returned. + """ + + batch_size = mlvl_feats[0].size(0) + input_img_h, input_img_w = img_metas[0]['batch_input_shape'] + img_masks = mlvl_feats[0].new_ones( + (batch_size, input_img_h, input_img_w)) + for img_id in range(batch_size): + img_h, img_w, _ = img_metas[img_id]['img_shape'] + img_masks[img_id, :img_h, :img_w] = 0 + + mlvl_masks = [] + mlvl_positional_encodings = [] + for feat in mlvl_feats: + mlvl_masks.append( + F.interpolate(img_masks[None], + size=feat.shape[-2:]).to(torch.bool).squeeze(0)) + mlvl_positional_encodings.append( + self.positional_encoding(mlvl_masks[-1])) + + query_embeds = None + if not self.as_two_stage: + query_embeds = self.query_embedding.weight + hs, init_reference, inter_references, \ + enc_outputs_class, enc_outputs_coord = self.transformer( + mlvl_feats, + mlvl_masks, + query_embeds, + mlvl_positional_encodings, + reg_branches=self.reg_branches if self.with_box_refine else None, # noqa:E501 + cls_branches=self.cls_branches if self.as_two_stage else None # noqa:E501 + ) + hs = hs.permute(0, 2, 1, 3) + outputs_classes = [] + outputs_coords = [] + + for lvl in range(hs.shape[0]): + if lvl == 0: + reference = init_reference + else: + reference = inter_references[lvl - 1] + reference = inverse_sigmoid(reference) + outputs_class = self.cls_branches[lvl](hs[lvl]) + tmp = self.reg_branches[lvl](hs[lvl]) + if reference.shape[-1] == 4: + tmp += reference + else: + assert reference.shape[-1] == 2 + tmp[..., :2] += reference + outputs_coord = tmp.sigmoid() + outputs_classes.append(outputs_class) + outputs_coords.append(outputs_coord) + + outputs_classes = torch.stack(outputs_classes) + outputs_coords = torch.stack(outputs_coords) + if self.as_two_stage: + return outputs_classes, outputs_coords, \ + enc_outputs_class, \ + enc_outputs_coord.sigmoid() + else: + return outputs_classes, outputs_coords, \ + None, None + + @force_fp32(apply_to=('all_cls_scores_list', 'all_bbox_preds_list')) + def loss(self, + all_cls_scores, + all_bbox_preds, + enc_cls_scores, + enc_bbox_preds, + gt_bboxes_list, + gt_labels_list, + img_metas, + gt_bboxes_ignore=None): + """"Loss function. + + Args: + all_cls_scores (Tensor): Classification score of all + decoder layers, has shape + [nb_dec, bs, num_query, cls_out_channels]. + all_bbox_preds (Tensor): Sigmoid regression + outputs of all decode layers. Each is a 4D-tensor with + normalized coordinate format (cx, cy, w, h) and shape + [nb_dec, bs, num_query, 4]. + enc_cls_scores (Tensor): Classification scores of + points on encode feature map , has shape + (N, h*w, num_classes). Only be passed when as_two_stage is + True, otherwise is None. + enc_bbox_preds (Tensor): Regression results of each points + on the encode feature map, has shape (N, h*w, 4). Only be + passed when as_two_stage is True, otherwise is None. + gt_bboxes_list (list[Tensor]): Ground truth bboxes for each image + with shape (num_gts, 4) in [tl_x, tl_y, br_x, br_y] format. + gt_labels_list (list[Tensor]): Ground truth class indices for each + image with shape (num_gts, ). + img_metas (list[dict]): List of image meta information. + gt_bboxes_ignore (list[Tensor], optional): Bounding boxes + which can be ignored for each image. Default None. + + Returns: + dict[str, Tensor]: A dictionary of loss components. + """ + assert gt_bboxes_ignore is None, \ + f'{self.__class__.__name__} only supports ' \ + f'for gt_bboxes_ignore setting to None.' + + num_dec_layers = len(all_cls_scores) + all_gt_bboxes_list = [gt_bboxes_list for _ in range(num_dec_layers)] + all_gt_labels_list = [gt_labels_list for _ in range(num_dec_layers)] + all_gt_bboxes_ignore_list = [ + gt_bboxes_ignore for _ in range(num_dec_layers) + ] + img_metas_list = [img_metas for _ in range(num_dec_layers)] + + losses_cls, losses_bbox, losses_iou = multi_apply( + self.loss_single, all_cls_scores, all_bbox_preds, + all_gt_bboxes_list, all_gt_labels_list, img_metas_list, + all_gt_bboxes_ignore_list) + + loss_dict = dict() + # loss of proposal generated from encode feature map. + if enc_cls_scores is not None: + binary_labels_list = [ + torch.zeros_like(gt_labels_list[i]) + for i in range(len(img_metas)) + ] + enc_loss_cls, enc_losses_bbox, enc_losses_iou = \ + self.loss_single(enc_cls_scores, enc_bbox_preds, + gt_bboxes_list, binary_labels_list, + img_metas, gt_bboxes_ignore) + loss_dict['enc_loss_cls'] = enc_loss_cls + loss_dict['enc_loss_bbox'] = enc_losses_bbox + loss_dict['enc_loss_iou'] = enc_losses_iou + + # loss from the last decoder layer + loss_dict['loss_cls'] = losses_cls[-1] + loss_dict['loss_bbox'] = losses_bbox[-1] + loss_dict['loss_iou'] = losses_iou[-1] + # loss from other decoder layers + num_dec_layer = 0 + for loss_cls_i, loss_bbox_i, loss_iou_i in zip(losses_cls[:-1], + losses_bbox[:-1], + losses_iou[:-1]): + loss_dict[f'd{num_dec_layer}.loss_cls'] = loss_cls_i + loss_dict[f'd{num_dec_layer}.loss_bbox'] = loss_bbox_i + loss_dict[f'd{num_dec_layer}.loss_iou'] = loss_iou_i + num_dec_layer += 1 + return loss_dict + + @force_fp32(apply_to=('all_cls_scores_list', 'all_bbox_preds_list')) + def get_bboxes(self, + all_cls_scores, + all_bbox_preds, + enc_cls_scores, + enc_bbox_preds, + img_metas, + rescale=False): + """Transform network outputs for a batch into bbox predictions. + + Args: + all_cls_scores (Tensor): Classification score of all + decoder layers, has shape + [nb_dec, bs, num_query, cls_out_channels]. + all_bbox_preds (Tensor): Sigmoid regression + outputs of all decode layers. Each is a 4D-tensor with + normalized coordinate format (cx, cy, w, h) and shape + [nb_dec, bs, num_query, 4]. + enc_cls_scores (Tensor): Classification scores of + points on encode feature map , has shape + (N, h*w, num_classes). Only be passed when as_two_stage is + True, otherwise is None. + enc_bbox_preds (Tensor): Regression results of each points + on the encode feature map, has shape (N, h*w, 4). Only be + passed when as_two_stage is True, otherwise is None. + img_metas (list[dict]): Meta information of each image. + rescale (bool, optional): If True, return boxes in original + image space. Default False. + + Returns: + list[list[Tensor, Tensor]]: Each item in result_list is 2-tuple. \ + The first item is an (n, 5) tensor, where the first 4 columns \ + are bounding box positions (tl_x, tl_y, br_x, br_y) and the \ + 5-th column is a score between 0 and 1. The second item is a \ + (n,) tensor where each item is the predicted class label of \ + the corresponding box. + """ + cls_scores = all_cls_scores[-1] + bbox_preds = all_bbox_preds[-1] + + result_list = [] + for img_id in range(len(img_metas)): + cls_score = cls_scores[img_id] + bbox_pred = bbox_preds[img_id] + img_shape = img_metas[img_id]['img_shape'] + scale_factor = img_metas[img_id]['scale_factor'] + proposals = self._get_bboxes_single(cls_score, bbox_pred, + img_shape, scale_factor, + rescale) + result_list.append(proposals) + return result_list diff --git a/detection/mmdet_custom/models/dense_heads/detr_head.py b/detection/mmdet_custom/models/dense_heads/detr_head.py new file mode 100644 index 0000000..c832e59 --- /dev/null +++ b/detection/mmdet_custom/models/dense_heads/detr_head.py @@ -0,0 +1,954 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +import torch.nn as nn +import torch.nn.functional as F +from mmcv.cnn import Conv2d, Linear, build_activation_layer +from mmcv.cnn.bricks.transformer import FFN, build_positional_encoding +from mmcv.runner import force_fp32 + +from mmdet.core import (bbox_cxcywh_to_xyxy, bbox_xyxy_to_cxcywh, + build_assigner, build_sampler, multi_apply, + reduce_mean) +from mmdet.models.utils import build_transformer +from mmdet.models.builder import HEADS, build_loss +from mmdet.models.dense_heads.anchor_free_head import AnchorFreeHead +import numpy as np + + +@HEADS.register_module(force=True) +class DETRHead(AnchorFreeHead): + """Implements the DETR transformer head. + + See `paper: End-to-End Object Detection with Transformers + `_ for details. + + Args: + num_classes (int): Number of categories excluding the background. + in_channels (int): Number of channels in the input feature map. + num_query (int): Number of query in Transformer. + num_reg_fcs (int, optional): Number of fully-connected layers used in + `FFN`, which is then used for the regression head. Default 2. + transformer (obj:`mmcv.ConfigDict`|dict): Config for transformer. + Default: None. + sync_cls_avg_factor (bool): Whether to sync the avg_factor of + all ranks. Default to False. + positional_encoding (obj:`mmcv.ConfigDict`|dict): + Config for position encoding. + loss_cls (obj:`mmcv.ConfigDict`|dict): Config of the + classification loss. Default `CrossEntropyLoss`. + loss_bbox (obj:`mmcv.ConfigDict`|dict): Config of the + regression loss. Default `L1Loss`. + loss_iou (obj:`mmcv.ConfigDict`|dict): Config of the + regression iou loss. Default `GIoULoss`. + tran_cfg (obj:`mmcv.ConfigDict`|dict): Training config of + transformer head. + test_cfg (obj:`mmcv.ConfigDict`|dict): Testing config of + transformer head. + init_cfg (dict or list[dict], optional): Initialization config dict. + Default: None + """ + + _version = 2 + + def __init__(self, + num_classes, + in_channels, + num_query=100, + num_reg_fcs=2, + transformer=None, + sync_cls_avg_factor=False, + positional_encoding=dict( + type='SinePositionalEncoding', + num_feats=128, + normalize=True), + loss_cls=dict( + type='CrossEntropyLoss', + bg_cls_weight=0.1, + use_sigmoid=False, + loss_weight=1.0, + class_weight=1.0), + loss_bbox=dict(type='L1Loss', loss_weight=5.0), + loss_iou=dict(type='GIoULoss', loss_weight=2.0), + train_cfg=dict( + assigner=dict( + type='HungarianAssigner', + cls_cost=dict(type='ClassificationCost', weight=1.), + reg_cost=dict(type='BBoxL1Cost', weight=5.0), + iou_cost=dict( + type='IoUCost', iou_mode='giou', weight=2.0))), + test_cfg=dict(max_per_img=100), + init_cfg=None, + **kwargs): + # NOTE here use `AnchorFreeHead` instead of `TransformerHead`, + # since it brings inconvenience when the initialization of + # `AnchorFreeHead` is called. + super(AnchorFreeHead, self).__init__(init_cfg) + + self.bg_cls_weight = 0 + self.sync_cls_avg_factor = sync_cls_avg_factor + class_weight = loss_cls.get('class_weight', None) + if class_weight is not None and (self.__class__ is DETRHead): + # assert isinstance(class_weight, float), 'Expected ' \ + # 'class_weight to have type float. Found ' \ + # f'{type(class_weight)}.' + + # NOTE following the official DETR rep0, bg_cls_weight means + # relative classification weight of the no-object class. + bg_cls_weight = loss_cls.get('bg_cls_weight', class_weight) + + assert isinstance(bg_cls_weight, float), 'Expected ' \ + 'bg_cls_weight to have type float. Found ' \ + f'{type(bg_cls_weight)}.' + if isinstance(class_weight, list): + class_weight.append(bg_cls_weight) + class_weight = np.array(class_weight) + class_weight = torch.from_numpy(class_weight) + class_weight = torch.ones(num_classes + 1) * class_weight + elif isinstance(class_weight, float): + class_weight = torch.ones(num_classes + 1) * class_weight + # set background class as the last indice + class_weight[num_classes] = bg_cls_weight + loss_cls.update({'class_weight': class_weight}) + if 'bg_cls_weight' in loss_cls: + loss_cls.pop('bg_cls_weight') + self.bg_cls_weight = bg_cls_weight + + if train_cfg: + assert 'assigner' in train_cfg, 'assigner should be provided ' \ + 'when train_cfg is set.' + assigner = train_cfg['assigner'] + # assert loss_cls['loss_weight'] == assigner['cls_cost']['weight'], + # 'The classification weight for loss and matcher should be' \ + # 'exactly the same.' + # assert loss_bbox['loss_weight'] == assigner['reg_cost'][ + # 'weight'], 'The regression L1 weight for loss and matcher '\ + # 'should be exactly the same.' + # assert loss_iou['loss_weight'] == assigner['iou_cost']['weight'], + # 'The regression iou weight for loss and matcher should be' \ + # 'exactly the same.' + self.assigner = build_assigner(assigner) + # DETR sampling=False, so use PseudoSampler + sampler_cfg = dict(type='PseudoSampler') + self.sampler = build_sampler(sampler_cfg, context=self) + + self.num_query = num_query + self.num_classes = num_classes + self.in_channels = in_channels + self.num_reg_fcs = num_reg_fcs + self.train_cfg = train_cfg + self.test_cfg = test_cfg + self.fp16_enabled = False + self.loss_cls = build_loss(loss_cls) + self.loss_bbox = build_loss(loss_bbox) + self.loss_iou = build_loss(loss_iou) + + if self.loss_cls.use_sigmoid: + self.cls_out_channels = num_classes + else: + self.cls_out_channels = num_classes + 1 + self.act_cfg = transformer.get('act_cfg', + dict(type='ReLU', inplace=True)) + self.activate = build_activation_layer(self.act_cfg) + self.positional_encoding = build_positional_encoding( + positional_encoding) + self.transformer = build_transformer(transformer) + self.embed_dims = self.transformer.embed_dims + assert 'num_feats' in positional_encoding + num_feats = positional_encoding['num_feats'] + assert num_feats * 2 == self.embed_dims, 'embed_dims should' \ + f' be exactly 2 times of num_feats. Found {self.embed_dims}' \ + f' and {num_feats}.' + + self._init_layers() + + def _init_layers(self): + """Initialize layers of the transformer head.""" + self.input_proj = Conv2d( + self.in_channels, self.embed_dims, kernel_size=1) + self.fc_cls = Linear(self.embed_dims, self.cls_out_channels) + self.reg_ffn = FFN( + self.embed_dims, + self.embed_dims, + self.num_reg_fcs, + self.act_cfg, + dropout=0.0, + add_residual=False) + self.fc_reg = Linear(self.embed_dims, 4) + self.query_embedding = nn.Embedding(self.num_query, self.embed_dims) + + def init_weights(self): + """Initialize weights of the transformer head.""" + # The initialization for transformer is important + self.transformer.init_weights() + + def _load_from_state_dict(self, state_dict, prefix, local_metadata, strict, + missing_keys, unexpected_keys, error_msgs): + """load checkpoints.""" + # NOTE here use `AnchorFreeHead` instead of `TransformerHead`, + # since `AnchorFreeHead._load_from_state_dict` should not be + # called here. Invoking the default `Module._load_from_state_dict` + # is enough. + + # Names of some parameters in has been changed. + version = local_metadata.get('version', None) + if (version is None or version < 2) and self.__class__ is DETRHead: + convert_dict = { + '.self_attn.': '.attentions.0.', + '.ffn.': '.ffns.0.', + '.multihead_attn.': '.attentions.1.', + '.decoder.norm.': '.decoder.post_norm.' + } + state_dict_keys = list(state_dict.keys()) + for k in state_dict_keys: + for ori_key, convert_key in convert_dict.items(): + if ori_key in k: + convert_key = k.replace(ori_key, convert_key) + state_dict[convert_key] = state_dict[k] + del state_dict[k] + + super(AnchorFreeHead, + self)._load_from_state_dict(state_dict, prefix, local_metadata, + strict, missing_keys, + unexpected_keys, error_msgs) + + def forward(self, feats, img_metas): + """Forward function. + + Args: + feats (tuple[Tensor]): Features from the upstream network, each is + a 4D-tensor. + img_metas (list[dict]): List of image information. + + Returns: + tuple[list[Tensor], list[Tensor]]: Outputs for all scale levels. + + - all_cls_scores_list (list[Tensor]): Classification scores \ + for each scale level. Each is a 4D-tensor with shape \ + [nb_dec, bs, num_query, cls_out_channels]. Note \ + `cls_out_channels` should includes background. + - all_bbox_preds_list (list[Tensor]): Sigmoid regression \ + outputs for each scale level. Each is a 4D-tensor with \ + normalized coordinate format (cx, cy, w, h) and shape \ + [nb_dec, bs, num_query, 4]. + """ + num_levels = len(feats) + img_metas_list = [img_metas for _ in range(num_levels)] + return multi_apply(self.forward_single, feats, img_metas_list) + + def forward_single(self, x, img_metas): + """"Forward function for a single feature level. + + Args: + x (Tensor): Input feature from backbone's single stage, shape + [bs, c, h, w]. + img_metas (list[dict]): List of image information. + + Returns: + all_cls_scores (Tensor): Outputs from the classification head, + shape [nb_dec, bs, num_query, cls_out_channels]. Note + cls_out_channels should includes background. + all_bbox_preds (Tensor): Sigmoid outputs from the regression + head with normalized coordinate format (cx, cy, w, h). + Shape [nb_dec, bs, num_query, 4]. + """ + # construct binary masks which used for the transformer. + # NOTE following the official DETR repo, non-zero values representing + # ignored positions, while zero values means valid positions. + batch_size = x.size(0) + input_img_h, input_img_w = img_metas[0]['batch_input_shape'] + masks = x.new_ones((batch_size, input_img_h, input_img_w)) + for img_id in range(batch_size): + img_h, img_w, _ = img_metas[img_id]['img_shape'] + masks[img_id, :img_h, :img_w] = 0 + + x = self.input_proj(x) + # interpolate masks to have the same spatial shape with x + masks = F.interpolate( + masks.unsqueeze(1), size=x.shape[-2:]).to(torch.bool).squeeze(1) + # position encoding + pos_embed = self.positional_encoding(masks) # [bs, embed_dim, h, w] + # outs_dec: [nb_dec, bs, num_query, embed_dim] + outs_dec, _ = self.transformer(x, masks, self.query_embedding.weight, + pos_embed) + + all_cls_scores = self.fc_cls(outs_dec) + all_bbox_preds = self.fc_reg(self.activate( + self.reg_ffn(outs_dec))).sigmoid() + return all_cls_scores, all_bbox_preds + + @force_fp32(apply_to=('all_cls_scores_list', 'all_bbox_preds_list')) + def loss(self, + all_cls_scores_list, + all_bbox_preds_list, + gt_bboxes_list, + gt_labels_list, + img_metas, + gt_bboxes_ignore=None): + """"Loss function. + + Only outputs from the last feature level are used for computing + losses by default. + + Args: + all_cls_scores_list (list[Tensor]): Classification outputs + for each feature level. Each is a 4D-tensor with shape + [nb_dec, bs, num_query, cls_out_channels]. + all_bbox_preds_list (list[Tensor]): Sigmoid regression + outputs for each feature level. Each is a 4D-tensor with + normalized coordinate format (cx, cy, w, h) and shape + [nb_dec, bs, num_query, 4]. + gt_bboxes_list (list[Tensor]): Ground truth bboxes for each image + with shape (num_gts, 4) in [tl_x, tl_y, br_x, br_y] format. + gt_labels_list (list[Tensor]): Ground truth class indices for each + image with shape (num_gts, ). + img_metas (list[dict]): List of image meta information. + gt_bboxes_ignore (list[Tensor], optional): Bounding boxes + which can be ignored for each image. Default None. + + Returns: + dict[str, Tensor]: A dictionary of loss components. + """ + # NOTE defaultly only the outputs from the last feature scale is used. + all_cls_scores = all_cls_scores_list[-1] + all_bbox_preds = all_bbox_preds_list[-1] + assert gt_bboxes_ignore is None, \ + 'Only supports for gt_bboxes_ignore setting to None.' + + num_dec_layers = len(all_cls_scores) + all_gt_bboxes_list = [gt_bboxes_list for _ in range(num_dec_layers)] + all_gt_labels_list = [gt_labels_list for _ in range(num_dec_layers)] + all_gt_bboxes_ignore_list = [ + gt_bboxes_ignore for _ in range(num_dec_layers) + ] + img_metas_list = [img_metas for _ in range(num_dec_layers)] + + losses_cls, losses_bbox, losses_iou = multi_apply( + self.loss_single, all_cls_scores, all_bbox_preds, + all_gt_bboxes_list, all_gt_labels_list, img_metas_list, + all_gt_bboxes_ignore_list) + + loss_dict = dict() + # loss from the last decoder layer + loss_dict['loss_cls'] = losses_cls[-1] + loss_dict['loss_bbox'] = losses_bbox[-1] + loss_dict['loss_iou'] = losses_iou[-1] + # loss from other decoder layers + num_dec_layer = 0 + for loss_cls_i, loss_bbox_i, loss_iou_i in zip(losses_cls[:-1], + losses_bbox[:-1], + losses_iou[:-1]): + loss_dict[f'd{num_dec_layer}.loss_cls'] = loss_cls_i + loss_dict[f'd{num_dec_layer}.loss_bbox'] = loss_bbox_i + loss_dict[f'd{num_dec_layer}.loss_iou'] = loss_iou_i + num_dec_layer += 1 + return loss_dict + + def get_fed_loss_classes(self, gt_classes, num_fed_loss_classes, num_classes, weight): + """ + Args: + gt_classes: a long tensor of shape R that contains the gt class label of each proposal. + num_fed_loss_classes: minimum number of classes to keep when calculating federated loss. + Will sample negative classes if number of unique gt_classes is smaller than this value. + num_classes: number of foreground classes + weight: probabilities used to sample negative classes + Returns: + Tensor: + classes to keep when calculating the federated loss, including both unique gt + classes and sampled negative classes. + """ + unique_gt_classes = torch.unique(gt_classes) + prob = unique_gt_classes.new_ones(num_classes + 1).float() + prob[-1] = 0 + if len(unique_gt_classes) < num_fed_loss_classes: + prob[:num_classes] = weight.float().clone() + prob[unique_gt_classes] = 0 + sampled_negative_classes = torch.multinomial( + prob, num_fed_loss_classes - len(unique_gt_classes), replacement=False + ) + fed_loss_classes = torch.cat([unique_gt_classes, sampled_negative_classes]) + else: + fed_loss_classes = unique_gt_classes + return fed_loss_classes + + def loss_single(self, + cls_scores, + bbox_preds, + gt_bboxes_list, + gt_labels_list, + img_metas, + gt_bboxes_ignore_list=None): + """"Loss function for outputs from a single decoder layer of a single + feature level. + + Args: + cls_scores (Tensor): Box score logits from a single decoder layer + for all images. Shape [bs, num_query, cls_out_channels]. + bbox_preds (Tensor): Sigmoid outputs from a single decoder layer + for all images, with normalized coordinate (cx, cy, w, h) and + shape [bs, num_query, 4]. + gt_bboxes_list (list[Tensor]): Ground truth bboxes for each image + with shape (num_gts, 4) in [tl_x, tl_y, br_x, br_y] format. + gt_labels_list (list[Tensor]): Ground truth class indices for each + image with shape (num_gts, ). + img_metas (list[dict]): List of image meta information. + gt_bboxes_ignore_list (list[Tensor], optional): Bounding + boxes which can be ignored for each image. Default None. + + Returns: + dict[str, Tensor]: A dictionary of loss components for outputs from + a single decoder layer. + """ + num_imgs = cls_scores.size(0) + cls_scores_list = [cls_scores[i] for i in range(num_imgs)] + bbox_preds_list = [bbox_preds[i] for i in range(num_imgs)] + cls_reg_targets = self.get_targets(cls_scores_list, bbox_preds_list, + gt_bboxes_list, gt_labels_list, + img_metas, gt_bboxes_ignore_list) + (labels_list, label_weights_list, bbox_targets_list, bbox_weights_list, + num_total_pos, num_total_neg) = cls_reg_targets + + labels = torch.cat(labels_list, 0) + label_weights = torch.cat(label_weights_list, 0) + bbox_targets = torch.cat(bbox_targets_list, 0) + bbox_weights = torch.cat(bbox_weights_list, 0) + + # classification loss + cls_scores = cls_scores.reshape(-1, self.cls_out_channels) + # construct weighted avg_factor to match with the official DETR repo + cls_avg_factor = num_total_pos * 1.0 + \ + num_total_neg * self.bg_cls_weight + if self.sync_cls_avg_factor: + cls_avg_factor = reduce_mean( + cls_scores.new_tensor([cls_avg_factor])) + cls_avg_factor = max(cls_avg_factor, 1) + + loss_cls = self.loss_cls( + cls_scores, labels, label_weights, avg_factor=cls_avg_factor) + + # Compute the average number of gt boxes across all gpus, for + # normalization purposes + num_total_pos = loss_cls.new_tensor([num_total_pos]) + num_total_pos = torch.clamp(reduce_mean(num_total_pos), min=1).item() + + # construct factors used for rescale bboxes + factors = [] + for img_meta, bbox_pred in zip(img_metas, bbox_preds): + img_h, img_w, _ = img_meta['img_shape'] + factor = bbox_pred.new_tensor([img_w, img_h, img_w, + img_h]).unsqueeze(0).repeat( + bbox_pred.size(0), 1) + factors.append(factor) + factors = torch.cat(factors, 0) + + # DETR regress the relative position of boxes (cxcywh) in the image, + # thus the learning target is normalized by the image size. So here + # we need to re-scale them for calculating IoU loss + bbox_preds = bbox_preds.reshape(-1, 4) + bboxes = bbox_cxcywh_to_xyxy(bbox_preds) * factors + bboxes_gt = bbox_cxcywh_to_xyxy(bbox_targets) * factors + + # regression IoU loss, defaultly GIoU loss + loss_iou = self.loss_iou( + bboxes, bboxes_gt, bbox_weights, avg_factor=num_total_pos) + + # regression L1 loss + loss_bbox = self.loss_bbox( + bbox_preds, bbox_targets, bbox_weights, avg_factor=num_total_pos) + return loss_cls, loss_bbox, loss_iou + + def get_targets(self, + cls_scores_list, + bbox_preds_list, + gt_bboxes_list, + gt_labels_list, + img_metas, + gt_bboxes_ignore_list=None): + """"Compute regression and classification targets for a batch image. + + Outputs from a single decoder layer of a single feature level are used. + + Args: + cls_scores_list (list[Tensor]): Box score logits from a single + decoder layer for each image with shape [num_query, + cls_out_channels]. + bbox_preds_list (list[Tensor]): Sigmoid outputs from a single + decoder layer for each image, with normalized coordinate + (cx, cy, w, h) and shape [num_query, 4]. + gt_bboxes_list (list[Tensor]): Ground truth bboxes for each image + with shape (num_gts, 4) in [tl_x, tl_y, br_x, br_y] format. + gt_labels_list (list[Tensor]): Ground truth class indices for each + image with shape (num_gts, ). + img_metas (list[dict]): List of image meta information. + gt_bboxes_ignore_list (list[Tensor], optional): Bounding + boxes which can be ignored for each image. Default None. + + Returns: + tuple: a tuple containing the following targets. + + - labels_list (list[Tensor]): Labels for all images. + - label_weights_list (list[Tensor]): Label weights for all \ + images. + - bbox_targets_list (list[Tensor]): BBox targets for all \ + images. + - bbox_weights_list (list[Tensor]): BBox weights for all \ + images. + - num_total_pos (int): Number of positive samples in all \ + images. + - num_total_neg (int): Number of negative samples in all \ + images. + """ + assert gt_bboxes_ignore_list is None, \ + 'Only supports for gt_bboxes_ignore setting to None.' + num_imgs = len(cls_scores_list) + gt_bboxes_ignore_list = [ + gt_bboxes_ignore_list for _ in range(num_imgs) + ] + + (labels_list, label_weights_list, bbox_targets_list, + bbox_weights_list, pos_inds_list, neg_inds_list) = multi_apply( + self._get_target_single, cls_scores_list, bbox_preds_list, + gt_bboxes_list, gt_labels_list, img_metas, gt_bboxes_ignore_list) + num_total_pos = sum((inds.numel() for inds in pos_inds_list)) + num_total_neg = sum((inds.numel() for inds in neg_inds_list)) + return (labels_list, label_weights_list, bbox_targets_list, + bbox_weights_list, num_total_pos, num_total_neg) + + def _get_area_thr(self, img_shape, type): + MIN_V = 0 + MAX_V = 1e10 + short_edge = min(img_shape[0], img_shape[1]) + if type == 'v1': + DELTA = 4 + if short_edge <= 600: + min_edge = 128 - DELTA + max_edge = MAX_V + elif 600 < short_edge <= 800: + min_edge = 96 - DELTA + max_edge = MAX_V + elif 800 < short_edge <= 1000: + min_edge = 64 - DELTA + max_edge = MAX_V + elif 1000 < short_edge <= 1200: + min_edge = 32 - DELTA + max_edge = MAX_V + elif 1200 < short_edge <= 1400: + min_edge = MIN_V + max_edge = MAX_V + else: + min_edge = MIN_V + max_edge = 2 + DELTA + elif type == 'v2': + if short_edge <= 1000: + min_edge = 112 + max_edge = MAX_V + elif 1000 < short_edge <= 1400: + min_edge = 32 + max_edge = 160 + elif short_edge > 1400: + min_edge = 0 + max_edge = 80 + elif type == 'v3': + if short_edge <= 800: + min_edge = 96 + max_edge = MAX_V + elif 800 < short_edge <= 1000: + min_edge = 64 + max_edge = MAX_V + elif 1000 < short_edge <= 1400: + min_edge = MIN_V + max_edge = MAX_V + elif 1400 < short_edge <= 1600: + min_edge = MIN_V + max_edge = 96 + elif short_edge > 1600: + min_edge = MIN_V + max_edge = 64 + elif type == 'v4': + DELTA = 4 + if short_edge <= 800: + min_edge = 96 - DELTA + max_edge = MAX_V + elif 800 < short_edge <= 1000: + min_edge = 64 - DELTA + max_edge = MAX_V + elif 1000 < short_edge <= 1400: + min_edge = MIN_V + max_edge = MAX_V + elif 1400 < short_edge <= 1600: + min_edge = MIN_V + max_edge = 64 + DELTA + elif short_edge > 1600: + min_edge = MIN_V + max_edge = 32 + DELTA + + return min_edge ** 2, max_edge ** 2 + + def _get_target_single(self, + cls_score, + bbox_pred, + gt_bboxes, + gt_labels, + img_meta, + gt_bboxes_ignore=None): + """"Compute regression and classification targets for one image. + + Outputs from a single decoder layer of a single feature level are used. + + Args: + cls_score (Tensor): Box score logits from a single decoder layer + for one image. Shape [num_query, cls_out_channels]. + bbox_pred (Tensor): Sigmoid outputs from a single decoder layer + for one image, with normalized coordinate (cx, cy, w, h) and + shape [num_query, 4]. + gt_bboxes (Tensor): Ground truth bboxes for one image with + shape (num_gts, 4) in [tl_x, tl_y, br_x, br_y] format. + gt_labels (Tensor): Ground truth class indices for one image + with shape (num_gts, ). + img_meta (dict): Meta information for one image. + gt_bboxes_ignore (Tensor, optional): Bounding boxes + which can be ignored. Default None. + + Returns: + tuple[Tensor]: a tuple containing the following for one image. + + - labels (Tensor): Labels of each image. + - label_weights (Tensor]): Label weights of each image. + - bbox_targets (Tensor): BBox targets of each image. + - bbox_weights (Tensor): BBox weights of each image. + - pos_inds (Tensor): Sampled positive indices for each image. + - neg_inds (Tensor): Sampled negative indices for each image. + """ + + num_bboxes = bbox_pred.size(0) + # assigner and sampler + assign_result = self.assigner.assign(bbox_pred, cls_score, gt_bboxes, + gt_labels, img_meta, + gt_bboxes_ignore) + sampling_result = self.sampler.sample(assign_result, bbox_pred, + gt_bboxes) + pos_inds = sampling_result.pos_inds + neg_inds = sampling_result.neg_inds + + # label targets + labels = gt_bboxes.new_full((num_bboxes, ), + self.num_classes, + dtype=torch.long) + labels[pos_inds] = gt_labels[sampling_result.pos_assigned_gt_inds] + label_weights = gt_bboxes.new_ones(num_bboxes) + + # bbox targets + bbox_targets = torch.zeros_like(bbox_pred) + bbox_weights = torch.zeros_like(bbox_pred) + bbox_weights[pos_inds] = 1.0 + img_h, img_w, _ = img_meta['img_shape'] + + # DETR regress the relative position of boxes (cxcywh) in the image. + # Thus the learning target should be normalized by the image size, also + # the box format should be converted from defaultly x1y1x2y2 to cxcywh. + factor = bbox_pred.new_tensor([img_w, img_h, img_w, + img_h]).unsqueeze(0) + pos_gt_bboxes_normalized = sampling_result.pos_gt_bboxes / factor + pos_gt_bboxes_targets = bbox_xyxy_to_cxcywh(pos_gt_bboxes_normalized) + bbox_targets[pos_inds] = pos_gt_bboxes_targets + return (labels, label_weights, bbox_targets, bbox_weights, pos_inds, + neg_inds) + + # over-write because img_metas are needed as inputs for bbox_head. + def forward_train(self, + x, + img_metas, + gt_bboxes, + gt_labels=None, + gt_bboxes_ignore=None, + proposal_cfg=None, + **kwargs): + """Forward function for training mode. + + Args: + x (list[Tensor]): Features from backbone. + img_metas (list[dict]): Meta information of each image, e.g., + image size, scaling factor, etc. + gt_bboxes (Tensor): Ground truth bboxes of the image, + shape (num_gts, 4). + gt_labels (Tensor): Ground truth labels of each box, + shape (num_gts,). + gt_bboxes_ignore (Tensor): Ground truth bboxes to be + ignored, shape (num_ignored_gts, 4). + proposal_cfg (mmcv.Config): Test / postprocessing configuration, + if None, test_cfg would be used. + + Returns: + dict[str, Tensor]: A dictionary of loss components. + """ + assert proposal_cfg is None, '"proposal_cfg" must be None' + outs = self(x, img_metas) + if gt_labels is None: + loss_inputs = outs + (gt_bboxes, img_metas) + else: + loss_inputs = outs + (gt_bboxes, gt_labels, img_metas) + losses = self.loss(*loss_inputs, gt_bboxes_ignore=gt_bboxes_ignore) + return losses + + @force_fp32(apply_to=('all_cls_scores_list', 'all_bbox_preds_list')) + def get_bboxes(self, + all_cls_scores_list, + all_bbox_preds_list, + img_metas, + rescale=False): + """Transform network outputs for a batch into bbox predictions. + + Args: + all_cls_scores_list (list[Tensor]): Classification outputs + for each feature level. Each is a 4D-tensor with shape + [nb_dec, bs, num_query, cls_out_channels]. + all_bbox_preds_list (list[Tensor]): Sigmoid regression + outputs for each feature level. Each is a 4D-tensor with + normalized coordinate format (cx, cy, w, h) and shape + [nb_dec, bs, num_query, 4]. + img_metas (list[dict]): Meta information of each image. + rescale (bool, optional): If True, return boxes in original + image space. Default False. + + Returns: + list[list[Tensor, Tensor]]: Each item in result_list is 2-tuple. \ + The first item is an (n, 5) tensor, where the first 4 columns \ + are bounding box positions (tl_x, tl_y, br_x, br_y) and the \ + 5-th column is a score between 0 and 1. The second item is a \ + (n,) tensor where each item is the predicted class label of \ + the corresponding box. + """ + # NOTE defaultly only using outputs from the last feature level, + # and only the outputs from the last decoder layer is used. + cls_scores = all_cls_scores_list[-1][-1] + bbox_preds = all_bbox_preds_list[-1][-1] + + result_list = [] + for img_id in range(len(img_metas)): + cls_score = cls_scores[img_id] + bbox_pred = bbox_preds[img_id] + img_shape = img_metas[img_id]['img_shape'] + scale_factor = img_metas[img_id]['scale_factor'] + proposals = self._get_bboxes_single(cls_score, bbox_pred, + img_shape, scale_factor, + rescale) + result_list.append(proposals) + + return result_list + + def _get_bboxes_single(self, + cls_score, + bbox_pred, + img_shape, + scale_factor, + rescale=False): + """Transform outputs from the last decoder layer into bbox predictions + for each image. + + Args: + cls_score (Tensor): Box score logits from the last decoder layer + for each image. Shape [num_query, cls_out_channels]. + bbox_pred (Tensor): Sigmoid outputs from the last decoder layer + for each image, with coordinate format (cx, cy, w, h) and + shape [num_query, 4]. + img_shape (tuple[int]): Shape of input image, (height, width, 3). + scale_factor (ndarray, optional): Scale factor of the image arange + as (w_scale, h_scale, w_scale, h_scale). + rescale (bool, optional): If True, return boxes in original image + space. Default False. + + Returns: + tuple[Tensor]: Results of detected bboxes and labels. + + - det_bboxes: Predicted bboxes with shape [num_query, 5], \ + where the first 4 columns are bounding box positions \ + (tl_x, tl_y, br_x, br_y) and the 5-th column are scores \ + between 0 and 1. + - det_labels: Predicted labels of the corresponding box with \ + shape [num_query]. + """ + assert len(cls_score) == len(bbox_pred) + max_per_img = self.test_cfg.get('max_per_img', self.num_query) + # exclude background + if self.loss_cls.use_sigmoid: + cls_score = cls_score.sigmoid() + scores, indexes = cls_score.view(-1).topk(max_per_img) + det_labels = indexes % self.num_classes + bbox_index = indexes // self.num_classes + bbox_pred = bbox_pred[bbox_index] + else: + scores, det_labels = F.softmax(cls_score, dim=-1)[..., :-1].max(-1) + scores, bbox_index = scores.topk(max_per_img) + bbox_pred = bbox_pred[bbox_index] + det_labels = det_labels[bbox_index] + + det_bboxes = bbox_cxcywh_to_xyxy(bbox_pred) + det_bboxes[:, 0::2] = det_bboxes[:, 0::2] * img_shape[1] + det_bboxes[:, 1::2] = det_bboxes[:, 1::2] * img_shape[0] + det_bboxes[:, 0::2].clamp_(min=0, max=img_shape[1]) + det_bboxes[:, 1::2].clamp_(min=0, max=img_shape[0]) + if rescale: + det_bboxes /= det_bboxes.new_tensor(scale_factor) + det_bboxes = torch.cat((det_bboxes, scores.unsqueeze(1)), -1) + + return det_bboxes, det_labels + + def simple_test_bboxes(self, feats, img_metas, rescale=False): + """Test det bboxes without test-time augmentation. + + Args: + feats (tuple[torch.Tensor]): Multi-level features from the + upstream network, each is a 4D-tensor. + img_metas (list[dict]): List of image information. + rescale (bool, optional): Whether to rescale the results. + Defaults to False. + + Returns: + list[tuple[Tensor, Tensor]]: Each item in result_list is 2-tuple. + The first item is ``bboxes`` with shape (n, 5), + where 5 represent (tl_x, tl_y, br_x, br_y, score). + The shape of the second tensor in the tuple is ``labels`` + with shape (n,) + """ + # forward of this head requires img_metas + outs = self.forward(feats, img_metas) + results_list = self.get_bboxes(*outs, img_metas, rescale=rescale) + return results_list + + def forward_onnx(self, feats, img_metas): + """Forward function for exporting to ONNX. + + Over-write `forward` because: `masks` is directly created with + zero (valid position tag) and has the same spatial size as `x`. + Thus the construction of `masks` is different from that in `forward`. + + Args: + feats (tuple[Tensor]): Features from the upstream network, each is + a 4D-tensor. + img_metas (list[dict]): List of image information. + + Returns: + tuple[list[Tensor], list[Tensor]]: Outputs for all scale levels. + + - all_cls_scores_list (list[Tensor]): Classification scores \ + for each scale level. Each is a 4D-tensor with shape \ + [nb_dec, bs, num_query, cls_out_channels]. Note \ + `cls_out_channels` should includes background. + - all_bbox_preds_list (list[Tensor]): Sigmoid regression \ + outputs for each scale level. Each is a 4D-tensor with \ + normalized coordinate format (cx, cy, w, h) and shape \ + [nb_dec, bs, num_query, 4]. + """ + num_levels = len(feats) + img_metas_list = [img_metas for _ in range(num_levels)] + return multi_apply(self.forward_single_onnx, feats, img_metas_list) + + def forward_single_onnx(self, x, img_metas): + """"Forward function for a single feature level with ONNX exportation. + + Args: + x (Tensor): Input feature from backbone's single stage, shape + [bs, c, h, w]. + img_metas (list[dict]): List of image information. + + Returns: + all_cls_scores (Tensor): Outputs from the classification head, + shape [nb_dec, bs, num_query, cls_out_channels]. Note + cls_out_channels should includes background. + all_bbox_preds (Tensor): Sigmoid outputs from the regression + head with normalized coordinate format (cx, cy, w, h). + Shape [nb_dec, bs, num_query, 4]. + """ + # Note `img_shape` is not dynamically traceable to ONNX, + # since the related augmentation was done with numpy under + # CPU. Thus `masks` is directly created with zeros (valid tag) + # and the same spatial shape as `x`. + # The difference between torch and exported ONNX model may be + # ignored, since the same performance is achieved (e.g. + # 40.1 vs 40.1 for DETR) + batch_size = x.size(0) + h, w = x.size()[-2:] + masks = x.new_zeros((batch_size, h, w)) # [B,h,w] + + x = self.input_proj(x) + # interpolate masks to have the same spatial shape with x + masks = F.interpolate( + masks.unsqueeze(1), size=x.shape[-2:]).to(torch.bool).squeeze(1) + pos_embed = self.positional_encoding(masks) + outs_dec, _ = self.transformer(x, masks, self.query_embedding.weight, + pos_embed) + + all_cls_scores = self.fc_cls(outs_dec) + all_bbox_preds = self.fc_reg(self.activate( + self.reg_ffn(outs_dec))).sigmoid() + return all_cls_scores, all_bbox_preds + + def onnx_export(self, all_cls_scores_list, all_bbox_preds_list, img_metas): + """Transform network outputs into bbox predictions, with ONNX + exportation. + + Args: + all_cls_scores_list (list[Tensor]): Classification outputs + for each feature level. Each is a 4D-tensor with shape + [nb_dec, bs, num_query, cls_out_channels]. + all_bbox_preds_list (list[Tensor]): Sigmoid regression + outputs for each feature level. Each is a 4D-tensor with + normalized coordinate format (cx, cy, w, h) and shape + [nb_dec, bs, num_query, 4]. + img_metas (list[dict]): Meta information of each image. + + Returns: + tuple[Tensor, Tensor]: dets of shape [N, num_det, 5] + and class labels of shape [N, num_det]. + """ + assert len(img_metas) == 1, \ + 'Only support one input image while in exporting to ONNX' + + cls_scores = all_cls_scores_list[-1][-1] + bbox_preds = all_bbox_preds_list[-1][-1] + + # Note `img_shape` is not dynamically traceable to ONNX, + # here `img_shape_for_onnx` (padded shape of image tensor) + # is used. + img_shape = img_metas[0]['img_shape_for_onnx'] + max_per_img = self.test_cfg.get('max_per_img', self.num_query) + batch_size = cls_scores.size(0) + # `batch_index_offset` is used for the gather of concatenated tensor + batch_index_offset = torch.arange(batch_size).to( + cls_scores.device) * max_per_img + batch_index_offset = batch_index_offset.unsqueeze(1).expand( + batch_size, max_per_img) + + # supports dynamical batch inference + if self.loss_cls.use_sigmoid: + cls_scores = cls_scores.sigmoid() + scores, indexes = cls_scores.view(batch_size, -1).topk( + max_per_img, dim=1) + det_labels = indexes % self.num_classes + bbox_index = indexes // self.num_classes + bbox_index = (bbox_index + batch_index_offset).view(-1) + bbox_preds = bbox_preds.view(-1, 4)[bbox_index] + bbox_preds = bbox_preds.view(batch_size, -1, 4) + else: + scores, det_labels = F.softmax( + cls_scores, dim=-1)[..., :-1].max(-1) + scores, bbox_index = scores.topk(max_per_img, dim=1) + bbox_index = (bbox_index + batch_index_offset).view(-1) + bbox_preds = bbox_preds.view(-1, 4)[bbox_index] + det_labels = det_labels.view(-1)[bbox_index] + bbox_preds = bbox_preds.view(batch_size, -1, 4) + det_labels = det_labels.view(batch_size, -1) + + det_bboxes = bbox_cxcywh_to_xyxy(bbox_preds) + # use `img_shape_tensor` for dynamically exporting to ONNX + img_shape_tensor = img_shape.flip(0).repeat(2) # [w,h,w,h] + img_shape_tensor = img_shape_tensor.unsqueeze(0).unsqueeze(0).expand( + batch_size, det_bboxes.size(1), 4) + det_bboxes = det_bboxes * img_shape_tensor + # dynamically clip bboxes + x1, y1, x2, y2 = det_bboxes.split((1, 1, 1, 1), dim=-1) + from mmdet.core.export import dynamic_clip_for_onnx + x1, y1, x2, y2 = dynamic_clip_for_onnx(x1, y1, x2, y2, img_shape) + det_bboxes = torch.cat([x1, y1, x2, y2], dim=-1) + det_bboxes = torch.cat((det_bboxes, scores.unsqueeze(-1)), -1) + + return det_bboxes, det_labels diff --git a/detection/mmdet_custom/models/dense_heads/dino_head.py b/detection/mmdet_custom/models/dense_heads/dino_head.py new file mode 100644 index 0000000..7cc1eae --- /dev/null +++ b/detection/mmdet_custom/models/dense_heads/dino_head.py @@ -0,0 +1,365 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +import torch.nn as nn +import torch.nn.functional as F + +from mmdet.core import (bbox_cxcywh_to_xyxy, bbox_xyxy_to_cxcywh, multi_apply, + reduce_mean) +from ..utils import build_dn_generator +from mmdet.models.utils.transformer import inverse_sigmoid +from mmdet.models.builder import HEADS +from .deformable_detr_head import DeformableDETRHead +from mmcv.runner import force_fp32 + + +@HEADS.register_module() +class DINOHead(DeformableDETRHead): + + def __init__(self, *args, dn_cfg=None, **kwargs): + super(DINOHead, self).__init__(*args, **kwargs) + self._init_layers() + self.init_denoising(dn_cfg) + assert self.as_two_stage, \ + 'as_two_stage must be True for DINO' + assert self.with_box_refine, \ + 'with_box_refine must be True for DINO' + + def _init_layers(self): + super()._init_layers() + # NOTE The original repo of DINO set the num_embeddings 92 for coco, + # 91 (0~90) of which represents target classes and the 92 (91) + # indicates [Unknown] class. However, the embedding of unknown class + # is not used in the original DINO + self.label_embedding = nn.Embedding(self.cls_out_channels, + self.embed_dims) + + def init_denoising(self, dn_cfg): + if dn_cfg is not None: + dn_cfg['num_classes'] = self.num_classes + dn_cfg['num_queries'] = self.num_query + dn_cfg['hidden_dim'] = self.embed_dims + self.dn_generator = build_dn_generator(dn_cfg) + + def forward_train(self, + x, + img_metas, + gt_bboxes, + gt_labels=None, + gt_bboxes_ignore=None, + proposal_cfg=None, + **kwargs): + assert proposal_cfg is None, '"proposal_cfg" must be None' + assert self.dn_generator is not None, '"dn_cfg" must be set' + dn_label_query, dn_bbox_query, attn_mask, dn_meta = \ + self.dn_generator(gt_bboxes, gt_labels, + self.label_embedding, img_metas) + outs = self(x, img_metas, dn_label_query, dn_bbox_query, attn_mask) + if gt_labels is None: + loss_inputs = outs + (gt_bboxes, img_metas, dn_meta) + else: + loss_inputs = outs + (gt_bboxes, gt_labels, img_metas, dn_meta) + losses = self.loss(*loss_inputs, gt_bboxes_ignore=gt_bboxes_ignore) + return losses + + def forward(self, + mlvl_feats, + img_metas, + dn_label_query=None, + dn_bbox_query=None, + attn_mask=None): + batch_size = mlvl_feats[0].size(0) + input_img_h, input_img_w = img_metas[0]['batch_input_shape'] + img_masks = mlvl_feats[0].new_ones( + (batch_size, input_img_h, input_img_w)) + for img_id in range(batch_size): + if img_id >= len(img_metas): img_id = 0 + img_h, img_w, _ = img_metas[img_id]['img_shape'] + img_masks[img_id, :img_h, :img_w] = 0 + + mlvl_masks = [] + mlvl_positional_encodings = [] + for feat in mlvl_feats: + mlvl_masks.append( + F.interpolate( + img_masks[None], + size=feat.shape[-2:]).to(torch.bool).squeeze(0)) + mlvl_positional_encodings.append( + self.positional_encoding(mlvl_masks[-1])) + + query_embeds = None + hs, inter_references, topk_score, topk_anchor = \ + self.transformer( + mlvl_feats, + mlvl_masks, + query_embeds, + mlvl_positional_encodings, + dn_label_query, + dn_bbox_query, + attn_mask, + reg_branches=self.reg_branches if self.with_box_refine else None, # noqa:E501 + cls_branches=self.cls_branches if self.as_two_stage else None # noqa:E501 + ) + hs = hs.permute(0, 2, 1, 3) + + if dn_label_query is not None and dn_label_query.size(1) == 0: + # NOTE: If there is no target in the image, the parameters of + # label_embedding won't be used in producing loss, which raises + # RuntimeError when using distributed mode. + hs[0] += self.label_embedding.weight[0, 0] * 0.0 + + outputs_classes = [] + outputs_coords = [] + + for lvl in range(hs.shape[0]): + reference = inter_references[lvl] + reference = inverse_sigmoid(reference, eps=1e-3) + outputs_class = self.cls_branches[lvl](hs[lvl]) + tmp = self.reg_branches[lvl](hs[lvl]) + if reference.shape[-1] == 4: + tmp += reference + else: + assert reference.shape[-1] == 2 + tmp[..., :2] += reference + outputs_coord = tmp.sigmoid() + outputs_classes.append(outputs_class) + outputs_coords.append(outputs_coord) + + outputs_classes = torch.stack(outputs_classes) + outputs_coords = torch.stack(outputs_coords) + + return outputs_classes, outputs_coords, topk_score, topk_anchor + + @force_fp32(apply_to=('all_cls_scores', 'all_bbox_preds')) + def loss(self, + all_cls_scores, + all_bbox_preds, + enc_topk_scores, + enc_topk_anchors, + gt_bboxes_list, + gt_labels_list, + img_metas, + dn_meta=None, + gt_bboxes_ignore=None): + assert gt_bboxes_ignore is None, \ + f'{self.__class__.__name__} only supports ' \ + f'for gt_bboxes_ignore setting to None.' + + loss_dict = dict() + + # extract denoising and matching part of outputs + all_cls_scores, all_bbox_preds, dn_cls_scores, dn_bbox_preds = \ + self.extract_dn_outputs(all_cls_scores, all_bbox_preds, dn_meta) + + if enc_topk_scores is not None: + # calculate loss from encode feature maps + # NOTE The DeformDETR calculate binary cls loss + # for all encoder embeddings, while DINO calculate + # multi-class loss for topk embeddings. + enc_loss_cls, enc_losses_bbox, enc_losses_iou = \ + self.loss_single(enc_topk_scores, enc_topk_anchors, + gt_bboxes_list, gt_labels_list, + img_metas, gt_bboxes_ignore) + + # collate loss from encode feature maps + loss_dict['interm_loss_cls'] = enc_loss_cls + loss_dict['interm_loss_bbox'] = enc_losses_bbox + loss_dict['interm_loss_iou'] = enc_losses_iou + + # calculate loss from all decoder layers + num_dec_layers = len(all_cls_scores) + all_gt_bboxes_list = [gt_bboxes_list for _ in range(num_dec_layers)] + all_gt_labels_list = [gt_labels_list for _ in range(num_dec_layers)] + all_gt_bboxes_ignore_list = [ + gt_bboxes_ignore for _ in range(num_dec_layers) + ] + img_metas_list = [img_metas for _ in range(num_dec_layers)] + losses_cls, losses_bbox, losses_iou = multi_apply( + self.loss_single, all_cls_scores, all_bbox_preds, + all_gt_bboxes_list, all_gt_labels_list, img_metas_list, + all_gt_bboxes_ignore_list) + + # collate loss from the last decoder layer + loss_dict['loss_cls'] = losses_cls[-1] + loss_dict['loss_bbox'] = losses_bbox[-1] + loss_dict['loss_iou'] = losses_iou[-1] + + # collate loss from other decoder layers + num_dec_layer = 0 + for loss_cls_i, loss_bbox_i, loss_iou_i in zip(losses_cls[:-1], + losses_bbox[:-1], + losses_iou[:-1]): + loss_dict[f'd{num_dec_layer}.loss_cls'] = loss_cls_i + loss_dict[f'd{num_dec_layer}.loss_bbox'] = loss_bbox_i + loss_dict[f'd{num_dec_layer}.loss_iou'] = loss_iou_i + num_dec_layer += 1 + + if dn_cls_scores is not None: + # calculate denoising loss from all decoder layers + dn_meta = [dn_meta for _ in img_metas] + dn_losses_cls, dn_losses_bbox, dn_losses_iou = self.loss_dn( + dn_cls_scores, dn_bbox_preds, gt_bboxes_list, gt_labels_list, + img_metas, dn_meta) + + # collate denoising loss + loss_dict['dn_loss_cls'] = dn_losses_cls[-1] + loss_dict['dn_loss_bbox'] = dn_losses_bbox[-1] + loss_dict['dn_loss_iou'] = dn_losses_iou[-1] + num_dec_layer = 0 + for loss_cls_i, loss_bbox_i, loss_iou_i in zip( + dn_losses_cls[:-1], dn_losses_bbox[:-1], + dn_losses_iou[:-1]): + loss_dict[f'd{num_dec_layer}.dn_loss_cls'] = loss_cls_i + loss_dict[f'd{num_dec_layer}.dn_loss_bbox'] = loss_bbox_i + loss_dict[f'd{num_dec_layer}.dn_loss_iou'] = loss_iou_i + num_dec_layer += 1 + + return loss_dict + + def loss_dn(self, dn_cls_scores, dn_bbox_preds, gt_bboxes_list, + gt_labels_list, img_metas, dn_meta): + num_dec_layers = len(dn_cls_scores) + all_gt_bboxes_list = [gt_bboxes_list for _ in range(num_dec_layers)] + all_gt_labels_list = [gt_labels_list for _ in range(num_dec_layers)] + img_metas_list = [img_metas for _ in range(num_dec_layers)] + dn_meta_list = [dn_meta for _ in range(num_dec_layers)] + return multi_apply(self.loss_dn_single, dn_cls_scores, dn_bbox_preds, + all_gt_bboxes_list, all_gt_labels_list, + img_metas_list, dn_meta_list) + + def loss_dn_single(self, dn_cls_scores, dn_bbox_preds, gt_bboxes_list, + gt_labels_list, img_metas, dn_meta): + num_imgs = dn_cls_scores.size(0) + bbox_preds_list = [dn_bbox_preds[i] for i in range(num_imgs)] + cls_reg_targets = self.get_dn_target(bbox_preds_list, gt_bboxes_list, + gt_labels_list, img_metas, + dn_meta) + (labels_list, label_weights_list, bbox_targets_list, bbox_weights_list, + num_total_pos, num_total_neg) = cls_reg_targets + labels = torch.cat(labels_list, 0) + label_weights = torch.cat(label_weights_list, 0) + bbox_targets = torch.cat(bbox_targets_list, 0) + bbox_weights = torch.cat(bbox_weights_list, 0) + + # classification loss + cls_scores = dn_cls_scores.reshape(-1, self.cls_out_channels) + # construct weighted avg_factor to match with the official DETR repo + cls_avg_factor = \ + num_total_pos * 1.0 + num_total_neg * self.bg_cls_weight + if self.sync_cls_avg_factor: + cls_avg_factor = reduce_mean( + cls_scores.new_tensor([cls_avg_factor])) + cls_avg_factor = max(cls_avg_factor, 1) + + if len(cls_scores) > 0: + loss_cls = self.loss_cls( + cls_scores, labels, label_weights, avg_factor=cls_avg_factor) + else: + loss_cls = torch.zeros( # TODO: How to better return zero loss + 1, + dtype=cls_scores.dtype, + device=cls_scores.device) + + # Compute the average number of gt boxes across all gpus, for + # normalization purposes + num_total_pos = loss_cls.new_tensor([num_total_pos]) + num_total_pos = torch.clamp(reduce_mean(num_total_pos), min=1).item() + + # construct factors used for rescale bboxes + factors = [] + for img_meta, bbox_pred in zip(img_metas, dn_bbox_preds): + img_h, img_w, _ = img_meta['img_shape'] + factor = bbox_pred.new_tensor([img_w, img_h, img_w, + img_h]).unsqueeze(0).repeat( + bbox_pred.size(0), 1) + factors.append(factor) + factors = torch.cat(factors, 0) + + # DETR regress the relative position of boxes (cxcywh) in the image, + # thus the learning target is normalized by the image size. So here + # we need to re-scale them for calculating IoU loss + bbox_preds = dn_bbox_preds.reshape(-1, 4) + bboxes = bbox_cxcywh_to_xyxy(bbox_preds) * factors + bboxes_gt = bbox_cxcywh_to_xyxy(bbox_targets) * factors + + # regression IoU loss, defaultly GIoU loss + loss_iou = self.loss_iou( + bboxes, bboxes_gt, bbox_weights, avg_factor=num_total_pos) + + # regression L1 loss + loss_bbox = self.loss_bbox( + bbox_preds, bbox_targets, bbox_weights, avg_factor=num_total_pos) + return loss_cls, loss_bbox, loss_iou + + def get_dn_target(self, dn_bbox_preds_list, gt_bboxes_list, gt_labels_list, + img_metas, dn_meta): + (labels_list, label_weights_list, bbox_targets_list, bbox_weights_list, + pos_inds_list, + neg_inds_list) = multi_apply(self._get_dn_target_single, + dn_bbox_preds_list, gt_bboxes_list, + gt_labels_list, img_metas, dn_meta) + num_total_pos = sum((inds.numel() for inds in pos_inds_list)) + num_total_neg = sum((inds.numel() for inds in neg_inds_list)) + return (labels_list, label_weights_list, bbox_targets_list, + bbox_weights_list, num_total_pos, num_total_neg) + + def _get_dn_target_single(self, dn_bbox_pred, gt_bboxes, gt_labels, + img_meta, dn_meta): + num_groups = dn_meta['num_dn_group'] + pad_size = dn_meta['pad_size'] + assert pad_size % num_groups == 0 + single_pad = pad_size // num_groups + num_bboxes = dn_bbox_pred.size(0) + + if len(gt_labels) > 0: + t = torch.range(0, len(gt_labels) - 1).long().cuda() + t = t.unsqueeze(0).repeat(num_groups, 1) + pos_assigned_gt_inds = t.flatten() + pos_inds = (torch.tensor(range(num_groups)) * + single_pad).long().cuda().unsqueeze(1) + t + pos_inds = pos_inds.flatten() + else: + pos_inds = pos_assigned_gt_inds = torch.tensor([]).long().cuda() + neg_inds = pos_inds + single_pad // 2 + + # label targets + labels = gt_bboxes.new_full((num_bboxes, ), + self.num_classes, + dtype=torch.long) + labels[pos_inds] = gt_labels[pos_assigned_gt_inds] + label_weights = gt_bboxes.new_ones(num_bboxes) + + # bbox targets + bbox_targets = torch.zeros_like(dn_bbox_pred) + bbox_weights = torch.zeros_like(dn_bbox_pred) + bbox_weights[pos_inds] = 1.0 + img_h, img_w, _ = img_meta['img_shape'] + + # DETR regress the relative position of boxes (cxcywh) in the image. + # Thus the learning target should be normalized by the image size, also + # the box format should be converted from defaultly x1y1x2y2 to cxcywh. + factor = dn_bbox_pred.new_tensor([img_w, img_h, img_w, + img_h]).unsqueeze(0) + gt_bboxes_normalized = gt_bboxes / factor + gt_bboxes_targets = bbox_xyxy_to_cxcywh(gt_bboxes_normalized) + bbox_targets[pos_inds] = gt_bboxes_targets.repeat([num_groups, 1]) + + return (labels, label_weights, bbox_targets, bbox_weights, pos_inds, + neg_inds) + + @staticmethod + def extract_dn_outputs(all_cls_scores, all_bbox_preds, dn_meta): + # if dn_meta and dn_meta['pad_size'] > 0: + if dn_meta is not None: + denoising_cls_scores = all_cls_scores[:, :, : + dn_meta['pad_size'], :] + denoising_bbox_preds = all_bbox_preds[:, :, : + dn_meta['pad_size'], :] + matching_cls_scores = all_cls_scores[:, :, dn_meta['pad_size']:, :] + matching_bbox_preds = all_bbox_preds[:, :, dn_meta['pad_size']:, :] + else: + denoising_cls_scores = None + denoising_bbox_preds = None + matching_cls_scores = all_cls_scores + matching_bbox_preds = all_bbox_preds + return (matching_cls_scores, matching_bbox_preds, denoising_cls_scores, + denoising_bbox_preds) diff --git a/detection/mmdet_custom/models/dense_heads/mask_rcnn.py b/detection/mmdet_custom/models/dense_heads/mask_rcnn.py new file mode 100644 index 0000000..a05384a --- /dev/null +++ b/detection/mmdet_custom/models/dense_heads/mask_rcnn.py @@ -0,0 +1,27 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from mmdet.models.builder import DETECTORS +from .two_stage import TwoStageDetector + + +@DETECTORS.register_module() +class MaskRCNN_(TwoStageDetector): + """Implementation of `Mask R-CNN `_""" + + def __init__(self, + backbone, + rpn_head, + roi_head, + train_cfg, + test_cfg, + neck=None, + pretrained=None, + init_cfg=None): + super(MaskRCNN_, self).__init__( + backbone=backbone, + neck=neck, + rpn_head=rpn_head, + roi_head=roi_head, + train_cfg=train_cfg, + test_cfg=test_cfg, + pretrained=pretrained, + init_cfg=init_cfg) diff --git a/detection/mmdet_custom/models/dense_heads/msda.py b/detection/mmdet_custom/models/dense_heads/msda.py new file mode 100644 index 0000000..41ea272 --- /dev/null +++ b/detection/mmdet_custom/models/dense_heads/msda.py @@ -0,0 +1,369 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + +import torch +from torch.cuda.amp import custom_bwd, custom_fwd +from torch.autograd.function import Function, once_differentiable +from mmcv.utils import ext_loader +ext_module = ext_loader.load_ext( + '_ext', ['ms_deform_attn_backward', 'ms_deform_attn_forward']) + +class MultiScaleDeformableAttnFunction_fp16(Function): + + @staticmethod + @custom_fwd(cast_inputs=torch.float16) + def forward(ctx, value, value_spatial_shapes, value_level_start_index, + sampling_locations, attention_weights, im2col_step): + """GPU version of multi-scale deformable attention. + + Args: + value (Tensor): The value has shape + (bs, num_keys, mum_heads, embed_dims//num_heads) + value_spatial_shapes (Tensor): Spatial shape of + each feature map, has shape (num_levels, 2), + last dimension 2 represent (h, w) + sampling_locations (Tensor): The location of sampling points, + has shape + (bs ,num_queries, num_heads, num_levels, num_points, 2), + the last dimension 2 represent (x, y). + attention_weights (Tensor): The weight of sampling points used + when calculate the attention, has shape + (bs ,num_queries, num_heads, num_levels, num_points), + im2col_step (Tensor): The step used in image to column. + + Returns: + Tensor: has shape (bs, num_queries, embed_dims) + """ + ctx.im2col_step = im2col_step + output = ext_module.ms_deform_attn_forward( + value, + value_spatial_shapes, + value_level_start_index, + sampling_locations, + attention_weights, + im2col_step=ctx.im2col_step) + ctx.save_for_backward(value, value_spatial_shapes, + value_level_start_index, sampling_locations, + attention_weights) + return output + + @staticmethod + @once_differentiable + @custom_bwd + def backward(ctx, grad_output): + """GPU version of backward function. + + Args: + grad_output (Tensor): Gradient + of output tensor of forward. + + Returns: + Tuple[Tensor]: Gradient + of input tensors in forward. + """ + value, value_spatial_shapes, value_level_start_index, \ + sampling_locations, attention_weights = ctx.saved_tensors + grad_value = torch.zeros_like(value) + grad_sampling_loc = torch.zeros_like(sampling_locations) + grad_attn_weight = torch.zeros_like(attention_weights) + + ext_module.ms_deform_attn_backward( + value, + value_spatial_shapes, + value_level_start_index, + sampling_locations, + attention_weights, + grad_output.contiguous(), + grad_value, + grad_sampling_loc, + grad_attn_weight, + im2col_step=ctx.im2col_step) + + return grad_value, None, None, \ + grad_sampling_loc, grad_attn_weight, None + + + +shm_size_dict = { + "8.0": 163000, + "8.6": 99000, + "8.7": 163000, + "8.9": 99000, + "9.0": 227000, + "7.5": 64000, + "7.0": 96000, +} + +cuda_capability = f"{torch.cuda.get_device_properties(0).major}.{torch.cuda.get_device_properties(0).minor}" + +if cuda_capability not in shm_size_dict: + raise NotImplementedError + +shm_size_cap = shm_size_dict[cuda_capability] + + +class MultiScaleDeformableAttnFunction_fp32_old(Function): + + @staticmethod + @custom_fwd(cast_inputs=torch.float32) + def forward(ctx, value, value_spatial_shapes, value_level_start_index, + sampling_locations, attention_weights, im2col_step): + """GPU version of multi-scale deformable attention. + + Args: + value (Tensor): The value has shape + (bs, num_keys, mum_heads, embed_dims//num_heads) + value_spatial_shapes (Tensor): Spatial shape of + each feature map, has shape (num_levels, 2), + last dimension 2 represent (h, w) + sampling_locations (Tensor): The location of sampling points, + has shape + (bs ,num_queries, num_heads, num_levels, num_points, 2), + the last dimension 2 represent (x, y). + attention_weights (Tensor): The weight of sampling points used + when calculate the attention, has shape + (bs ,num_queries, num_heads, num_levels, num_points), + im2col_step (Tensor): The step used in image to column. + + Returns: + Tensor: has shape (bs, num_queries, embed_dims) + """ + + ctx.im2col_step = im2col_step + output = ext_module.ms_deform_attn_forward( + value, + value_spatial_shapes, + value_level_start_index, + sampling_locations, + attention_weights, + im2col_step=ctx.im2col_step) + ctx.save_for_backward(value, value_spatial_shapes, + value_level_start_index, sampling_locations, + attention_weights) + return output + + @staticmethod + @once_differentiable + @custom_bwd + def backward(ctx, grad_output): + """GPU version of backward function. + + Args: + grad_output (Tensor): Gradient + of output tensor of forward. + + Returns: + Tuple[Tensor]: Gradient + of input tensors in forward. + """ + value, value_spatial_shapes, value_level_start_index, \ + sampling_locations, attention_weights = ctx.saved_tensors + grad_value = torch.zeros_like(value) + grad_sampling_loc = torch.zeros_like(sampling_locations) + grad_attn_weight = torch.zeros_like(attention_weights) + + ext_module.ms_deform_attn_backward( + value, + value_spatial_shapes, + value_level_start_index, + sampling_locations, + attention_weights, + grad_output.contiguous(), + grad_value, + grad_sampling_loc, + grad_attn_weight, + im2col_step=ctx.im2col_step) + + return grad_value, None, None, \ + grad_sampling_loc, grad_attn_weight, None + + +# Copyright (c) OpenMMLab. All rights reserved. +import math +import warnings + +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.autograd.function import Function, once_differentiable + +from mmcv import deprecated_api_warning +from mmcv.cnn import constant_init, xavier_init +from mmcv.cnn.bricks.registry import ATTENTION +from mmcv.runner import BaseModule + +ext_module = ext_loader.load_ext( + '_ext', ['ms_deform_attn_backward', 'ms_deform_attn_forward']) +import functools +import time +from collections import defaultdict +import torch +from mmcv.ops import MultiScaleDeformableAttention +@ATTENTION.register_module() +class FlashMultiScaleDeformableAttention(MultiScaleDeformableAttention): + """An attention module used in Deformable-Detr. + + `Deformable DETR: Deformable Transformers for End-to-End Object Detection. + `_. + + Args: + embed_dims (int): The embedding dimension of Attention. + Default: 256. + num_heads (int): Parallel attention heads. Default: 64. + num_levels (int): The number of feature map used in + Attention. Default: 4. + num_points (int): The number of sampling points for + each query in each head. Default: 4. + im2col_step (int): The step used in image_to_column. + Default: 64. + dropout (float): A Dropout layer on `inp_identity`. + Default: 0.1. + batch_first (bool): Key, Query and Value are shape of + (batch, n, embed_dim) + or (n, batch, embed_dim). Default to False. + norm_cfg (dict): Config dict for normalization layer. + Default: None. + init_cfg (obj:`mmcv.ConfigDict`): The Config for initialization. + Default: None. + """ + + def __init__(self, + use_flash=False, + use_softmax=True, + **kwargs +): + super().__init__(**kwargs) + + self.use_flash = use_flash + self.use_softmax = use_softmax + + @deprecated_api_warning({'residual': 'identity'}, + cls_name='FlashMultiScaleDeformableAttention') + # @run_time('ms_attention') + def forward(self, + query, + key=None, + value=None, + identity=None, + query_pos=None, + key_padding_mask=None, + reference_points=None, + spatial_shapes=None, + level_start_index=None, + **kwargs): + """Forward Function of MultiScaleDeformAttention. + + Args: + query (torch.Tensor): Query of Transformer with shape + (num_query, bs, embed_dims). + key (torch.Tensor): The key tensor with shape + `(num_key, bs, embed_dims)`. + value (torch.Tensor): The value tensor with shape + `(num_key, bs, embed_dims)`. + identity (torch.Tensor): The tensor used for addition, with the + same shape as `query`. Default None. If None, + `query` will be used. + query_pos (torch.Tensor): The positional encoding for `query`. + Default: None. + key_pos (torch.Tensor): The positional encoding for `key`. Default + None. + reference_points (torch.Tensor): The normalized reference + points with shape (bs, num_query, num_levels, 2), + all elements is range in [0, 1], top-left (0,0), + bottom-right (1, 1), including padding area. + or (N, Length_{query}, num_levels, 4), add + additional two dimensions is (w, h) to + form reference boxes. + key_padding_mask (torch.Tensor): ByteTensor for `query`, with + shape [bs, num_key]. + spatial_shapes (torch.Tensor): Spatial shape of features in + different levels. With shape (num_levels, 2), + last dimension represents (h, w). + level_start_index (torch.Tensor): The start index of each level. + A tensor has shape ``(num_levels, )`` and can be represented + as [0, h_0*w_0, h_0*w_0+h_1*w_1, ...]. + + Returns: + torch.Tensor: forwarded results with shape + [num_query, bs, embed_dims]. + """ + + if value is None: + value = query + + if identity is None: + identity = query + if query_pos is not None: + query = query + query_pos + if not self.batch_first: + # change to (bs, num_query ,embed_dims) + query = query.permute(1, 0, 2) + value = value.permute(1, 0, 2) + + bs, num_query, _ = query.shape + bs, num_value, _ = value.shape + assert (spatial_shapes[:, 0] * spatial_shapes[:, 1]).sum() == num_value + + value = self.value_proj(value) + if key_padding_mask is not None: + value = value.masked_fill(key_padding_mask[..., None], 0.0) + value = value.view(bs, num_value, self.num_heads, -1) + sampling_offsets = self.sampling_offsets(query).view( + bs, num_query, self.num_heads, self.num_levels, self.num_points, 2) + attention_weights = self.attention_weights(query).view( + bs, num_query, self.num_heads, self.num_levels * self.num_points) + + if not self.use_flash: + if self.use_softmax: + attention_weights = attention_weights.softmax(-1) + attention_weights = attention_weights.view(bs, num_query, + self.num_heads, + self.num_levels, + self.num_points) + + else: + attention_weights = attention_weights.view(bs, num_query, + self.num_heads, + self.num_levels, + self.num_points, 1) + + if reference_points.shape[-1] == 2: + offset_normalizer = torch.stack( + [spatial_shapes[..., 1], spatial_shapes[..., 0]], -1) + sampling_locations = reference_points[:, :, None, :, None, :] \ + + sampling_offsets \ + / offset_normalizer[None, None, None, :, None, :] + elif reference_points.shape[-1] == 4: + sampling_locations = reference_points[:, :, None, :, None, :2] \ + + sampling_offsets / self.num_points \ + * reference_points[:, :, None, :, None, 2:] \ + * 0.5 + else: + raise ValueError( + f'Last dim of reference_points must be' + f' 2 or 4, but get {reference_points.shape[-1]} instead.') + sampling_locations = sampling_locations.to(sampling_offsets.dtype) + if torch.cuda.is_available() and value.is_cuda: + if self.use_flash: + assert False + else: + MultiScaleDeformableAttnFunction = MultiScaleDeformableAttnFunction_fp32_old + + output = MultiScaleDeformableAttnFunction.apply( + value, spatial_shapes, level_start_index, sampling_locations, + attention_weights, self.im2col_step) + + else: + output = multi_scale_deformable_attn_pytorch( + value, spatial_shapes, sampling_locations, attention_weights) + + output = self.output_proj(output) + + if not self.batch_first: + # (num_query, bs ,embed_dims) + output = output.permute(1, 0, 2) + + return self.dropout(output) + identity diff --git a/detection/mmdet_custom/models/dense_heads/two_stage.py b/detection/mmdet_custom/models/dense_heads/two_stage.py new file mode 100644 index 0000000..ead405d --- /dev/null +++ b/detection/mmdet_custom/models/dense_heads/two_stage.py @@ -0,0 +1,225 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import warnings + +import torch + +from mmdet.models.builder import DETECTORS, build_backbone, build_head, build_neck +from mmdet.models.detectors.base import BaseDetector +from mmcv.runner import BaseModule, force_fp32, auto_fp16 +import functools +import time +from collections import defaultdict +import torch + + + +# DETECTORS.register_module() +class TwoStageDetector(BaseDetector): + """Base class for two-stage detectors. + + Two-stage detectors typically consisting of a region proposal network and a + task-specific regression head. + """ + + def __init__(self, + backbone, + neck=None, + rpn_head=None, + roi_head=None, + train_cfg=None, + test_cfg=None, + pretrained=None, + init_cfg=None): + super(TwoStageDetector, self).__init__(init_cfg) + if pretrained: + warnings.warn('DeprecationWarning: pretrained is deprecated, ' + 'please use "init_cfg" instead') + backbone.pretrained = pretrained + self.backbone = build_backbone(backbone) + + if neck is not None: + self.neck = build_neck(neck) + + if rpn_head is not None: + rpn_train_cfg = train_cfg.rpn if train_cfg is not None else None + rpn_head_ = rpn_head.copy() + rpn_head_.update(train_cfg=rpn_train_cfg, test_cfg=test_cfg.rpn) + self.rpn_head = build_head(rpn_head_) + + if roi_head is not None: + # update train and test cfg here for now + # TODO: refactor assigner & sampler + rcnn_train_cfg = train_cfg.rcnn if train_cfg is not None else None + roi_head.update(train_cfg=rcnn_train_cfg) + roi_head.update(test_cfg=test_cfg.rcnn) + roi_head.pretrained = pretrained + self.roi_head = build_head(roi_head) + + self.train_cfg = train_cfg + self.test_cfg = test_cfg + + @property + def with_rpn(self): + """bool: whether the detector has RPN""" + return hasattr(self, 'rpn_head') and self.rpn_head is not None + + @property + def with_roi_head(self): + """bool: whether the detector has a RoI head""" + return hasattr(self, 'roi_head') and self.roi_head is not None + + def use_backbone(self, img): + return self.backbone(img) + + # @auto_fp16(apply_to=('img',)) + def use_neck(self, img): + # x = self.neck([each.float() for each in img]) + return self.neck(img) + + def extract_feat(self, img): + """Directly extract features from the backbone+neck.""" + x = self.use_backbone(img) + if self.with_neck: + x = self.use_neck(x) + return x + + def forward_dummy(self, img): + """Used for computing network flops. + + See `mmdetection/tools/analysis_tools/get_flops.py` + """ + outs = () + # backbone + x = self.extract_feat(img) + # rpn + if self.with_rpn: + rpn_outs = self.rpn_head(x) + outs = outs + (rpn_outs, ) + proposals = torch.randn(1000, 4).to(img.device) + # roi_head + roi_outs = self.roi_head.forward_dummy(x, proposals) + outs = outs + (roi_outs, ) + return outs + + def forward_train(self, + img, + img_metas, + gt_bboxes, + gt_labels, + gt_bboxes_ignore=None, + gt_masks=None, + proposals=None, + **kwargs): + """ + Args: + img (Tensor): of shape (N, C, H, W) encoding input images. + Typically these should be mean centered and std scaled. + + img_metas (list[dict]): list of image info dict where each dict + has: 'img_shape', 'scale_factor', 'flip', and may also contain + 'filename', 'ori_shape', 'pad_shape', and 'img_norm_cfg'. + For details on the values of these keys see + `mmdet/datasets/pipelines/formatting.py:Collect`. + + gt_bboxes (list[Tensor]): Ground truth bboxes for each image with + shape (num_gts, 4) in [tl_x, tl_y, br_x, br_y] format. + + gt_labels (list[Tensor]): class indices corresponding to each box + + gt_bboxes_ignore (None | list[Tensor]): specify which bounding + boxes can be ignored when computing the loss. + + gt_masks (None | Tensor) : true segmentation masks for each box + used if the architecture supports a segmentation task. + + proposals : override rpn proposals with custom proposals. Use when + `with_rpn` is False. + + Returns: + dict[str, Tensor]: a dictionary of loss components + """ + x = self.extract_feat(img) + + losses = dict() + + # RPN forward and loss + if self.with_rpn: + proposal_cfg = self.train_cfg.get('rpn_proposal', + self.test_cfg.rpn) + rpn_losses, proposal_list = self.rpn_head.forward_train( + x, + img_metas, + gt_bboxes, + gt_labels=None, + gt_bboxes_ignore=gt_bboxes_ignore, + proposal_cfg=proposal_cfg, + **kwargs) + losses.update(rpn_losses) + else: + proposal_list = proposals + + roi_losses = self.roi_head.forward_train(x, img_metas, proposal_list, + gt_bboxes, gt_labels, + gt_bboxes_ignore, gt_masks, + **kwargs) + losses.update(roi_losses) + + return losses + + async def async_simple_test(self, + img, + img_meta, + proposals=None, + rescale=False): + """Async test without augmentation.""" + assert self.with_bbox, 'Bbox head must be implemented.' + x = self.extract_feat(img) + + if proposals is None: + proposal_list = await self.rpn_head.async_simple_test_rpn( + x, img_meta) + else: + proposal_list = proposals + + return await self.roi_head.async_simple_test( + x, proposal_list, img_meta, rescale=rescale) + + def simple_test(self, img, img_metas, proposals=None, rescale=False): + """Test without augmentation.""" + + assert self.with_bbox, 'Bbox head must be implemented.' + x = self.extract_feat(img) + if proposals is None: + proposal_list = self.rpn_head.simple_test_rpn(x, img_metas) + else: + proposal_list = proposals + + return self.roi_head.simple_test( + x, proposal_list, img_metas, rescale=rescale) + + def aug_test(self, imgs, img_metas, rescale=False): + """Test with augmentations. + + If rescale is False, then returned bboxes and masks will fit the scale + of imgs[0]. + """ + x = self.extract_feats(imgs) + proposal_list = self.rpn_head.aug_test_rpn(x, img_metas) + return self.roi_head.aug_test( + x, proposal_list, img_metas, rescale=rescale) + + def onnx_export(self, img, img_metas): + + img_shape = torch._shape_as_tensor(img)[2:] + img_metas[0]['img_shape_for_onnx'] = img_shape + x = self.extract_feat(img) + proposals = self.rpn_head.onnx_export(x, img_metas) + if hasattr(self.roi_head, 'onnx_export'): + return self.roi_head.onnx_export(x, proposals, img_metas) + else: + raise NotImplementedError( + f'{self.__class__.__name__} can not ' + f'be exported to ONNX. Please refer to the ' + f'list of supported models,' + f'https://mmdetection.readthedocs.io/en/latest/tutorials/pytorch2onnx.html#list-of-supported-models-exportable-to-onnx' # noqa E501 + ) diff --git a/detection/mmdet_custom/models/detectors/__init__.py b/detection/mmdet_custom/models/detectors/__init__.py new file mode 100644 index 0000000..cbe834d --- /dev/null +++ b/detection/mmdet_custom/models/detectors/__init__.py @@ -0,0 +1,9 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + +from .dino import DINO + +__all__ = ['DINO'] \ No newline at end of file diff --git a/detection/mmdet_custom/models/detectors/dino.py b/detection/mmdet_custom/models/detectors/dino.py new file mode 100644 index 0000000..f3567d1 --- /dev/null +++ b/detection/mmdet_custom/models/detectors/dino.py @@ -0,0 +1,10 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from mmdet.models.builder import DETECTORS +from mmdet.models.detectors.detr import DETR + + +@DETECTORS.register_module() +class DINO(DETR): + + def __init__(self, *args, **kwargs): + super(DETR, self).__init__(*args, **kwargs) \ No newline at end of file diff --git a/detection/mmdet_custom/models/necks/fpn.py b/detection/mmdet_custom/models/necks/fpn.py new file mode 100644 index 0000000..e0f5a5c --- /dev/null +++ b/detection/mmdet_custom/models/necks/fpn.py @@ -0,0 +1,207 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch.nn as nn +import torch.nn.functional as F +from mmcv.cnn import ConvModule +from mmcv.runner import BaseModule, auto_fp16 +from ..utils import ConvModule_Norm + +from mmdet.models.builder import NECKS + + +@NECKS.register_module() +class FPN_vitdet(BaseModule): + r"""Feature Pyramid Network. + + This is an implementation of paper `Feature Pyramid Networks for Object + Detection `_. + + Args: + in_channels (List[int]): Number of input channels per scale. + out_channels (int): Number of output channels (used at each scale) + num_outs (int): Number of output scales. + start_level (int): Index of the start input backbone level used to + build the feature pyramid. Default: 0. + end_level (int): Index of the end input backbone level (exclusive) to + build the feature pyramid. Default: -1, which means the last level. + add_extra_convs (bool | str): If bool, it decides whether to add conv + layers on top of the original feature maps. Default to False. + If True, it is equivalent to `add_extra_convs='on_input'`. + If str, it specifies the source feature map of the extra convs. + Only the following options are allowed + + - 'on_input': Last feat map of neck inputs (i.e. backbone feature). + - 'on_lateral': Last feature map after lateral convs. + - 'on_output': The last output feature map after fpn convs. + relu_before_extra_convs (bool): Whether to apply relu before the extra + conv. Default: False. + no_norm_on_lateral (bool): Whether to apply norm on lateral. + Default: False. + conv_cfg (dict): Config dict for convolution layer. Default: None. + norm_cfg (dict): Config dict for normalization layer. Default: None. + act_cfg (str): Config dict for activation layer in ConvModule. + Default: None. + upsample_cfg (dict): Config dict for interpolate layer. + Default: `dict(mode='nearest')` + init_cfg (dict or list[dict], optional): Initialization config dict. + + Example: + >>> import torch + >>> in_channels = [2, 3, 5, 7] + >>> scales = [340, 170, 84, 43] + >>> inputs = [torch.rand(1, c, s, s) + ... for c, s in zip(in_channels, scales)] + >>> self = FPN(in_channels, 11, len(in_channels)).eval() + >>> outputs = self.forward(inputs) + >>> for i in range(len(outputs)): + ... print(f'outputs[{i}].shape = {outputs[i].shape}') + outputs[0].shape = torch.Size([1, 11, 340, 340]) + outputs[1].shape = torch.Size([1, 11, 170, 170]) + outputs[2].shape = torch.Size([1, 11, 84, 84]) + outputs[3].shape = torch.Size([1, 11, 43, 43]) + """ + + def __init__(self, + in_channels, + out_channels, + num_outs, + start_level=0, + end_level=-1, + add_extra_convs=False, + relu_before_extra_convs=False, + no_norm_on_lateral=False, + conv_cfg=None, + norm_cfg=None, + act_cfg=None, + use_residual=True, + upsample_cfg=dict(mode='nearest'), + init_cfg=dict( + type='Xavier', layer='Conv2d', distribution='uniform')): + super(FPN_vitdet, self).__init__(init_cfg) + assert isinstance(in_channels, list) + self.in_channels = in_channels + self.out_channels = out_channels + self.num_ins = len(in_channels) + self.num_outs = num_outs + self.relu_before_extra_convs = relu_before_extra_convs + self.no_norm_on_lateral = no_norm_on_lateral + self.fp16_enabled = False + self.upsample_cfg = upsample_cfg.copy() + self.use_residual = use_residual + + if end_level == -1: + self.backbone_end_level = self.num_ins + assert num_outs >= self.num_ins - start_level + else: + # if end_level < inputs, no extra level is allowed + self.backbone_end_level = end_level + assert end_level <= len(in_channels) + assert num_outs == end_level - start_level + self.start_level = start_level + self.end_level = end_level + self.add_extra_convs = add_extra_convs + assert isinstance(add_extra_convs, (str, bool)) + if isinstance(add_extra_convs, str): + # Extra_convs_source choices: 'on_input', 'on_lateral', 'on_output' + assert add_extra_convs in ('on_input', 'on_lateral', 'on_output') + elif add_extra_convs: # True + self.add_extra_convs = 'on_input' + + self.lateral_convs = nn.ModuleList() + self.fpn_convs = nn.ModuleList() + + for i in range(self.start_level, self.backbone_end_level): + l_conv = ConvModule_Norm( + in_channels[i], + out_channels, + 1, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg if not self.no_norm_on_lateral else None, + act_cfg=act_cfg, + inplace=False) + fpn_conv = ConvModule_Norm( + out_channels, + out_channels, + 3, + padding=1, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg, + inplace=False) + + self.lateral_convs.append(l_conv) + self.fpn_convs.append(fpn_conv) + + # add extra conv layers (e.g., RetinaNet) + extra_levels = num_outs - self.backbone_end_level + self.start_level + if self.add_extra_convs and extra_levels >= 1: + for i in range(extra_levels): + if i == 0 and self.add_extra_convs == 'on_input': + in_channels = self.in_channels[self.backbone_end_level - 1] + else: + in_channels = out_channels + extra_fpn_conv = ConvModule_Norm( + in_channels, + out_channels, + 3, + stride=2, + padding=1, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg, + inplace=False) + self.fpn_convs.append(extra_fpn_conv) + + @auto_fp16() + def forward(self, inputs): + """Forward function.""" + assert len(inputs) == len(self.in_channels) + + # build laterals + laterals = [ + lateral_conv(inputs[i + self.start_level]) + for i, lateral_conv in enumerate(self.lateral_convs) + ] + + # build top-down path + used_backbone_levels = len(laterals) + if self.use_residual: + for i in range(used_backbone_levels - 1, 0, -1): + # In some cases, fixing `scale factor` (e.g. 2) is preferred, but + # it cannot co-exist with `size` in `F.interpolate`. + if 'scale_factor' in self.upsample_cfg: + laterals[i - 1] += F.interpolate(laterals[i], + **self.upsample_cfg) + else: + prev_shape = laterals[i - 1].shape[2:] + laterals[i - 1] += F.interpolate( + laterals[i], size=prev_shape, **self.upsample_cfg) + + # build outputs + # part 1: from original levels + outs = [ + self.fpn_convs[i](laterals[i]) for i in range(used_backbone_levels) + ] + # part 2: add extra levels + if self.num_outs > len(outs): + # use max pool to get more levels on top of outputs + # (e.g., Faster R-CNN, Mask R-CNN) + if not self.add_extra_convs: + for i in range(self.num_outs - used_backbone_levels): + outs.append(F.max_pool2d(outs[-1], 1, stride=2)) + # add conv layers on top of original feature maps (RetinaNet) + else: + if self.add_extra_convs == 'on_input': + extra_source = inputs[self.backbone_end_level - 1] + elif self.add_extra_convs == 'on_lateral': + extra_source = laterals[-1] + elif self.add_extra_convs == 'on_output': + extra_source = outs[-1] + else: + raise NotImplementedError + outs.append(self.fpn_convs[used_backbone_levels](extra_source)) + for i in range(used_backbone_levels + 1, self.num_outs): + if self.relu_before_extra_convs: + outs.append(self.fpn_convs[i](F.relu(outs[-1]))) + else: + outs.append(self.fpn_convs[i](outs[-1])) + return tuple(outs) diff --git a/detection/mmdet_custom/models/utils/__init__.py b/detection/mmdet_custom/models/utils/__init__.py new file mode 100644 index 0000000..a5bdc3e --- /dev/null +++ b/detection/mmdet_custom/models/utils/__init__.py @@ -0,0 +1,6 @@ +from .query_denoising import build_dn_generator +from .transformer import (DinoTransformer, DinoTransformerDecoder) +from .convModule_norm import ConvModule_Norm + + +__all__ = ['build_dn_generator', 'DinoTransformer', 'DinoTransformerDecoder'] \ No newline at end of file diff --git a/detection/mmdet_custom/models/utils/convModule_norm.py b/detection/mmdet_custom/models/utils/convModule_norm.py new file mode 100644 index 0000000..aa57499 --- /dev/null +++ b/detection/mmdet_custom/models/utils/convModule_norm.py @@ -0,0 +1,34 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + +from mmcv.cnn.bricks.conv_module import ConvModule + +class ConvModule_Norm(ConvModule): + def __init__(self, in_channels, + out_channels, + kernel, **kwargs): + super().__init__(in_channels, out_channels, kernel, **kwargs) + + self.normType = kwargs.get('norm_cfg', {'type':''}) + if self.normType is not None: + self.normType = self.normType['type'] + + def forward(self, x, activate=True, norm=True): + for layer in self.order: + if layer == 'conv': + if self.with_explicit_padding: + x = self.padding_layer(x) + x = self.conv(x) + elif layer == 'norm' and norm and self.with_norm: + if 'LN' in self.normType: + x = x.permute(0, 2, 3, 1) + x = self.norm(x) + x = x.permute(0, 3, 1, 2).contiguous() + else: + x = self.norm(x) + elif layer == 'act' and activate and self.with_activation: + x = self.activate(x) + return x \ No newline at end of file diff --git a/detection/mmdet_custom/models/utils/query_denoising.py b/detection/mmdet_custom/models/utils/query_denoising.py new file mode 100644 index 0000000..09b8c23 --- /dev/null +++ b/detection/mmdet_custom/models/utils/query_denoising.py @@ -0,0 +1,234 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +from mmcv.runner import BaseModule + +from mmdet.core import bbox_xyxy_to_cxcywh +from mmdet.models.utils.transformer import inverse_sigmoid + + +class DnQueryGenerator(BaseModule): + + def __init__(self, + num_queries, + hidden_dim, + num_classes, + noise_scale=dict(label=0.5, box=0.4), + group_cfg=dict( + dynamic=True, num_groups=None, num_dn_queries=None)): + super(DnQueryGenerator, self).__init__() + self.num_queries = num_queries + self.hidden_dim = hidden_dim + self.num_classes = num_classes + self.label_noise_scale = noise_scale['label'] + self.box_noise_scale = noise_scale['box'] + self.dynamic_dn_groups = group_cfg.get('dynamic', False) + if self.dynamic_dn_groups: + assert 'num_dn_queries' in group_cfg, \ + 'num_dn_queries should be set when using ' \ + 'dynamic dn groups' + self.num_dn = group_cfg['num_dn_queries'] + else: + assert 'num_groups' in group_cfg, \ + 'num_groups should be set when using ' \ + 'static dn groups' + self.num_dn = group_cfg['num_groups'] + assert isinstance(self.num_dn, int) and self.num_dn >= 1, \ + f'Expected the num in group_cfg to have type int. ' \ + f'Found {type(self.num_dn)} ' + + def get_num_groups(self, group_queries=None): + """ + Args: + group_queries (int): Number of dn queries in one group. + """ + if self.dynamic_dn_groups: + assert group_queries is not None, \ + 'group_queries should be provided when using ' \ + 'dynamic dn groups' + if group_queries == 0: + num_groups = 1 + else: + num_groups = self.num_dn // group_queries + else: + num_groups = self.num_dn + if num_groups < 1: # avoid num_groups < 1 in query generator + num_groups = 1 + return int(num_groups) + + def forward(self, + gt_bboxes, + gt_labels=None, + label_enc=None, + img_metas=None): + """ + Args: + gt_bboxes (List[Tensor]): List of ground truth bboxes + of the image, shape of each (num_gts, 4). + gt_labels (List[Tensor]): List of ground truth labels + of the image, shape of each (num_gts,), if None, + TODO:noisy_label would be None. + Returns: + TODO + """ + # TODO: temp only support for CDN + # TODO: temp assert gt_labels is not None and label_enc is not None + + if self.training: + if gt_labels is not None: + assert len(gt_bboxes) == len(gt_labels), \ + f'the length of provided gt_labels ' \ + f'{len(gt_labels)} should be equal to' \ + f' that of gt_bboxes {len(gt_bboxes)}' + assert gt_labels is not None \ + and label_enc is not None \ + and img_metas is not None # TODO: adjust args + batch_size = len(gt_bboxes) + + # convert bbox + gt_bboxes_list = [] + for img_meta, bboxes in zip(img_metas, gt_bboxes): + img_h, img_w, _ = img_meta['img_shape'] + factor = bboxes.new_tensor([img_w, img_h, img_w, + img_h]).unsqueeze(0) + bboxes_normalized = bbox_xyxy_to_cxcywh(bboxes) / factor + gt_bboxes_list.append(bboxes_normalized) + gt_bboxes = gt_bboxes_list + + known = [torch.ones_like(labels) for labels in gt_labels] + known_num = [sum(k) for k in known] + + num_groups = self.get_num_groups(int(max(known_num))) + + unmask_bbox = unmask_label = torch.cat(known) + labels = torch.cat(gt_labels) + boxes = torch.cat(gt_bboxes) + batch_idx = torch.cat([ + torch.full_like(t.long(), i) for i, t in enumerate(gt_labels) + ]) + + known_indice = torch.nonzero(unmask_label + unmask_bbox) + known_indice = known_indice.view(-1) + + known_indice = known_indice.repeat(2 * num_groups, 1).view(-1) + known_labels = labels.repeat(2 * num_groups, 1).view(-1) + known_bid = batch_idx.repeat(2 * num_groups, 1).view(-1) + known_bboxs = boxes.repeat(2 * num_groups, 1) + known_labels_expand = known_labels.clone() + known_bbox_expand = known_bboxs.clone() + + if self.label_noise_scale > 0: + p = torch.rand_like(known_labels_expand.float()) + chosen_indice = torch.nonzero( + p < (self.label_noise_scale * 0.5)).view(-1) + new_label = torch.randint_like(chosen_indice, 0, + self.num_classes) + known_labels_expand.scatter_(0, chosen_indice, new_label) + single_pad = int(max(known_num)) # TODO + + pad_size = int(single_pad * 2 * num_groups) + positive_idx = torch.tensor(range( + len(boxes))).long().cuda().unsqueeze(0).repeat(num_groups, 1) + positive_idx += (torch.tensor(range(num_groups)) * len(boxes) * + 2).long().cuda().unsqueeze(1) + positive_idx = positive_idx.flatten() + negative_idx = positive_idx + len(boxes) + if self.box_noise_scale > 0: + known_bbox_ = torch.zeros_like(known_bboxs) + known_bbox_[:, : 2] = \ + known_bboxs[:, : 2] - known_bboxs[:, 2:] / 2 + known_bbox_[:, 2:] = \ + known_bboxs[:, :2] + known_bboxs[:, 2:] / 2 + + diff = torch.zeros_like(known_bboxs) + diff[:, :2] = known_bboxs[:, 2:] / 2 + diff[:, 2:] = known_bboxs[:, 2:] / 2 + + rand_sign = torch.randint_like( + known_bboxs, low=0, high=2, dtype=torch.float32) + rand_sign = rand_sign * 2.0 - 1.0 + rand_part = torch.rand_like(known_bboxs) + rand_part[negative_idx] += 1.0 + rand_part *= rand_sign + known_bbox_ += \ + torch.mul(rand_part, diff).cuda() * self.box_noise_scale + known_bbox_ = known_bbox_.clamp(min=0.0, max=1.0) + known_bbox_expand[:, :2] = \ + (known_bbox_[:, :2] + known_bbox_[:, 2:]) / 2 + known_bbox_expand[:, 2:] = \ + known_bbox_[:, 2:] - known_bbox_[:, :2] + + m = known_labels_expand.long().to('cuda') + input_label_embed = label_enc(m) + input_bbox_embed = inverse_sigmoid(known_bbox_expand, eps=1e-3) + + padding_label = torch.zeros(pad_size, self.hidden_dim).cuda() + padding_bbox = torch.zeros(pad_size, 4).cuda() + + input_query_label = padding_label.repeat(batch_size, 1, 1) + input_query_bbox = padding_bbox.repeat(batch_size, 1, 1) + + map_known_indice = torch.tensor([]).to('cuda') + if len(known_num): + map_known_indice = torch.cat( + [torch.tensor(range(num)) for num in known_num]) + map_known_indice = torch.cat([ + map_known_indice + single_pad * i + for i in range(2 * num_groups) + ]).long() + if len(known_bid): + input_query_label[(known_bid.long(), + map_known_indice)] = input_label_embed + input_query_bbox[(known_bid.long(), + map_known_indice)] = input_bbox_embed + + tgt_size = pad_size + self.num_queries + attn_mask = torch.ones(tgt_size, tgt_size).to('cuda') < 0 + # match query cannot see the reconstruct + attn_mask[pad_size:, :pad_size] = True + # reconstruct cannot see each other + for i in range(num_groups): + if i == 0: + attn_mask[single_pad * 2 * i:single_pad * 2 * (i + 1), + single_pad * 2 * (i + 1):pad_size] = True + if i == num_groups - 1: + attn_mask[single_pad * 2 * i:single_pad * 2 * + (i + 1), :single_pad * i * 2] = True + else: + attn_mask[single_pad * 2 * i:single_pad * 2 * (i + 1), + single_pad * 2 * (i + 1):pad_size] = True + attn_mask[single_pad * 2 * i:single_pad * 2 * + (i + 1), :single_pad * 2 * i] = True + + dn_meta = { + 'pad_size': pad_size, + 'num_dn_group': num_groups, + } + else: + input_query_label = None + input_query_bbox = None + attn_mask = None + dn_meta = None + return input_query_label, input_query_bbox, attn_mask, dn_meta + + +class CdnQueryGenerator(DnQueryGenerator): + + def __init__(self, *args, **kwargs): + super(CdnQueryGenerator, self).__init__(*args, **kwargs) + + +def build_dn_generator(dn_args): + """ + Args: + dn_args (dict): + Returns: + """ + if dn_args is None: + return None + type = dn_args.pop('type') + if type == 'DnQueryGenerator': + return DnQueryGenerator(**dn_args) + elif type == 'CdnQueryGenerator': + return CdnQueryGenerator(**dn_args) + else: + raise NotImplementedError(f'{type} is not supported yet') \ No newline at end of file diff --git a/detection/mmdet_custom/models/utils/transformer.py b/detection/mmdet_custom/models/utils/transformer.py new file mode 100644 index 0000000..4380924 --- /dev/null +++ b/detection/mmdet_custom/models/utils/transformer.py @@ -0,0 +1,278 @@ +import math +import torch +import torch.nn as nn +from mmdet.models.utils.builder import TRANSFORMER +from mmcv.cnn.bricks.registry import ( + TRANSFORMER_LAYER_SEQUENCE, FEEDFORWARD_NETWORK, DROPOUT_LAYERS) +from mmdet.models.utils.transformer import (inverse_sigmoid, + DeformableDetrTransformerDecoder, + DeformableDetrTransformer) + + +def build_MLP(input_dim, hidden_dim, output_dim, num_layers): + # TODO: It can be implemented by add an out_channel arg of + # mmcv.cnn.bricks.transformer.FFN + assert num_layers > 1, \ + f'num_layers should be greater than 1 but got {num_layers}' + h = [hidden_dim] * (num_layers - 1) + layers = list() + for n, k in zip([input_dim] + h[:-1], h): + layers.extend((nn.Linear(n, k), nn.ReLU())) + # Note that the relu func of MLP in original DETR repo is set + # 'inplace=False', however the ReLU cfg of FFN in mmdet is set + # 'inplace=True' by default. + layers.append(nn.Linear(hidden_dim, output_dim)) + return nn.Sequential(*layers) + + +@TRANSFORMER_LAYER_SEQUENCE.register_module() +class DinoTransformerDecoder(DeformableDetrTransformerDecoder): + + def __init__(self, *args, with_rp_noise=False, **kwargs): + super(DinoTransformerDecoder, self).__init__(*args, **kwargs) + self.with_rp_noise = with_rp_noise + self._init_layers() + + def _init_layers(self): + self.ref_point_head = build_MLP( + self.embed_dims * 2, + self.embed_dims, + self.embed_dims, + 2) + self.norm = nn.LayerNorm(self.embed_dims) + + # @staticmethod + def gen_sineembed_for_position(self, pos_tensor): + # n_query, bs, _ = pos_tensor.size() + # sineembed_tensor = torch.zeros(n_query, bs, 256) + scale = 2 * math.pi + dim_t = torch.arange( + self.embed_dims//2, dtype=torch.float32, device=pos_tensor.device) + dim_t = 10000**(2 * (dim_t // 2) / (self.embed_dims//2)) + x_embed = pos_tensor[:, :, 0] * scale + y_embed = pos_tensor[:, :, 1] * scale + pos_x = x_embed[:, :, None] / dim_t + pos_y = y_embed[:, :, None] / dim_t + pos_x = torch.stack((pos_x[:, :, 0::2].sin(), pos_x[:, :, 1::2].cos()), + dim=3).flatten(2) + pos_y = torch.stack((pos_y[:, :, 0::2].sin(), pos_y[:, :, 1::2].cos()), + dim=3).flatten(2) + if pos_tensor.size(-1) == 2: + pos = torch.cat((pos_y, pos_x), dim=2) + elif pos_tensor.size(-1) == 4: + w_embed = pos_tensor[:, :, 2] * scale + pos_w = w_embed[:, :, None] / dim_t + pos_w = torch.stack( + (pos_w[:, :, 0::2].sin(), pos_w[:, :, 1::2].cos()), + dim=3).flatten(2) + + h_embed = pos_tensor[:, :, 3] * scale + pos_h = h_embed[:, :, None] / dim_t + pos_h = torch.stack( + (pos_h[:, :, 0::2].sin(), pos_h[:, :, 1::2].cos()), + dim=3).flatten(2) + + pos = torch.cat((pos_y, pos_x, pos_w, pos_h), dim=2) + else: + raise ValueError('Unknown pos_tensor shape(-1):{}'.format( + pos_tensor.size(-1))) + return pos + + def forward(self, + query, + *args, + reference_points=None, + valid_ratios=None, + reg_branches=None, + **kwargs): + output = query + intermediate = [] + intermediate_reference_points = [reference_points] + for lid, layer in enumerate(self.layers): + if reference_points.shape[-1] == 4: + reference_points_input = \ + reference_points[:, :, None] * torch.cat( + [valid_ratios, valid_ratios], -1)[:, None] + else: + assert reference_points.shape[-1] == 2 + reference_points_input = \ + reference_points[:, :, None] * valid_ratios[:, None] + + if self.with_rp_noise and self.training: + device = reference_points.device + b, n, d = reference_points.size() + noise = torch.rand(b, n, d).to(device) * 0.02 - 0.01 + reference_points = (reference_points + noise).clamp(0, 1) + + query_sine_embed = self.gen_sineembed_for_position( + reference_points_input[:, :, 0, :]) + query_pos = self.ref_point_head(query_sine_embed) + + query_pos = query_pos.permute(1, 0, 2) + output = layer( + output, + *args, + query_pos=query_pos, + reference_points=reference_points_input, + **kwargs) + output = output.permute(1, 0, 2) + + if reg_branches is not None: + tmp = reg_branches[lid](output) + assert reference_points.shape[-1] == 4 + new_reference_points = tmp + inverse_sigmoid( + reference_points, eps=1e-3) + new_reference_points = new_reference_points.sigmoid() + reference_points = new_reference_points.detach() + + output = output.permute(1, 0, 2) + if self.return_intermediate: + intermediate.append(self.norm(output)) + intermediate_reference_points.append(new_reference_points) + # NOTE this is for the "Look Forward Twice" module, + # in the DeformDETR, reference_points was appended. + + if self.return_intermediate: + return torch.stack(intermediate), torch.stack( + intermediate_reference_points) + + return output, reference_points + + +@TRANSFORMER.register_module() +class DinoTransformer(DeformableDetrTransformer): + + def __init__(self, *args, **kwargs): + super(DinoTransformer, self).__init__(*args, **kwargs) + + def init_layers(self): + """Initialize layers of the DinoTransformer.""" + self.level_embeds = nn.Parameter( + torch.Tensor(self.num_feature_levels, self.embed_dims)) + self.enc_output = nn.Linear(self.embed_dims, self.embed_dims) + self.enc_output_norm = nn.LayerNorm(self.embed_dims) + self.query_embed = nn.Embedding(self.two_stage_num_proposals, + self.embed_dims) + + def init_weights(self): + super().init_weights() + nn.init.normal_(self.query_embed.weight.data) + + def forward(self, + mlvl_feats, + mlvl_masks, + query_embed, + mlvl_pos_embeds, + dn_label_query, + dn_bbox_query, + attn_mask, + reg_branches=None, + cls_branches=None, + **kwargs): + assert self.as_two_stage and query_embed is None, \ + 'as_two_stage must be True for DINO' + + feat_flatten = [] + mask_flatten = [] + lvl_pos_embed_flatten = [] + spatial_shapes = [] + for lvl, (feat, mask, pos_embed) in enumerate( + zip(mlvl_feats, mlvl_masks, mlvl_pos_embeds)): + bs, c, h, w = feat.shape + spatial_shape = (h, w) + spatial_shapes.append(spatial_shape) + feat = feat.flatten(2).transpose(1, 2) + mask = mask.flatten(1) + pos_embed = pos_embed.flatten(2).transpose(1, 2) + lvl_pos_embed = pos_embed + self.level_embeds[lvl].view(1, 1, -1) + lvl_pos_embed_flatten.append(lvl_pos_embed) + feat_flatten.append(feat) + mask_flatten.append(mask) + feat_flatten = torch.cat(feat_flatten, 1) + mask_flatten = torch.cat(mask_flatten, 1) + lvl_pos_embed_flatten = torch.cat(lvl_pos_embed_flatten, 1) + spatial_shapes = torch.as_tensor( + spatial_shapes, dtype=torch.long, device=feat_flatten.device) + level_start_index = torch.cat((spatial_shapes.new_zeros( + (1, )), spatial_shapes.prod(1).cumsum(0)[:-1])) + valid_ratios = torch.stack( + [self.get_valid_ratio(m) for m in mlvl_masks], 1) + + reference_points = self.get_reference_points( + spatial_shapes, valid_ratios, device=feat.device) + + feat_flatten = feat_flatten.permute(1, 0, 2) # (H*W, bs, embed_dims) + lvl_pos_embed_flatten = lvl_pos_embed_flatten.permute( + 1, 0, 2) # (H*W, bs, embed_dims) + memory = self.encoder( + query=feat_flatten, + key=None, + value=None, + query_pos=lvl_pos_embed_flatten, + query_key_padding_mask=mask_flatten, + spatial_shapes=spatial_shapes, + reference_points=reference_points, + level_start_index=level_start_index, + valid_ratios=valid_ratios, + **kwargs) + + memory = memory.permute(1, 0, 2) + bs, _, c = memory.shape + + output_memory, output_proposals = self.gen_encoder_output_proposals( + memory, mask_flatten, spatial_shapes) + enc_outputs_class = cls_branches[self.decoder.num_layers]( + output_memory) + enc_outputs_coord_unact = reg_branches[self.decoder.num_layers]( + output_memory) + output_proposals + cls_out_features = cls_branches[self.decoder.num_layers].out_features + topk = self.two_stage_num_proposals + # NOTE In DeformDETR, enc_outputs_class[..., 0] is used for topk TODO + topk_indices = torch.topk(enc_outputs_class.max(-1)[0], topk, dim=1)[1] + # topk_proposal = torch.gather( + # output_proposals, 1, + # topk_indices.unsqueeze(-1).repeat(1, 1, 4)).sigmoid() + # topk_memory = torch.gather( + # output_memory, 1, + # topk_indices.unsqueeze(-1).repeat(1, 1, self.embed_dims)) + topk_score = torch.gather( + enc_outputs_class, 1, + topk_indices.unsqueeze(-1).repeat(1, 1, cls_out_features)) + topk_coords_unact = torch.gather( + enc_outputs_coord_unact, 1, + topk_indices.unsqueeze(-1).repeat(1, 1, 4)) + topk_anchor = topk_coords_unact.sigmoid() + # NOTE In the original DeformDETR, init_reference_out is obtained + # from detached topk_coords_unact, which is different with DINO. TODO + topk_coords_unact = topk_coords_unact.detach() + + query = self.query_embed.weight[:, None, :].repeat(1, bs, + 1).transpose(0, 1) + if dn_label_query is not None: + query = torch.cat([dn_label_query, query], dim=1) + if dn_bbox_query is not None: + reference_points = torch.cat([dn_bbox_query, topk_coords_unact], + dim=1) + else: + reference_points = topk_coords_unact + reference_points = reference_points.sigmoid() + + # decoder + query = query.permute(1, 0, 2) + memory = memory.permute(1, 0, 2) + inter_states, inter_references = self.decoder( + query=query, + key=None, + value=memory, + attn_masks=attn_mask, + key_padding_mask=mask_flatten, + reference_points=reference_points, + spatial_shapes=spatial_shapes, + level_start_index=level_start_index, + valid_ratios=valid_ratios, + reg_branches=reg_branches, + **kwargs) + + inter_references_out = inter_references + + return inter_states, inter_references_out, topk_score, topk_anchor diff --git a/detection/ops_dcnv3/functions/__init__.py b/detection/ops_dcnv3/functions/__init__.py new file mode 100644 index 0000000..4ff82f3 --- /dev/null +++ b/detection/ops_dcnv3/functions/__init__.py @@ -0,0 +1,7 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + +from .dcnv3_func import DCNv3Function, dcnv3_core_pytorch diff --git a/detection/ops_dcnv3/functions/dcnv3_func.py b/detection/ops_dcnv3/functions/dcnv3_func.py new file mode 100644 index 0000000..8c97fc8 --- /dev/null +++ b/detection/ops_dcnv3/functions/dcnv3_func.py @@ -0,0 +1,188 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + +from __future__ import absolute_import +from __future__ import print_function +from __future__ import division + +import torch +import torch.nn.functional as F +from torch.autograd import Function +from torch.autograd.function import once_differentiable +from torch.cuda.amp import custom_bwd, custom_fwd +import DCNv3 + + +class DCNv3Function(Function): + @staticmethod + @custom_fwd + def forward( + ctx, input, offset, mask, + kernel_h, kernel_w, stride_h, stride_w, + pad_h, pad_w, dilation_h, dilation_w, + group, group_channels, offset_scale, im2col_step): + ctx.kernel_h = kernel_h + ctx.kernel_w = kernel_w + ctx.stride_h = stride_h + ctx.stride_w = stride_w + ctx.pad_h = pad_h + ctx.pad_w = pad_w + ctx.dilation_h = dilation_h + ctx.dilation_w = dilation_w + ctx.group = group + ctx.group_channels = group_channels + ctx.offset_scale = offset_scale + ctx.im2col_step = im2col_step + output = DCNv3.dcnv3_forward( + input, offset, mask, kernel_h, + kernel_w, stride_h, stride_w, pad_h, + pad_w, dilation_h, dilation_w, group, + group_channels, offset_scale, ctx.im2col_step) + ctx.save_for_backward(input, offset, mask) + + return output + + @staticmethod + @once_differentiable + @custom_bwd + def backward(ctx, grad_output): + input, offset, mask = ctx.saved_tensors + grad_input, grad_offset, grad_mask = \ + DCNv3.dcnv3_backward( + input, offset, mask, ctx.kernel_h, + ctx.kernel_w, ctx.stride_h, ctx.stride_w, ctx.pad_h, + ctx.pad_w, ctx.dilation_h, ctx.dilation_w, ctx.group, + ctx.group_channels, ctx.offset_scale, grad_output.contiguous(), ctx.im2col_step) + + return grad_input, grad_offset, grad_mask, \ + None, None, None, None, None, None, None, None, None, None, None, None + + @staticmethod + def symbolic(g, input, offset, mask, kernel_h, kernel_w, stride_h, + stride_w, pad_h, pad_w, dilation_h, dilation_w, group, + group_channels, offset_scale, im2col_step): + """Symbolic function for mmdeploy::DCNv3. + + Returns: + DCNv3 op for onnx. + """ + return g.op( + 'mmdeploy::TRTDCNv3', + input, + offset, + mask, + kernel_h_i=int(kernel_h), + kernel_w_i=int(kernel_w), + stride_h_i=int(stride_h), + stride_w_i=int(stride_w), + pad_h_i=int(pad_h), + pad_w_i=int(pad_w), + dilation_h_i=int(dilation_h), + dilation_w_i=int(dilation_w), + group_i=int(group), + group_channels_i=int(group_channels), + offset_scale_f=float(offset_scale), + im2col_step_i=int(im2col_step), + ) + +def _get_reference_points(spatial_shapes, device, kernel_h, kernel_w, dilation_h, dilation_w, pad_h=0, pad_w=0, stride_h=1, stride_w=1): + _, H_, W_, _ = spatial_shapes + H_out = (H_ - (dilation_h * (kernel_h - 1) + 1)) // stride_h + 1 + W_out = (W_ - (dilation_w * (kernel_w - 1) + 1)) // stride_w + 1 + + ref_y, ref_x = torch.meshgrid( + torch.linspace( + # pad_h + 0.5, + # H_ - pad_h - 0.5, + (dilation_h * (kernel_h - 1)) // 2 + 0.5, + (dilation_h * (kernel_h - 1)) // 2 + 0.5 + (H_out - 1) * stride_h, + H_out, + dtype=torch.float32, + device=device), + torch.linspace( + # pad_w + 0.5, + # W_ - pad_w - 0.5, + (dilation_w * (kernel_w - 1)) // 2 + 0.5, + (dilation_w * (kernel_w - 1)) // 2 + 0.5 + (W_out - 1) * stride_w, + W_out, + dtype=torch.float32, + device=device)) + ref_y = ref_y.reshape(-1)[None] / H_ + ref_x = ref_x.reshape(-1)[None] / W_ + + ref = torch.stack((ref_x, ref_y), -1).reshape( + 1, H_out, W_out, 1, 2) + + return ref + + +def _generate_dilation_grids(spatial_shapes, kernel_h, kernel_w, dilation_h, dilation_w, group, device): + _, H_, W_, _ = spatial_shapes + points_list = [] + x, y = torch.meshgrid( + torch.linspace( + -((dilation_w * (kernel_w - 1)) // 2), + -((dilation_w * (kernel_w - 1)) // 2) + + (kernel_w - 1) * dilation_w, kernel_w, + dtype=torch.float32, + device=device), + torch.linspace( + -((dilation_h * (kernel_h - 1)) // 2), + -((dilation_h * (kernel_h - 1)) // 2) + + (kernel_h - 1) * dilation_h, kernel_h, + dtype=torch.float32, + device=device)) + + points_list.extend([x / W_, y / H_]) + grid = torch.stack(points_list, -1).reshape(-1, 1, 2).\ + repeat(1, group, 1).permute(1, 0, 2) + grid = grid.reshape(1, 1, 1, group * kernel_h * kernel_w, 2) + + return grid + + +def dcnv3_core_pytorch( + input, offset, mask, kernel_h, + kernel_w, stride_h, stride_w, pad_h, + pad_w, dilation_h, dilation_w, group, + group_channels, offset_scale): + # for debug and test only, + # need to use cuda version instead + input = F.pad( + input, + [0, 0, pad_h, pad_h, pad_w, pad_w]) + N_, H_in, W_in, _ = input.shape + _, H_out, W_out, _ = offset.shape + + ref = _get_reference_points( + input.shape, input.device, kernel_h, kernel_w, dilation_h, dilation_w, pad_h, pad_w, stride_h, stride_w) + grid = _generate_dilation_grids( + input.shape, kernel_h, kernel_w, dilation_h, dilation_w, group, input.device) + spatial_norm = torch.tensor([W_in, H_in]).reshape(1, 1, 1, 2).\ + repeat(1, 1, 1, group*kernel_h*kernel_w).to(input.device) + + sampling_locations = (ref + grid * offset_scale).repeat(N_, 1, 1, 1, 1).flatten(3, 4) + \ + offset * offset_scale / spatial_norm + + P_ = kernel_h * kernel_w + sampling_grids = 2 * sampling_locations - 1 + # N_, H_in, W_in, group*group_channels -> N_, H_in*W_in, group*group_channels -> N_, group*group_channels, H_in*W_in -> N_*group, group_channels, H_in, W_in + input_ = input.view(N_, H_in*W_in, group*group_channels).transpose(1, 2).\ + reshape(N_*group, group_channels, H_in, W_in) + # N_, H_out, W_out, group*P_*2 -> N_, H_out*W_out, group, P_, 2 -> N_, group, H_out*W_out, P_, 2 -> N_*group, H_out*W_out, P_, 2 + sampling_grid_ = sampling_grids.view(N_, H_out*W_out, group, P_, 2).transpose(1, 2).\ + flatten(0, 1) + # N_*group, group_channels, H_out*W_out, P_ + sampling_input_ = F.grid_sample( + input_, sampling_grid_, mode='bilinear', padding_mode='zeros', align_corners=False) + + # (N_, H_out, W_out, group*P_) -> N_, H_out*W_out, group, P_ -> (N_, group, H_out*W_out, P_) -> (N_*group, 1, H_out*W_out, P_) + mask = mask.view(N_, H_out*W_out, group, P_).transpose(1, 2).\ + reshape(N_*group, 1, H_out*W_out, P_) + output = (sampling_input_ * mask).sum(-1).view(N_, + group*group_channels, H_out*W_out) + + return output.transpose(1, 2).reshape(N_, H_out, W_out, -1).contiguous() diff --git a/detection/ops_dcnv3/make.sh b/detection/ops_dcnv3/make.sh new file mode 100755 index 0000000..0240533 --- /dev/null +++ b/detection/ops_dcnv3/make.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + +python setup.py build install diff --git a/detection/ops_dcnv3/modules/__init__.py b/detection/ops_dcnv3/modules/__init__.py new file mode 100644 index 0000000..4bba45c --- /dev/null +++ b/detection/ops_dcnv3/modules/__init__.py @@ -0,0 +1,7 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + +from .dcnv3 import DCNv3, DCNv3_pytorch \ No newline at end of file diff --git a/detection/ops_dcnv3/modules/dcnv3.py b/detection/ops_dcnv3/modules/dcnv3.py new file mode 100644 index 0000000..31955f4 --- /dev/null +++ b/detection/ops_dcnv3/modules/dcnv3.py @@ -0,0 +1,346 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + + +from __future__ import absolute_import +from __future__ import print_function +from __future__ import division + +import warnings +import torch +from torch import nn +import torch.nn.functional as F +from torch.nn.init import xavier_uniform_, constant_ +from ..functions import DCNv3Function, dcnv3_core_pytorch + + +class to_channels_first(nn.Module): + + def __init__(self): + super().__init__() + + def forward(self, x): + return x.permute(0, 3, 1, 2) + + +class to_channels_last(nn.Module): + + def __init__(self): + super().__init__() + + def forward(self, x): + return x.permute(0, 2, 3, 1) + + +def build_norm_layer(dim, + norm_layer, + in_format='channels_last', + out_format='channels_last', + eps=1e-6): + layers = [] + if norm_layer == 'BN': + if in_format == 'channels_last': + layers.append(to_channels_first()) + layers.append(nn.BatchNorm2d(dim)) + if out_format == 'channels_last': + layers.append(to_channels_last()) + elif norm_layer == 'LN': + if in_format == 'channels_first': + layers.append(to_channels_last()) + layers.append(nn.LayerNorm(dim, eps=eps)) + if out_format == 'channels_first': + layers.append(to_channels_first()) + else: + raise NotImplementedError( + f'build_norm_layer does not support {norm_layer}') + return nn.Sequential(*layers) + + +def build_act_layer(act_layer): + if act_layer == 'ReLU': + return nn.ReLU(inplace=True) + elif act_layer == 'SiLU': + return nn.SiLU(inplace=True) + elif act_layer == 'GELU': + return nn.GELU() + + raise NotImplementedError(f'build_act_layer does not support {act_layer}') + + +def _is_power_of_2(n): + if (not isinstance(n, int)) or (n < 0): + raise ValueError( + "invalid input for _is_power_of_2: {} (type: {})".format(n, type(n))) + + return (n & (n - 1) == 0) and n != 0 + + +class CenterFeatureScaleModule(nn.Module): + def forward(self, + query, + center_feature_scale_proj_weight, + center_feature_scale_proj_bias): + center_feature_scale = F.linear(query, + weight=center_feature_scale_proj_weight, + bias=center_feature_scale_proj_bias).sigmoid() + return center_feature_scale + + +class DCNv3_pytorch(nn.Module): + def __init__( + self, + channels=64, + kernel_size=3, + dw_kernel_size=None, + stride=1, + pad=1, + dilation=1, + group=4, + offset_scale=1.0, + act_layer='GELU', + norm_layer='LN', + center_feature_scale=False): + """ + DCNv3 Module + :param channels + :param kernel_size + :param stride + :param pad + :param dilation + :param group + :param offset_scale + :param act_layer + :param norm_layer + """ + super().__init__() + if channels % group != 0: + raise ValueError( + f'channels must be divisible by group, but got {channels} and {group}') + _d_per_group = channels // group + dw_kernel_size = dw_kernel_size if dw_kernel_size is not None else kernel_size + # you'd better set _d_per_group to a power of 2 which is more efficient in our CUDA implementation + if not _is_power_of_2(_d_per_group): + warnings.warn( + "You'd better set channels in DCNv3 to make the dimension of each attention head a power of 2 " + "which is more efficient in our CUDA implementation.") + + self.offset_scale = offset_scale + self.channels = channels + self.kernel_size = kernel_size + self.dw_kernel_size = dw_kernel_size + self.stride = stride + self.dilation = dilation + self.pad = pad + self.group = group + self.group_channels = channels // group + self.offset_scale = offset_scale + self.center_feature_scale = center_feature_scale + + self.dw_conv = nn.Sequential( + nn.Conv2d( + channels, + channels, + kernel_size=dw_kernel_size, + stride=1, + padding=(dw_kernel_size - 1) // 2, + groups=channels), + build_norm_layer( + channels, + norm_layer, + 'channels_first', + 'channels_last'), + build_act_layer(act_layer)) + self.offset = nn.Linear( + channels, + group * kernel_size * kernel_size * 2) + self.mask = nn.Linear( + channels, + group * kernel_size * kernel_size) + self.input_proj = nn.Linear(channels, channels) + self.output_proj = nn.Linear(channels, channels) + self._reset_parameters() + + if center_feature_scale: + self.center_feature_scale_proj_weight = nn.Parameter( + torch.zeros((group, channels), dtype=torch.float)) + self.center_feature_scale_proj_bias = nn.Parameter( + torch.tensor(0.0, dtype=torch.float).view((1,)).repeat(group, )) + self.center_feature_scale_module = CenterFeatureScaleModule() + + def _reset_parameters(self): + constant_(self.offset.weight.data, 0.) + constant_(self.offset.bias.data, 0.) + constant_(self.mask.weight.data, 0.) + constant_(self.mask.bias.data, 0.) + xavier_uniform_(self.input_proj.weight.data) + constant_(self.input_proj.bias.data, 0.) + xavier_uniform_(self.output_proj.weight.data) + constant_(self.output_proj.bias.data, 0.) + + def forward(self, input): + """ + :param query (N, H, W, C) + :return output (N, H, W, C) + """ + N, H, W, _ = input.shape + + x = self.input_proj(input) + x_proj = x + + x1 = input.permute(0, 3, 1, 2) + x1 = self.dw_conv(x1) + offset = self.offset(x1) + mask = self.mask(x1).reshape(N, H, W, self.group, -1) + mask = F.softmax(mask, -1).reshape(N, H, W, -1) + + x = dcnv3_core_pytorch( + x, offset, mask, + self.kernel_size, self.kernel_size, + self.stride, self.stride, + self.pad, self.pad, + self.dilation, self.dilation, + self.group, self.group_channels, + self.offset_scale) + if self.center_feature_scale: + center_feature_scale = self.center_feature_scale_module( + x1, self.center_feature_scale_proj_weight, self.center_feature_scale_proj_bias) + # N, H, W, groups -> N, H, W, groups, 1 -> N, H, W, groups, _d_per_group -> N, H, W, channels + center_feature_scale = center_feature_scale[..., None].repeat( + 1, 1, 1, 1, self.channels // self.group).flatten(-2) + x = x * (1 - center_feature_scale) + x_proj * center_feature_scale + x = self.output_proj(x) + + return x + + +class DCNv3(nn.Module): + def __init__( + self, + channels=64, + kernel_size=3, + dw_kernel_size=None, + stride=1, + pad=1, + dilation=1, + group=4, + offset_scale=1.0, + act_layer='GELU', + norm_layer='LN', + center_feature_scale=False): + """ + DCNv3 Module + :param channels + :param kernel_size + :param stride + :param pad + :param dilation + :param group + :param offset_scale + :param act_layer + :param norm_layer + """ + super().__init__() + if channels % group != 0: + raise ValueError( + f'channels must be divisible by group, but got {channels} and {group}') + _d_per_group = channels // group + dw_kernel_size = dw_kernel_size if dw_kernel_size is not None else kernel_size + # you'd better set _d_per_group to a power of 2 which is more efficient in our CUDA implementation + if not _is_power_of_2(_d_per_group): + warnings.warn( + "You'd better set channels in DCNv3 to make the dimension of each attention head a power of 2 " + "which is more efficient in our CUDA implementation.") + + self.offset_scale = offset_scale + self.channels = channels + self.kernel_size = kernel_size + self.dw_kernel_size = dw_kernel_size + self.stride = stride + self.dilation = dilation + self.pad = pad + self.group = group + self.group_channels = channels // group + self.offset_scale = offset_scale + self.center_feature_scale = center_feature_scale + + self.dw_conv = nn.Sequential( + nn.Conv2d( + channels, + channels, + kernel_size=dw_kernel_size, + stride=1, + padding=(dw_kernel_size - 1) // 2, + groups=channels), + build_norm_layer( + channels, + norm_layer, + 'channels_first', + 'channels_last'), + build_act_layer(act_layer)) + self.offset = nn.Linear( + channels, + group * kernel_size * kernel_size * 2) + self.mask = nn.Linear( + channels, + group * kernel_size * kernel_size) + self.input_proj = nn.Linear(channels, channels) + self.output_proj = nn.Linear(channels, channels) + self._reset_parameters() + + if center_feature_scale: + self.center_feature_scale_proj_weight = nn.Parameter( + torch.zeros((group, channels), dtype=torch.float)) + self.center_feature_scale_proj_bias = nn.Parameter( + torch.tensor(0.0, dtype=torch.float).view((1,)).repeat(group, )) + self.center_feature_scale_module = CenterFeatureScaleModule() + + def _reset_parameters(self): + constant_(self.offset.weight.data, 0.) + constant_(self.offset.bias.data, 0.) + constant_(self.mask.weight.data, 0.) + constant_(self.mask.bias.data, 0.) + xavier_uniform_(self.input_proj.weight.data) + constant_(self.input_proj.bias.data, 0.) + xavier_uniform_(self.output_proj.weight.data) + constant_(self.output_proj.bias.data, 0.) + + def forward(self, input): + """ + :param query (N, H, W, C) + :return output (N, H, W, C) + """ + N, H, W, _ = input.shape + + x = self.input_proj(input) + x_proj = x + dtype = x.dtype + + x1 = input.permute(0, 3, 1, 2) + x1 = self.dw_conv(x1) + offset = self.offset(x1) + mask = self.mask(x1).reshape(N, H, W, self.group, -1) + mask = F.softmax(mask, -1).reshape(N, H, W, -1).type(dtype) + + x = DCNv3Function.apply( + x, offset, mask, + self.kernel_size, self.kernel_size, + self.stride, self.stride, + self.pad, self.pad, + self.dilation, self.dilation, + self.group, self.group_channels, + self.offset_scale, + 256) + + if self.center_feature_scale: + center_feature_scale = self.center_feature_scale_module( + x1, self.center_feature_scale_proj_weight, self.center_feature_scale_proj_bias) + # N, H, W, groups -> N, H, W, groups, 1 -> N, H, W, groups, _d_per_group -> N, H, W, channels + center_feature_scale = center_feature_scale[..., None].repeat( + 1, 1, 1, 1, self.channels // self.group).flatten(-2) + x = x * (1 - center_feature_scale) + x_proj * center_feature_scale + x = self.output_proj(x) + + return x diff --git a/detection/ops_dcnv3/setup.py b/detection/ops_dcnv3/setup.py new file mode 100644 index 0000000..196b0c7 --- /dev/null +++ b/detection/ops_dcnv3/setup.py @@ -0,0 +1,75 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + +import os +import glob + +import torch + +from torch.utils.cpp_extension import CUDA_HOME +from torch.utils.cpp_extension import CppExtension +from torch.utils.cpp_extension import CUDAExtension + +from setuptools import find_packages +from setuptools import setup + +requirements = ["torch", "torchvision"] + + +def get_extensions(): + this_dir = os.path.dirname(os.path.abspath(__file__)) + extensions_dir = os.path.join(this_dir, "src") + + main_file = glob.glob(os.path.join(extensions_dir, "*.cpp")) + source_cpu = glob.glob(os.path.join(extensions_dir, "cpu", "*.cpp")) + source_cuda = glob.glob(os.path.join(extensions_dir, "cuda", "*.cu")) + + sources = main_file + source_cpu + extension = CppExtension + extra_compile_args = {"cxx": []} + define_macros = [] + + if torch.cuda.is_available() and CUDA_HOME is not None: + extension = CUDAExtension + sources += source_cuda + define_macros += [("WITH_CUDA", None)] + extra_compile_args["nvcc"] = [ + # "-DCUDA_HAS_FP16=1", + # "-D__CUDA_NO_HALF_OPERATORS__", + # "-D__CUDA_NO_HALF_CONVERSIONS__", + # "-D__CUDA_NO_HALF2_OPERATORS__", + ] + else: + raise NotImplementedError('Cuda is not availabel') + + sources = [os.path.join(extensions_dir, s) for s in sources] + include_dirs = [extensions_dir] + ext_modules = [ + extension( + "DCNv3", + sources, + include_dirs=include_dirs, + define_macros=define_macros, + extra_compile_args=extra_compile_args, + ) + ] + return ext_modules + + +setup( + name="DCNv3", + version="1.0", + author="InternImage", + url="https://github.com/OpenGVLab/InternImage", + description= + "PyTorch Wrapper for CUDA Functions of DCNv3", + packages=find_packages(exclude=( + "configs", + "tests", + )), + ext_modules=get_extensions(), + cmdclass={"build_ext": torch.utils.cpp_extension.BuildExtension}, +) diff --git a/detection/ops_dcnv3/src/cpu/dcnv3_cpu.cpp b/detection/ops_dcnv3/src/cpu/dcnv3_cpu.cpp new file mode 100644 index 0000000..a3bddc1 --- /dev/null +++ b/detection/ops_dcnv3/src/cpu/dcnv3_cpu.cpp @@ -0,0 +1,37 @@ +/*! +************************************************************************************************** +* InternImage +* Copyright (c) 2022 OpenGVLab +* Licensed under The MIT License [see LICENSE for details] +************************************************************************************************** +* Modified from +*https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +************************************************************************************************** +*/ + +#include + +#include +#include + +at::Tensor dcnv3_cpu_forward(const at::Tensor &input, const at::Tensor &offset, + const at::Tensor &mask, const int kernel_h, + const int kernel_w, const int stride_h, + const int stride_w, const int pad_h, + const int pad_w, const int dilation_h, + const int dilation_w, const int group, + const int group_channels, const float offset_scale, + const int im2col_step) { + AT_ERROR("Not implement on cpu"); +} + +std::vector +dcnv3_cpu_backward(const at::Tensor &input, const at::Tensor &offset, + const at::Tensor &mask, const int kernel_h, + const int kernel_w, const int stride_h, const int stride_w, + const int pad_h, const int pad_w, const int dilation_h, + const int dilation_w, const int group, + const int group_channels, const float offset_scale, + const at::Tensor &grad_output, const int im2col_step) { + AT_ERROR("Not implement on cpu"); +} diff --git a/detection/ops_dcnv3/src/cpu/dcnv3_cpu.h b/detection/ops_dcnv3/src/cpu/dcnv3_cpu.h new file mode 100644 index 0000000..d457bcb --- /dev/null +++ b/detection/ops_dcnv3/src/cpu/dcnv3_cpu.h @@ -0,0 +1,31 @@ +/*! +************************************************************************************************** +* InternImage +* Copyright (c) 2022 OpenGVLab +* Licensed under The MIT License [see LICENSE for details] +************************************************************************************************** +* Modified from +*https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +************************************************************************************************** +*/ + +#pragma once +#include + +at::Tensor dcnv3_cpu_forward(const at::Tensor &input, const at::Tensor &offset, + const at::Tensor &mask, const int kernel_h, + const int kernel_w, const int stride_h, + const int stride_w, const int pad_h, + const int pad_w, const int dilation_h, + const int dilation_w, const int group, + const int group_channels, const float offset_scale, + const int im2col_step); + +std::vector +dcnv3_cpu_backward(const at::Tensor &input, const at::Tensor &offset, + const at::Tensor &mask, const int kernel_h, + const int kernel_w, const int stride_h, const int stride_w, + const int pad_h, const int pad_w, const int dilation_h, + const int dilation_w, const int group, + const int group_channels, const float offset_scale, + const at::Tensor &grad_output, const int im2col_step); diff --git a/detection/ops_dcnv3/src/cuda/dcnv3_cuda.cu b/detection/ops_dcnv3/src/cuda/dcnv3_cuda.cu new file mode 100644 index 0000000..5284095 --- /dev/null +++ b/detection/ops_dcnv3/src/cuda/dcnv3_cuda.cu @@ -0,0 +1,174 @@ +/*! +************************************************************************************************** +* InternImage +* Copyright (c) 2022 OpenGVLab +* Licensed under The MIT License [see LICENSE for details] +************************************************************************************************** +* Modified from +*https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +************************************************************************************************** +*/ + +#include "cuda/dcnv3_im2col_cuda.cuh" +#include + +#include +#include +#include +#include +#include + +at::Tensor dcnv3_cuda_forward(const at::Tensor &input, const at::Tensor &offset, + const at::Tensor &mask, const int kernel_h, + const int kernel_w, const int stride_h, + const int stride_w, const int pad_h, + const int pad_w, const int dilation_h, + const int dilation_w, const int group, + const int group_channels, + const float offset_scale, const int im2col_step) { + AT_ASSERTM(input.is_contiguous(), "input tensor has to be contiguous"); + AT_ASSERTM(offset.is_contiguous(), "offset tensor has to be contiguous"); + AT_ASSERTM(mask.is_contiguous(), "mask tensor has to be contiguous"); + AT_ASSERTM(input.type().is_cuda(), "input must be a CUDA tensor"); + AT_ASSERTM(offset.type().is_cuda(), "offset must be a CUDA tensor"); + AT_ASSERTM(mask.type().is_cuda(), "mask must be a CUDA tensor"); + + const int batch = input.size(0); + const int height_in = input.size(1); + const int width_in = input.size(2); + const int channels = input.size(3); + const int height_out = + (height_in + 2 * pad_h - (dilation_h * (kernel_h - 1) + 1)) / stride_h + + 1; + const int width_out = + (width_in + 2 * pad_w - (dilation_w * (kernel_w - 1) + 1)) / stride_w + + 1; + const int im2col_step_ = std::min(batch, im2col_step); + + AT_ASSERTM(batch % im2col_step_ == 0, + "batch(%d) must divide im2col_step(%d)", batch, im2col_step_); + AT_ASSERTM( + channels == (group * group_channels), + "Input channels and group times group channels wont match: (%d vs %d).", + channels, group * group_channels); + + auto output = + at::zeros({batch, height_out, width_out, group * group_channels}, + input.options()); + + const int batch_n = im2col_step_; + auto output_n = output.view({batch / batch_n, batch_n, height_out, + width_out, group * group_channels}); + auto per_input_size = height_in * width_in * group * group_channels; + auto per_offset_size = + height_out * width_out * group * kernel_h * kernel_w * 2; + auto per_mask_size = height_out * width_out * group * kernel_h * kernel_w; + for (int n = 0; n < batch / im2col_step_; ++n) { + auto columns = output_n.select(0, n); + // AT_DISPATCH_FLOATING_TYPES( + AT_DISPATCH_FLOATING_TYPES_AND_HALF( + input.type(), "ms_deform_attn_forward_cuda", ([&] { + dcnv3_im2col_cuda( + at::cuda::getCurrentCUDAStream(), + input.data() + n * im2col_step_ * per_input_size, + offset.data() + + n * im2col_step_ * per_offset_size, + mask.data() + n * im2col_step_ * per_mask_size, + columns.data(), kernel_h, kernel_w, stride_h, + stride_w, pad_h, pad_w, dilation_h, dilation_w, group, + group_channels, batch_n, height_in, width_in, height_out, + width_out, offset_scale); + })); + } + + return output; +} + +std::vector +dcnv3_cuda_backward(const at::Tensor &input, const at::Tensor &offset, + const at::Tensor &mask, const int kernel_h, + const int kernel_w, const int stride_h, const int stride_w, + const int pad_h, const int pad_w, const int dilation_h, + const int dilation_w, const int group, + const int group_channels, const float offset_scale, + const at::Tensor &grad_output, const int im2col_step) { + + AT_ASSERTM(input.is_contiguous(), "input tensor has to be contiguous"); + AT_ASSERTM(offset.is_contiguous(), "offset tensor has to be contiguous"); + AT_ASSERTM(mask.is_contiguous(), "mask tensor has to be contiguous"); + AT_ASSERTM(grad_output.is_contiguous(), + "grad_output tensor has to be contiguous"); + AT_ASSERTM(input.type().is_cuda(), "input must be a CUDA tensor"); + AT_ASSERTM(offset.type().is_cuda(), "offset must be a CUDA tensor"); + AT_ASSERTM(mask.type().is_cuda(), "mask must be a CUDA tensor"); + AT_ASSERTM(grad_output.type().is_cuda(), + "grad_output must be a CUDA tensor"); + + const int batch = input.size(0); + const int height_in = input.size(1); + const int width_in = input.size(2); + const int channels = input.size(3); + const int height_out = + (height_in + 2 * pad_h - (dilation_h * (kernel_h - 1) + 1)) / stride_h + + 1; + const int width_out = + (width_in + 2 * pad_w - (dilation_w * (kernel_w - 1) + 1)) / stride_w + + 1; + const int im2col_step_ = std::min(batch, im2col_step); + + AT_ASSERTM(batch % im2col_step_ == 0, + "batch(%d) must divide im2col_step(%d)", batch, im2col_step_); + AT_ASSERTM( + channels == (group * group_channels), + "Input channels and group times group channels wont match: (%d vs %d).", + channels, group * group_channels); + + auto dtype = input.dtype(); + if (dtype == at::kHalf) { + dtype = at::kFloat; + } + + auto grad_input = at::zeros_like(input, dtype); + auto grad_offset = at::zeros_like(offset, dtype); + auto grad_mask = at::zeros_like(mask, dtype); + + const int batch_n = im2col_step_; + auto per_input_size = height_in * width_in * group * group_channels; + auto per_offset_size = + height_out * width_out * group * kernel_h * kernel_w * 2; + auto per_mask_size = height_out * width_out * group * kernel_h * kernel_w; + auto grad_output_n = + grad_output.view({batch / im2col_step_, batch_n, height_out * width_out, + group, group_channels}); + + for (int n = 0; n < batch / im2col_step_; ++n) { + auto grad_output_g = grad_output_n.select(0, n); + // AT_DISPATCH_FLOATING_TYPES( + AT_DISPATCH_FLOATING_TYPES_AND_HALF( + input.type(), "ms_deform_attn_backward_cuda", ([&] { + dcnv3_col2im_cuda( + at::cuda::getCurrentCUDAStream(), + grad_output_g.data(), + input.data() + n * im2col_step_ * per_input_size, + offset.data() + + n * im2col_step_ * per_offset_size, + mask.data() + n * im2col_step_ * per_mask_size, + kernel_h, kernel_w, stride_h, stride_w, pad_h, pad_w, + dilation_h, dilation_w, group, group_channels, batch_n, + height_in, width_in, height_out, width_out, offset_scale, + grad_input.data() + + n * im2col_step_ * per_input_size, + grad_offset.data() + + n * im2col_step_ * per_offset_size, + grad_mask.data() + + n * im2col_step_ * per_mask_size); + })); + } + + if (input.dtype() == torch::kHalf) { + return {grad_input.to(torch::kHalf), grad_offset.to(torch::kHalf), + grad_mask.to(torch::kHalf)}; + } else { + return {grad_input, grad_offset, grad_mask}; + } +} \ No newline at end of file diff --git a/detection/ops_dcnv3/src/cuda/dcnv3_cuda.h b/detection/ops_dcnv3/src/cuda/dcnv3_cuda.h new file mode 100644 index 0000000..069f282 --- /dev/null +++ b/detection/ops_dcnv3/src/cuda/dcnv3_cuda.h @@ -0,0 +1,31 @@ +/*! +************************************************************************************************** +* InternImage +* Copyright (c) 2022 OpenGVLab +* Licensed under The MIT License [see LICENSE for details] +************************************************************************************************** +* Modified from +*https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +************************************************************************************************** +*/ + +#pragma once +#include + +at::Tensor dcnv3_cuda_forward(const at::Tensor &input, const at::Tensor &offset, + const at::Tensor &mask, const int kernel_h, + const int kernel_w, const int stride_h, + const int stride_w, const int pad_h, + const int pad_w, const int dilation_h, + const int dilation_w, const int group, + const int group_channels, + const float offset_scale, const int im2col_step); + +std::vector +dcnv3_cuda_backward(const at::Tensor &input, const at::Tensor &offset, + const at::Tensor &mask, const int kernel_h, + const int kernel_w, const int stride_h, const int stride_w, + const int pad_h, const int pad_w, const int dilation_h, + const int dilation_w, const int group, + const int group_channels, const float offset_scale, + const at::Tensor &grad_output, const int im2col_step); diff --git a/detection/ops_dcnv3/src/cuda/dcnv3_im2col_cuda.cuh b/detection/ops_dcnv3/src/cuda/dcnv3_im2col_cuda.cuh new file mode 100644 index 0000000..b551ba3 --- /dev/null +++ b/detection/ops_dcnv3/src/cuda/dcnv3_im2col_cuda.cuh @@ -0,0 +1,1045 @@ +/*! +************************************************************************************************** +* InternImage +* Copyright (c) 2022 OpenGVLab +* Licensed under The MIT License [see LICENSE for details] +************************************************************************************************** +* Modified from +*https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +************************************************************************************************** +*/ + +#include +#include +#include + +#include +#include +#include +#include + +#define CUDA_KERNEL_LOOP(i, n) \ + for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < (n); \ + i += blockDim.x * gridDim.x) + +const int CUDA_NUM_THREADS = 256; +inline int GET_BLOCKS(const int N, const int num_threads) { + return (N + num_threads - 1) / num_threads; +} + +#define opmath_t at::opmath_type + +template +__device__ opmath_t dcnv3_im2col_bilinear(const scalar_t *&bottom_data, + const int &height, const int &width, + const int &group, + const int &group_channels, + const opmath_t &h, const opmath_t &w, + const int &g, const int &c) { + const int h_low = floor(h); + const int w_low = floor(w); + const int h_high = h_low + 1; + const int w_high = w_low + 1; + + const opmath_t lh = h - h_low; + const opmath_t lw = w - w_low; + const opmath_t hh = 1 - lh, hw = 1 - lw; + + const int w_stride = group * group_channels; + const int h_stride = width * w_stride; + const int h_low_ptr_offset = h_low * h_stride; + const int h_high_ptr_offset = h_low_ptr_offset + h_stride; + const int w_low_ptr_offset = w_low * w_stride; + const int w_high_ptr_offset = w_low_ptr_offset + w_stride; + const int base_ptr = g * group_channels + c; + + opmath_t v1 = 0; + if (h_low >= 0 && w_low >= 0) { + const int ptr1 = h_low_ptr_offset + w_low_ptr_offset + base_ptr; + v1 = bottom_data[ptr1]; + } + opmath_t v2 = 0; + if (h_low >= 0 && w_high <= width - 1) { + const int ptr2 = h_low_ptr_offset + w_high_ptr_offset + base_ptr; + v2 = bottom_data[ptr2]; + } + opmath_t v3 = 0; + if (h_high <= height - 1 && w_low >= 0) { + const int ptr3 = h_high_ptr_offset + w_low_ptr_offset + base_ptr; + v3 = bottom_data[ptr3]; + } + opmath_t v4 = 0; + if (h_high <= height - 1 && w_high <= width - 1) { + const int ptr4 = h_high_ptr_offset + w_high_ptr_offset + base_ptr; + v4 = bottom_data[ptr4]; + } + const opmath_t w1 = hh * hw, w2 = hh * lw, w3 = lh * hw, w4 = lh * lw; + + const opmath_t val = (w1 * v1 + w2 * v2 + w3 * v3 + w4 * v4); + return val; +} + +template +__device__ void dcnv3_col2im_bilinear( + const scalar_t *&bottom_data, const int &height, const int &width, + const int &nheads, const int &group_channels, const opmath_t &h, + const opmath_t &w, const int &m, const int &c, const opmath_t offset_scale, + const opmath_t &top_grad, const opmath_t &mask, opmath_t *&grad_im, + opmath_t *grad_offset, opmath_t *grad_mask) { + const int h_low = floor(h); + const int w_low = floor(w); + const int h_high = h_low + 1; + const int w_high = w_low + 1; + + const opmath_t lh = h - h_low; + const opmath_t lw = w - w_low; + const opmath_t hh = 1 - lh, hw = 1 - lw; + + const int w_stride = nheads * group_channels; + const int h_stride = width * w_stride; + const int h_low_ptr_offset = h_low * h_stride; + const int h_high_ptr_offset = h_low_ptr_offset + h_stride; + const int w_low_ptr_offset = w_low * w_stride; + const int w_high_ptr_offset = w_low_ptr_offset + w_stride; + const int base_ptr = m * group_channels + c; + + const opmath_t w1 = hh * hw, w2 = hh * lw, w3 = lh * hw, w4 = lh * lw; + const opmath_t top_grad_im = top_grad * mask; + opmath_t grad_h_weight = 0, grad_w_weight = 0; + + opmath_t v1 = 0; + if (h_low >= 0 && w_low >= 0) { + const int ptr1 = h_low_ptr_offset + w_low_ptr_offset + base_ptr; + v1 = bottom_data[ptr1]; + grad_h_weight -= hw * v1; + grad_w_weight -= hh * v1; + atomicAdd(grad_im + ptr1, w1 * top_grad_im); + } + opmath_t v2 = 0; + if (h_low >= 0 && w_high <= width - 1) { + const int ptr2 = h_low_ptr_offset + w_high_ptr_offset + base_ptr; + v2 = bottom_data[ptr2]; + grad_h_weight -= lw * v2; + grad_w_weight += hh * v2; + atomicAdd(grad_im + ptr2, w2 * top_grad_im); + } + opmath_t v3 = 0; + if (h_high <= height - 1 && w_low >= 0) { + const int ptr3 = h_high_ptr_offset + w_low_ptr_offset + base_ptr; + v3 = bottom_data[ptr3]; + grad_h_weight += hw * v3; + grad_w_weight -= lh * v3; + atomicAdd(grad_im + ptr3, w3 * top_grad_im); + } + opmath_t v4 = 0; + if (h_high <= height - 1 && w_high <= width - 1) { + const int ptr4 = h_high_ptr_offset + w_high_ptr_offset + base_ptr; + v4 = bottom_data[ptr4]; + grad_h_weight += lw * v4; + grad_w_weight += lh * v4; + atomicAdd(grad_im + ptr4, w4 * top_grad_im); + } + + const opmath_t val = (w1 * v1 + w2 * v2 + w3 * v3 + w4 * v4); + *grad_mask = top_grad * val; + *grad_offset = offset_scale * grad_w_weight * top_grad_im; + *(grad_offset + 1) = offset_scale * grad_h_weight * top_grad_im; +} + +template +__device__ void dcnv3_col2im_bilinear_gm( + const scalar_t *&bottom_data, const int &height, const int &width, + const int &nheads, const int &group_channels, const opmath_t &h, + const opmath_t &w, const int &m, const int &c, const opmath_t offset_scale, + const opmath_t &top_grad, const opmath_t &mask, opmath_t *&grad_im, + opmath_t *grad_offset, opmath_t *grad_mask) { + const int h_low = floor(h); + const int w_low = floor(w); + const int h_high = h_low + 1; + const int w_high = w_low + 1; + + const opmath_t lh = h - h_low; + const opmath_t lw = w - w_low; + const opmath_t hh = 1 - lh, hw = 1 - lw; + + const int w_stride = nheads * group_channels; + const int h_stride = width * w_stride; + const int h_low_ptr_offset = h_low * h_stride; + const int h_high_ptr_offset = h_low_ptr_offset + h_stride; + const int w_low_ptr_offset = w_low * w_stride; + const int w_high_ptr_offset = w_low_ptr_offset + w_stride; + const int base_ptr = m * group_channels + c; + + const opmath_t w1 = hh * hw, w2 = hh * lw, w3 = lh * hw, w4 = lh * lw; + const opmath_t top_grad_im = top_grad * mask; + opmath_t grad_h_weight = 0, grad_w_weight = 0; + + opmath_t v1 = 0; + if (h_low >= 0 && w_low >= 0) { + const int ptr1 = h_low_ptr_offset + w_low_ptr_offset + base_ptr; + v1 = bottom_data[ptr1]; + grad_h_weight -= hw * v1; + grad_w_weight -= hh * v1; + atomicAdd(grad_im + ptr1, w1 * top_grad_im); + } + opmath_t v2 = 0; + if (h_low >= 0 && w_high <= width - 1) { + const int ptr2 = h_low_ptr_offset + w_high_ptr_offset + base_ptr; + v2 = bottom_data[ptr2]; + grad_h_weight -= lw * v2; + grad_w_weight += hh * v2; + atomicAdd(grad_im + ptr2, w2 * top_grad_im); + } + opmath_t v3 = 0; + if (h_high <= height - 1 && w_low >= 0) { + const int ptr3 = h_high_ptr_offset + w_low_ptr_offset + base_ptr; + v3 = bottom_data[ptr3]; + grad_h_weight += hw * v3; + grad_w_weight -= lh * v3; + atomicAdd(grad_im + ptr3, w3 * top_grad_im); + } + opmath_t v4 = 0; + if (h_high <= height - 1 && w_high <= width - 1) { + const int ptr4 = h_high_ptr_offset + w_high_ptr_offset + base_ptr; + v4 = bottom_data[ptr4]; + grad_h_weight += lw * v4; + grad_w_weight += lh * v4; + atomicAdd(grad_im + ptr4, w4 * top_grad_im); + } + + const opmath_t val = (w1 * v1 + w2 * v2 + w3 * v3 + w4 * v4); + atomicAdd(grad_mask, top_grad * val); + atomicAdd(grad_offset, offset_scale * grad_w_weight * top_grad_im); + atomicAdd(grad_offset + 1, offset_scale * grad_h_weight * top_grad_im); +} + +template +__global__ void dcnv3_im2col_gpu_kernel( + const int num_kernels, const scalar_t *data_im, const scalar_t *data_offset, + const scalar_t *data_mask, scalar_t *data_col, const int kernel_h, + const int kernel_w, const int stride_h, const int stride_w, const int pad_h, + const int pad_w, const int dilation_h, const int dilation_w, + const int group, const int group_channels, const int height_in, + const int width_in, const int height_out, const int width_out, + const opmath_t offset_scale) { + CUDA_KERNEL_LOOP(index, num_kernels) { + int _temp = index; + const int c_col = _temp % group_channels; + _temp /= group_channels; + const int sampling_index = _temp; + const int g_col = _temp % group; + _temp /= group; + const int p0_w = ((dilation_w * (kernel_w - 1)) >> 1) - pad_w + + (_temp % width_out) * stride_w; + _temp /= width_out; + const int p0_h = ((dilation_h * (kernel_h - 1)) >> 1) - pad_h + + (_temp % height_out) * stride_h; + _temp /= height_out; + const int b_col = _temp; + + const int input_size = height_in * width_in; + scalar_t *data_col_ptr = data_col + index; + const int kernel_size = kernel_h * kernel_w; + int data_weight_ptr = sampling_index * kernel_size; + int data_loc_w_ptr = data_weight_ptr << 1; + const int qid_stride = group * group_channels; + opmath_t col = 0; + const scalar_t *data_im_ptr = data_im + b_col * input_size * qid_stride; + // top-left + const opmath_t p0_w_ = + p0_w - ((dilation_w * (kernel_w - 1)) >> 1) * offset_scale; + const opmath_t p0_h_ = + p0_h - ((dilation_h * (kernel_h - 1)) >> 1) * offset_scale; + for (int i = 0; i < kernel_w; ++i) { + for (int j = 0; j < kernel_h; ++j) { + const opmath_t offset_w = data_offset[data_loc_w_ptr]; + const opmath_t offset_h = data_offset[data_loc_w_ptr + 1]; + const opmath_t loc_w = + p0_w_ + (i * dilation_w + offset_w) * offset_scale; + const opmath_t loc_h = + p0_h_ + (j * dilation_h + offset_h) * offset_scale; + const opmath_t weight = data_mask[data_weight_ptr]; + if (loc_h > -1 && loc_w > -1 && loc_h < height_in && + loc_w < width_in) { + col += dcnv3_im2col_bilinear( + data_im_ptr, height_in, width_in, group, + group_channels, loc_h, loc_w, g_col, c_col) * + weight; + } + data_weight_ptr += 1; + data_loc_w_ptr += 2; + } + } + *data_col_ptr = col; + } +} + +// debug +template +__global__ void dcnv3_col2im_gpu_kernel_shm_blocksize_aware_reduce_v1( + const int num_kernels, const scalar_t *grad_col, const scalar_t *data_im, + const scalar_t *data_offset, const scalar_t *data_mask, const int kernel_h, + const int kernel_w, const int stride_h, const int stride_w, const int pad_h, + const int pad_w, const int dilation_h, const int dilation_w, + const int group, const int group_channels, const int height_in, + const int width_in, const int height_out, const int width_out, + const opmath_t offset_scale, opmath_t *grad_im, opmath_t *grad_offset, + opmath_t *grad_mask) { + CUDA_KERNEL_LOOP(index, num_kernels) { + __shared__ opmath_t cache_grad_offset[blockSize * 2]; + __shared__ opmath_t cache_grad_mask[blockSize]; + unsigned int tid = threadIdx.x; + int _temp = index; + const int c_col = _temp % group_channels; + _temp /= group_channels; + const int sampling_index = _temp; + const int g_col = _temp % group; + _temp /= group; + const int p0_w = ((dilation_w * (kernel_w - 1)) >> 1) - pad_w + + (_temp % width_out) * stride_w; + _temp /= width_out; + const int p0_h = ((dilation_h * (kernel_h - 1)) >> 1) - pad_h + + (_temp % height_out) * stride_h; + _temp /= height_out; + const int b_col = _temp; + + const opmath_t top_grad = grad_col[index]; + const int input_size = height_in * width_in; + const int kernel_size = kernel_h * kernel_w; + int data_weight_ptr = sampling_index * kernel_size; + int data_loc_w_ptr = data_weight_ptr << 1; + const int grad_sampling_ptr = data_weight_ptr; + grad_offset += grad_sampling_ptr << 1; + grad_mask += grad_sampling_ptr; + const int qid_stride = group * group_channels; + const int im_ptr_offset = b_col * input_size * qid_stride; + const scalar_t *data_im_ptr = data_im + im_ptr_offset; + opmath_t *grad_im_ptr = grad_im + im_ptr_offset; + const opmath_t p0_w_ = + p0_w - ((dilation_w * (kernel_w - 1)) >> 1) * offset_scale; + const opmath_t p0_h_ = + p0_h - ((dilation_h * (kernel_h - 1)) >> 1) * offset_scale; + for (int i = 0; i < kernel_w; ++i) { + for (int j = 0; j < kernel_h; ++j) { + const opmath_t offset_w = data_offset[data_loc_w_ptr]; + const opmath_t offset_h = data_offset[data_loc_w_ptr + 1]; + const opmath_t loc_w = + p0_w_ + (i * dilation_w + offset_w) * offset_scale; + const opmath_t loc_h = + p0_h_ + (j * dilation_h + offset_h) * offset_scale; + const opmath_t weight = data_mask[data_weight_ptr]; + *(cache_grad_offset + (threadIdx.x << 1)) = 0; + *(cache_grad_offset + ((threadIdx.x << 1) + 1)) = 0; + *(cache_grad_mask + threadIdx.x) = 0; + if (loc_h > -1 && loc_w > -1 && loc_h < height_in && + loc_w < width_in) { + dcnv3_col2im_bilinear( + data_im_ptr, height_in, width_in, group, group_channels, + loc_h, loc_w, g_col, c_col, offset_scale, top_grad, + weight, grad_im_ptr, + cache_grad_offset + (threadIdx.x << 1), + cache_grad_mask + threadIdx.x); + } + + __syncthreads(); + if (tid == 0) { + opmath_t _grad_w = cache_grad_offset[0], + _grad_h = cache_grad_offset[1], + _grad_a = cache_grad_mask[0]; + int sid = 2; + for (unsigned int tid = 1; tid < blockSize; ++tid) { + _grad_w += cache_grad_offset[sid]; + _grad_h += cache_grad_offset[sid + 1]; + _grad_a += cache_grad_mask[tid]; + sid += 2; + } + + *grad_offset = _grad_w; + *(grad_offset + 1) = _grad_h; + *grad_mask = _grad_a; + } + __syncthreads(); + + data_weight_ptr += 1; + data_loc_w_ptr += 2; + grad_mask += 1; + grad_offset += 2; + } + } + } +} + +template +__global__ void dcnv3_col2im_gpu_kernel_shm_blocksize_aware_reduce_v2( + const int num_kernels, const scalar_t *grad_col, const scalar_t *data_im, + const scalar_t *data_offset, const scalar_t *data_mask, const int kernel_h, + const int kernel_w, const int stride_h, const int stride_w, const int pad_h, + const int pad_w, const int dilation_h, const int dilation_w, + const int group, const int group_channels, const int height_in, + const int width_in, const int height_out, const int width_out, + const opmath_t offset_scale, opmath_t *grad_im, opmath_t *grad_offset, + opmath_t *grad_mask) { + CUDA_KERNEL_LOOP(index, num_kernels) { + __shared__ opmath_t cache_grad_offset[blockSize * 2]; + __shared__ opmath_t cache_grad_mask[blockSize]; + unsigned int tid = threadIdx.x; + int _temp = index; + const int c_col = _temp % group_channels; + _temp /= group_channels; + const int sampling_index = _temp; + const int g_col = _temp % group; + _temp /= group; + const int p0_w = ((dilation_w * (kernel_w - 1)) >> 1) - pad_w + + (_temp % width_out) * stride_w; + _temp /= width_out; + const int p0_h = ((dilation_h * (kernel_h - 1)) >> 1) - pad_h + + (_temp % height_out) * stride_h; + _temp /= height_out; + const int b_col = _temp; + + const opmath_t top_grad = grad_col[index]; + const int input_size = height_in * width_in; + const int kernel_size = kernel_h * kernel_w; + int data_weight_ptr = sampling_index * kernel_size; + int data_loc_w_ptr = data_weight_ptr << 1; + const int grad_sampling_ptr = data_weight_ptr; + grad_offset += grad_sampling_ptr << 1; + grad_mask += grad_sampling_ptr; + const int qid_stride = group * group_channels; + const int im_ptr_offset = b_col * input_size * qid_stride; + const scalar_t *data_im_ptr = data_im + im_ptr_offset; + opmath_t *grad_im_ptr = grad_im + im_ptr_offset; + const opmath_t p0_w_ = + p0_w - ((dilation_w * (kernel_w - 1)) >> 1) * offset_scale; + const opmath_t p0_h_ = + p0_h - ((dilation_h * (kernel_h - 1)) >> 1) * offset_scale; + for (int i = 0; i < kernel_w; ++i) { + for (int j = 0; j < kernel_h; ++j) { + const opmath_t offset_w = data_offset[data_loc_w_ptr]; + const opmath_t offset_h = data_offset[data_loc_w_ptr + 1]; + const opmath_t loc_w = + p0_w_ + (i * dilation_w + offset_w) * offset_scale; + const opmath_t loc_h = + p0_h_ + (j * dilation_h + offset_h) * offset_scale; + const opmath_t weight = data_mask[data_weight_ptr]; + *(cache_grad_offset + (threadIdx.x << 1)) = 0; + *(cache_grad_offset + ((threadIdx.x << 1) + 1)) = 0; + *(cache_grad_mask + threadIdx.x) = 0; + if (loc_h > -1 && loc_w > -1 && loc_h < height_in && + loc_w < width_in) { + dcnv3_col2im_bilinear( + data_im_ptr, height_in, width_in, group, group_channels, + loc_h, loc_w, g_col, c_col, offset_scale, top_grad, + weight, grad_im_ptr, + cache_grad_offset + (threadIdx.x << 1), + cache_grad_mask + threadIdx.x); + } + + __syncthreads(); + + for (unsigned int s = blockSize / 2; s > 0; s >>= 1) { + if (tid < s) { + const unsigned int xid1 = tid << 1; + const unsigned int xid2 = (tid + s) << 1; + cache_grad_mask[tid] += cache_grad_mask[tid + s]; + cache_grad_offset[xid1] += cache_grad_offset[xid2]; + cache_grad_offset[xid1 + 1] += + cache_grad_offset[xid2 + 1]; + } + __syncthreads(); + } + + if (tid == 0) { + *grad_offset = cache_grad_offset[0]; + *(grad_offset + 1) = cache_grad_offset[1]; + *grad_mask = cache_grad_mask[0]; + } + __syncthreads(); + + data_weight_ptr += 1; + data_loc_w_ptr += 2; + grad_mask += 1; + grad_offset += 2; + } + } + } +} + +template +__global__ void dcnv3_col2im_gpu_kernel_shm_reduce_v1( + const int num_kernels, const scalar_t *grad_col, const scalar_t *data_im, + const scalar_t *data_offset, const scalar_t *data_mask, const int kernel_h, + const int kernel_w, const int stride_h, const int stride_w, const int pad_h, + const int pad_w, const int dilation_h, const int dilation_w, + const int group, const int group_channels, const int height_in, + const int width_in, const int height_out, const int width_out, + const opmath_t offset_scale, opmath_t *grad_im, opmath_t *grad_offset, + opmath_t *grad_mask) { + CUDA_KERNEL_LOOP(index, num_kernels) { + extern __shared__ int _s[]; + opmath_t *cache_grad_offset = (opmath_t *)_s; + opmath_t *cache_grad_mask = cache_grad_offset + 2 * blockDim.x; + unsigned int tid = threadIdx.x; + int _temp = index; + const int c_col = _temp % group_channels; + _temp /= group_channels; + const int sampling_index = _temp; + const int g_col = _temp % group; + _temp /= group; + const int p0_w = ((dilation_w * (kernel_w - 1)) >> 1) - pad_w + + (_temp % width_out) * stride_w; + _temp /= width_out; + const int p0_h = ((dilation_h * (kernel_h - 1)) >> 1) - pad_h + + (_temp % height_out) * stride_h; + _temp /= height_out; + const int b_col = _temp; + + const opmath_t top_grad = grad_col[index]; + const int input_size = height_in * width_in; + const int kernel_size = kernel_h * kernel_w; + int data_weight_ptr = sampling_index * kernel_size; + int data_loc_w_ptr = data_weight_ptr << 1; + const int grad_sampling_ptr = data_weight_ptr; + grad_offset += grad_sampling_ptr << 1; + grad_mask += grad_sampling_ptr; + const int qid_stride = group * group_channels; + const int im_ptr_offset = b_col * input_size * qid_stride; + const scalar_t *data_im_ptr = data_im + im_ptr_offset; + opmath_t *grad_im_ptr = grad_im + im_ptr_offset; + const opmath_t p0_w_ = + p0_w - ((dilation_w * (kernel_w - 1)) >> 1) * offset_scale; + const opmath_t p0_h_ = + p0_h - ((dilation_h * (kernel_h - 1)) >> 1) * offset_scale; + for (int i = 0; i < kernel_w; ++i) { + for (int j = 0; j < kernel_h; ++j) { + const opmath_t offset_w = data_offset[data_loc_w_ptr]; + const opmath_t offset_h = data_offset[data_loc_w_ptr + 1]; + const opmath_t loc_w = + p0_w_ + (i * dilation_w + offset_w) * offset_scale; + const opmath_t loc_h = + p0_h_ + (j * dilation_h + offset_h) * offset_scale; + const opmath_t weight = data_mask[data_weight_ptr]; + *(cache_grad_offset + (threadIdx.x << 1)) = 0; + *(cache_grad_offset + ((threadIdx.x << 1) + 1)) = 0; + *(cache_grad_mask + threadIdx.x) = 0; + if (loc_h > -1 && loc_w > -1 && loc_h < height_in && + loc_w < width_in) { + dcnv3_col2im_bilinear( + data_im_ptr, height_in, width_in, group, group_channels, + loc_h, loc_w, g_col, c_col, offset_scale, top_grad, + weight, grad_im_ptr, + cache_grad_offset + (threadIdx.x << 1), + cache_grad_mask + threadIdx.x); + } + + __syncthreads(); + if (tid == 0) { + opmath_t _grad_w = cache_grad_offset[0], + _grad_h = cache_grad_offset[1], + _grad_a = cache_grad_mask[0]; + int sid = 2; + for (unsigned int tid = 1; tid < blockDim.x; ++tid) { + _grad_w += cache_grad_offset[sid]; + _grad_h += cache_grad_offset[sid + 1]; + _grad_a += cache_grad_mask[tid]; + sid += 2; + } + + *grad_offset = _grad_w; + *(grad_offset + 1) = _grad_h; + *grad_mask = _grad_a; + } + __syncthreads(); + + data_weight_ptr += 1; + data_loc_w_ptr += 2; + grad_mask += 1; + grad_offset += 2; + } + } + } +} + +template +__global__ void dcnv3_col2im_gpu_kernel_shm_reduce_v2( + const int num_kernels, const scalar_t *grad_col, const scalar_t *data_im, + const scalar_t *data_offset, const scalar_t *data_mask, const int kernel_h, + const int kernel_w, const int stride_h, const int stride_w, const int pad_h, + const int pad_w, const int dilation_h, const int dilation_w, + const int group, const int group_channels, const int height_in, + const int width_in, const int height_out, const int width_out, + const opmath_t offset_scale, opmath_t *grad_im, opmath_t *grad_offset, + opmath_t *grad_mask) { + CUDA_KERNEL_LOOP(index, num_kernels) { + extern __shared__ int _s[]; + opmath_t *cache_grad_offset = (opmath_t *)_s; + opmath_t *cache_grad_mask = cache_grad_offset + 2 * blockDim.x; + unsigned int tid = threadIdx.x; + int _temp = index; + const int c_col = _temp % group_channels; + _temp /= group_channels; + const int sampling_index = _temp; + const int g_col = _temp % group; + _temp /= group; + const int p0_w = ((dilation_w * (kernel_w - 1)) >> 1) - pad_w + + (_temp % width_out) * stride_w; + _temp /= width_out; + const int p0_h = ((dilation_h * (kernel_h - 1)) >> 1) - pad_h + + (_temp % height_out) * stride_h; + _temp /= height_out; + const int b_col = _temp; + + const opmath_t top_grad = grad_col[index]; + const int input_size = height_in * width_in; + const int kernel_size = kernel_h * kernel_w; + int data_weight_ptr = sampling_index * kernel_size; + int data_loc_w_ptr = data_weight_ptr << 1; + const int grad_sampling_ptr = data_weight_ptr; + grad_offset += grad_sampling_ptr << 1; + grad_mask += grad_sampling_ptr; + const int qid_stride = group * group_channels; + const int im_ptr_offset = b_col * input_size * qid_stride; + const scalar_t *data_im_ptr = data_im + im_ptr_offset; + opmath_t *grad_im_ptr = grad_im + im_ptr_offset; + const opmath_t p0_w_ = + p0_w - ((dilation_w * (kernel_w - 1)) >> 1) * offset_scale; + const opmath_t p0_h_ = + p0_h - ((dilation_h * (kernel_h - 1)) >> 1) * offset_scale; + for (int i = 0; i < kernel_w; ++i) { + for (int j = 0; j < kernel_h; ++j) { + const opmath_t offset_w = data_offset[data_loc_w_ptr]; + const opmath_t offset_h = data_offset[data_loc_w_ptr + 1]; + const opmath_t loc_w = + p0_w_ + (i * dilation_w + offset_w) * offset_scale; + const opmath_t loc_h = + p0_h_ + (j * dilation_h + offset_h) * offset_scale; + const opmath_t weight = data_mask[data_weight_ptr]; + *(cache_grad_offset + (threadIdx.x << 1)) = 0; + *(cache_grad_offset + ((threadIdx.x << 1) + 1)) = 0; + *(cache_grad_mask + threadIdx.x) = 0; + if (loc_h > -1 && loc_w > -1 && loc_h < height_in && + loc_w < width_in) { + dcnv3_col2im_bilinear( + data_im_ptr, height_in, width_in, group, group_channels, + loc_h, loc_w, g_col, c_col, offset_scale, top_grad, + weight, grad_im_ptr, + cache_grad_offset + (threadIdx.x << 1), + cache_grad_mask + threadIdx.x); + } + + __syncthreads(); + + for (unsigned int s = blockDim.x / 2, spre = blockDim.x; s > 0; + s >>= 1, spre >>= 1) { + if (tid < s) { + const unsigned int xid1 = tid << 1; + const unsigned int xid2 = (tid + s) << 1; + cache_grad_mask[tid] += cache_grad_mask[tid + s]; + cache_grad_offset[xid1] += cache_grad_offset[xid2]; + cache_grad_offset[xid1 + 1] += + cache_grad_offset[xid2 + 1]; + if (tid + (s << 1) < spre) { + cache_grad_mask[tid] += + cache_grad_mask[tid + (s << 1)]; + cache_grad_offset[xid1] += + cache_grad_offset[xid2 + (s << 1)]; + cache_grad_offset[xid1 + 1] += + cache_grad_offset[xid2 + 1 + (s << 1)]; + } + } + __syncthreads(); + } + + if (tid == 0) { + *grad_offset = cache_grad_offset[0]; + *(grad_offset + 1) = cache_grad_offset[1]; + *grad_mask = cache_grad_mask[0]; + } + __syncthreads(); + + data_weight_ptr += 1; + data_loc_w_ptr += 2; + grad_mask += 1; + grad_offset += 2; + } + } + } +} + +template +__global__ void dcnv3_col2im_gpu_kernel_shm_reduce_v2_multi_blocks( + const int num_kernels, const scalar_t *grad_col, const scalar_t *data_im, + const scalar_t *data_offset, const scalar_t *data_mask, const int kernel_h, + const int kernel_w, const int stride_h, const int stride_w, const int pad_h, + const int pad_w, const int dilation_h, const int dilation_w, + const int group, const int group_channels, const int height_in, + const int width_in, const int height_out, const int width_out, + const opmath_t offset_scale, opmath_t *grad_im, opmath_t *grad_offset, + opmath_t *grad_mask) { + CUDA_KERNEL_LOOP(index, num_kernels) { + extern __shared__ int _s[]; + opmath_t *cache_grad_offset = (opmath_t *)_s; + opmath_t *cache_grad_mask = cache_grad_offset + 2 * blockDim.x; + unsigned int tid = threadIdx.x; + int _temp = index; + const int c_col = _temp % group_channels; + _temp /= group_channels; + const int sampling_index = _temp; + const int g_col = _temp % group; + _temp /= group; + const int p0_w = ((dilation_w * (kernel_w - 1)) >> 1) - pad_w + + (_temp % width_out) * stride_w; + _temp /= width_out; + const int p0_h = ((dilation_h * (kernel_h - 1)) >> 1) - pad_h + + (_temp % height_out) * stride_h; + _temp /= height_out; + const int b_col = _temp; + + const opmath_t top_grad = grad_col[index]; + const int input_size = height_in * width_in; + const int kernel_size = kernel_h * kernel_w; + int data_weight_ptr = sampling_index * kernel_size; + int data_loc_w_ptr = data_weight_ptr << 1; + const int grad_sampling_ptr = data_weight_ptr; + grad_offset += grad_sampling_ptr << 1; + grad_mask += grad_sampling_ptr; + const int qid_stride = group * group_channels; + const int im_ptr_offset = b_col * input_size * qid_stride; + const scalar_t *data_im_ptr = data_im + im_ptr_offset; + opmath_t *grad_im_ptr = grad_im + im_ptr_offset; + const opmath_t p0_w_ = + p0_w - ((dilation_w * (kernel_w - 1)) >> 1) * offset_scale; + const opmath_t p0_h_ = + p0_h - ((dilation_h * (kernel_h - 1)) >> 1) * offset_scale; + for (int i = 0; i < kernel_w; ++i) { + for (int j = 0; j < kernel_h; ++j) { + const opmath_t offset_w = data_offset[data_loc_w_ptr]; + const opmath_t offset_h = data_offset[data_loc_w_ptr + 1]; + const opmath_t loc_w = + p0_w_ + (i * dilation_w + offset_w) * offset_scale; + const opmath_t loc_h = + p0_h_ + (j * dilation_h + offset_h) * offset_scale; + const opmath_t weight = data_mask[data_weight_ptr]; + *(cache_grad_offset + (threadIdx.x << 1)) = 0; + *(cache_grad_offset + ((threadIdx.x << 1) + 1)) = 0; + *(cache_grad_mask + threadIdx.x) = 0; + if (loc_h > -1 && loc_w > -1 && loc_h < height_in && + loc_w < width_in) { + dcnv3_col2im_bilinear( + data_im_ptr, height_in, width_in, group, group_channels, + loc_h, loc_w, g_col, c_col, offset_scale, top_grad, + weight, grad_im_ptr, + cache_grad_offset + (threadIdx.x << 1), + cache_grad_mask + threadIdx.x); + } + + __syncthreads(); + + for (unsigned int s = blockDim.x / 2, spre = blockDim.x; s > 0; + s >>= 1, spre >>= 1) { + if (tid < s) { + const unsigned int xid1 = tid << 1; + const unsigned int xid2 = (tid + s) << 1; + cache_grad_mask[tid] += cache_grad_mask[tid + s]; + cache_grad_offset[xid1] += cache_grad_offset[xid2]; + cache_grad_offset[xid1 + 1] += + cache_grad_offset[xid2 + 1]; + if (tid + (s << 1) < spre) { + cache_grad_mask[tid] += + cache_grad_mask[tid + (s << 1)]; + cache_grad_offset[xid1] += + cache_grad_offset[xid2 + (s << 1)]; + cache_grad_offset[xid1 + 1] += + cache_grad_offset[xid2 + 1 + (s << 1)]; + } + } + __syncthreads(); + } + + if (tid == 0) { + atomicAdd(grad_offset, cache_grad_offset[0]); + atomicAdd(grad_offset + 1, cache_grad_offset[1]); + atomicAdd(grad_mask, cache_grad_mask[0]); + } + __syncthreads(); + + data_weight_ptr += 1; + data_loc_w_ptr += 2; + grad_mask += 1; + grad_offset += 2; + } + } + } +} + +template +__global__ void dcnv3_col2im_gpu_kernel_gm( + const int num_kernels, const scalar_t *grad_col, const scalar_t *data_im, + const scalar_t *data_offset, const scalar_t *data_mask, const int kernel_h, + const int kernel_w, const int stride_h, const int stride_w, const int pad_h, + const int pad_w, const int dilation_h, const int dilation_w, + const int group, const int group_channels, const int height_in, + const int width_in, const int height_out, const int width_out, + const opmath_t offset_scale, opmath_t *grad_im, opmath_t *grad_offset, + opmath_t *grad_mask) { + CUDA_KERNEL_LOOP(index, num_kernels) { + int _temp = index; + const int c_col = _temp % group_channels; + _temp /= group_channels; + const int sampling_index = _temp; + const int g_col = _temp % group; + _temp /= group; + const int p0_w = ((dilation_w * (kernel_w - 1)) >> 1) - pad_w + + (_temp % width_out) * stride_w; + _temp /= width_out; + const int p0_h = ((dilation_h * (kernel_h - 1)) >> 1) - pad_h + + (_temp % height_out) * stride_h; + _temp /= height_out; + const int b_col = _temp; + + const opmath_t top_grad = grad_col[index]; + const int input_size = height_in * width_in; + const int kernel_size = kernel_h * kernel_w; + int data_weight_ptr = sampling_index * kernel_size; + int data_loc_w_ptr = data_weight_ptr << 1; + const int grad_sampling_ptr = data_weight_ptr; + grad_offset += grad_sampling_ptr << 1; + grad_mask += grad_sampling_ptr; + const int qid_stride = group * group_channels; + const int im_ptr_offset = b_col * input_size * qid_stride; + const scalar_t *data_im_ptr = data_im + im_ptr_offset; + opmath_t *grad_im_ptr = grad_im + im_ptr_offset; + const opmath_t p0_w_ = + p0_w - ((dilation_w * (kernel_w - 1)) >> 1) * offset_scale; + const opmath_t p0_h_ = + p0_h - ((dilation_h * (kernel_h - 1)) >> 1) * offset_scale; + for (int i = 0; i < kernel_w; ++i) { + for (int j = 0; j < kernel_h; ++j) { + const opmath_t offset_w = data_offset[data_loc_w_ptr]; + const opmath_t offset_h = data_offset[data_loc_w_ptr + 1]; + const opmath_t loc_w = + p0_w_ + (i * dilation_w + offset_w) * offset_scale; + const opmath_t loc_h = + p0_h_ + (j * dilation_h + offset_h) * offset_scale; + const opmath_t weight = data_mask[data_weight_ptr]; + if (loc_h > -1 && loc_w > -1 && loc_h < height_in && + loc_w < width_in) { + dcnv3_col2im_bilinear_gm( + data_im_ptr, height_in, width_in, group, group_channels, + loc_h, loc_w, g_col, c_col, offset_scale, top_grad, + weight, grad_im_ptr, grad_offset, grad_mask); + } + data_weight_ptr += 1; + data_loc_w_ptr += 2; + grad_mask += 1; + grad_offset += 2; + } + } + } +} + +template +void dcnv3_im2col_cuda(cudaStream_t stream, const scalar_t *data_im, + const scalar_t *data_offset, const scalar_t *data_mask, + scalar_t *data_col, const int kernel_h, + const int kernel_w, const int stride_h, + const int stride_w, const int pad_h, const int pad_w, + const int dilation_h, const int dilation_w, + const int group, const int group_channels, + const int batch_n, const int height_in, + const int width_in, const int height_out, + const int width_out, const opmath_t offset_scale) { + const int num_kernels = + batch_n * height_out * width_out * group * group_channels; + const int num_actual_kernels = + batch_n * height_out * width_out * group * group_channels; + const int num_threads = CUDA_NUM_THREADS; + dcnv3_im2col_gpu_kernel + <<>>(num_kernels, data_im, data_offset, data_mask, data_col, + kernel_h, kernel_w, stride_h, stride_w, pad_h, pad_w, + dilation_h, dilation_w, group, group_channels, height_in, + width_in, height_out, width_out, offset_scale); + + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) { + printf("error in dcnv3_im2col_cuda: %s\n", cudaGetErrorString(err)); + } +} + +template +void dcnv3_col2im_cuda( + cudaStream_t stream, const scalar_t *grad_col, const scalar_t *data_im, + const scalar_t *data_offset, const scalar_t *data_mask, const int kernel_h, + const int kernel_w, const int stride_h, const int stride_w, const int pad_h, + const int pad_w, const int dilation_h, const int dilation_w, + const int group, const int group_channels, const int batch_n, + const int height_in, const int width_in, const int height_out, + const int width_out, const opmath_t offset_scale, opmath_t *grad_im, + opmath_t *grad_offset, opmath_t *grad_mask) { + const int num_threads = + (group_channels > CUDA_NUM_THREADS) ? CUDA_NUM_THREADS : group_channels; + const int num_kernels = + batch_n * height_out * width_out * group * group_channels; + const int num_actual_kernels = + batch_n * height_out * width_out * group * group_channels; + if (group_channels > 1024) { + if ((group_channels & 1023) == 0) { + dcnv3_col2im_gpu_kernel_shm_reduce_v2_multi_blocks + <<>>( + num_kernels, grad_col, data_im, data_offset, data_mask, + kernel_h, kernel_w, stride_h, stride_w, pad_h, pad_w, + dilation_h, dilation_w, group, group_channels, height_in, + width_in, height_out, width_out, offset_scale, grad_im, + grad_offset, grad_mask); + } else { + dcnv3_col2im_gpu_kernel_gm + <<>>(num_kernels, grad_col, data_im, data_offset, + data_mask, kernel_h, kernel_w, stride_h, stride_w, + pad_h, pad_w, dilation_h, dilation_w, group, + group_channels, height_in, width_in, height_out, + width_out, offset_scale, grad_im, grad_offset, + grad_mask); + } + } else { + switch (group_channels) { + case 1: + dcnv3_col2im_gpu_kernel_shm_blocksize_aware_reduce_v1 + <<>>(num_kernels, grad_col, data_im, data_offset, + data_mask, kernel_h, kernel_w, stride_h, stride_w, + pad_h, pad_w, dilation_h, dilation_w, group, + group_channels, height_in, width_in, height_out, + width_out, offset_scale, grad_im, grad_offset, + grad_mask); + break; + case 2: + dcnv3_col2im_gpu_kernel_shm_blocksize_aware_reduce_v1 + <<>>(num_kernels, grad_col, data_im, data_offset, + data_mask, kernel_h, kernel_w, stride_h, stride_w, + pad_h, pad_w, dilation_h, dilation_w, group, + group_channels, height_in, width_in, height_out, + width_out, offset_scale, grad_im, grad_offset, + grad_mask); + break; + case 4: + dcnv3_col2im_gpu_kernel_shm_blocksize_aware_reduce_v1 + <<>>(num_kernels, grad_col, data_im, data_offset, + data_mask, kernel_h, kernel_w, stride_h, stride_w, + pad_h, pad_w, dilation_h, dilation_w, group, + group_channels, height_in, width_in, height_out, + width_out, offset_scale, grad_im, grad_offset, + grad_mask); + break; + case 8: + dcnv3_col2im_gpu_kernel_shm_blocksize_aware_reduce_v1 + <<>>(num_kernels, grad_col, data_im, data_offset, + data_mask, kernel_h, kernel_w, stride_h, stride_w, + pad_h, pad_w, dilation_h, dilation_w, group, + group_channels, height_in, width_in, height_out, + width_out, offset_scale, grad_im, grad_offset, + grad_mask); + break; + case 16: + dcnv3_col2im_gpu_kernel_shm_blocksize_aware_reduce_v1 + <<>>(num_kernels, grad_col, data_im, data_offset, + data_mask, kernel_h, kernel_w, stride_h, stride_w, + pad_h, pad_w, dilation_h, dilation_w, group, + group_channels, height_in, width_in, height_out, + width_out, offset_scale, grad_im, grad_offset, + grad_mask); + break; + case 32: + dcnv3_col2im_gpu_kernel_shm_blocksize_aware_reduce_v1 + <<>>(num_kernels, grad_col, data_im, data_offset, + data_mask, kernel_h, kernel_w, stride_h, stride_w, + pad_h, pad_w, dilation_h, dilation_w, group, + group_channels, height_in, width_in, height_out, + width_out, offset_scale, grad_im, grad_offset, + grad_mask); + break; + case 64: + dcnv3_col2im_gpu_kernel_shm_blocksize_aware_reduce_v2 + <<>>(num_kernels, grad_col, data_im, data_offset, + data_mask, kernel_h, kernel_w, stride_h, stride_w, + pad_h, pad_w, dilation_h, dilation_w, group, + group_channels, height_in, width_in, height_out, + width_out, offset_scale, grad_im, grad_offset, + grad_mask); + break; + case 128: + dcnv3_col2im_gpu_kernel_shm_blocksize_aware_reduce_v2 + <<>>(num_kernels, grad_col, data_im, data_offset, + data_mask, kernel_h, kernel_w, stride_h, stride_w, + pad_h, pad_w, dilation_h, dilation_w, group, + group_channels, height_in, width_in, height_out, + width_out, offset_scale, grad_im, grad_offset, + grad_mask); + break; + case 256: + dcnv3_col2im_gpu_kernel_shm_blocksize_aware_reduce_v2 + <<>>(num_kernels, grad_col, data_im, data_offset, + data_mask, kernel_h, kernel_w, stride_h, stride_w, + pad_h, pad_w, dilation_h, dilation_w, group, + group_channels, height_in, width_in, height_out, + width_out, offset_scale, grad_im, grad_offset, + grad_mask); + break; + case 512: + dcnv3_col2im_gpu_kernel_shm_blocksize_aware_reduce_v2 + <<>>(num_kernels, grad_col, data_im, data_offset, + data_mask, kernel_h, kernel_w, stride_h, stride_w, + pad_h, pad_w, dilation_h, dilation_w, group, + group_channels, height_in, width_in, height_out, + width_out, offset_scale, grad_im, grad_offset, + grad_mask); + break; + case 1024: + dcnv3_col2im_gpu_kernel_shm_blocksize_aware_reduce_v2 + <<>>(num_kernels, grad_col, data_im, data_offset, + data_mask, kernel_h, kernel_w, stride_h, stride_w, + pad_h, pad_w, dilation_h, dilation_w, group, + group_channels, height_in, width_in, height_out, + width_out, offset_scale, grad_im, grad_offset, + grad_mask); + break; + default: + if (group_channels < 64) { + dcnv3_col2im_gpu_kernel_shm_reduce_v1 + <<>>( + num_kernels, grad_col, data_im, data_offset, data_mask, + kernel_h, kernel_w, stride_h, stride_w, pad_h, pad_w, + dilation_h, dilation_w, group, group_channels, + height_in, width_in, height_out, width_out, + offset_scale, grad_im, grad_offset, grad_mask); + } else { + dcnv3_col2im_gpu_kernel_shm_reduce_v2 + <<>>( + num_kernels, grad_col, data_im, data_offset, data_mask, + kernel_h, kernel_w, stride_h, stride_w, pad_h, pad_w, + dilation_h, dilation_w, group, group_channels, + height_in, width_in, height_out, width_out, + offset_scale, grad_im, grad_offset, grad_mask); + } + } + } + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) { + printf("error in dcnv3_col2im_cuda: %s\n", cudaGetErrorString(err)); + } +} \ No newline at end of file diff --git a/detection/ops_dcnv3/src/dcnv3.h b/detection/ops_dcnv3/src/dcnv3.h new file mode 100644 index 0000000..029648e --- /dev/null +++ b/detection/ops_dcnv3/src/dcnv3.h @@ -0,0 +1,59 @@ +/*! +************************************************************************************************** +* InternImage +* Copyright (c) 2022 OpenGVLab +* Licensed under The MIT License [see LICENSE for details] +************************************************************************************************** +* Modified from +*https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +************************************************************************************************** +*/ + +#pragma once + +#include "cpu/dcnv3_cpu.h" + +#ifdef WITH_CUDA +#include "cuda/dcnv3_cuda.h" +#endif + +at::Tensor dcnv3_forward(const at::Tensor &input, const at::Tensor &offset, + const at::Tensor &mask, const int kernel_h, + const int kernel_w, const int stride_h, + const int stride_w, const int pad_h, const int pad_w, + const int dilation_h, const int dilation_w, + const int group, const int group_channels, + const float offset_scale, const int im2col_step) { + if (input.type().is_cuda()) { +#ifdef WITH_CUDA + return dcnv3_cuda_forward(input, offset, mask, kernel_h, kernel_w, + stride_h, stride_w, pad_h, pad_w, dilation_h, + dilation_w, group, group_channels, + offset_scale, im2col_step); +#else + AT_ERROR("Not compiled with GPU support"); +#endif + } + AT_ERROR("Not implemented on the CPU"); +} + +std::vector +dcnv3_backward(const at::Tensor &input, const at::Tensor &offset, + const at::Tensor &mask, const int kernel_h, const int kernel_w, + const int stride_h, const int stride_w, const int pad_h, + const int pad_w, const int dilation_h, const int dilation_w, + const int group, const int group_channels, + const float offset_scale, const at::Tensor &grad_output, + const int im2col_step) { + if (input.type().is_cuda()) { +#ifdef WITH_CUDA + return dcnv3_cuda_backward(input, offset, mask, kernel_h, kernel_w, + stride_h, stride_w, pad_h, pad_w, dilation_h, + dilation_w, group, group_channels, + offset_scale, grad_output, im2col_step); +#else + AT_ERROR("Not compiled with GPU support"); +#endif + } + AT_ERROR("Not implemented on the CPU"); +} diff --git a/detection/ops_dcnv3/src/vision.cpp b/detection/ops_dcnv3/src/vision.cpp new file mode 100644 index 0000000..1f7a908 --- /dev/null +++ b/detection/ops_dcnv3/src/vision.cpp @@ -0,0 +1,17 @@ +/*! +************************************************************************************************** +* InternImage +* Copyright (c) 2022 OpenGVLab +* Licensed under The MIT License [see LICENSE for details] +************************************************************************************************** +* Modified from +*https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +************************************************************************************************** +*/ + +#include "dcnv3.h" + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("dcnv3_forward", &dcnv3_forward, "dcnv3_forward"); + m.def("dcnv3_backward", &dcnv3_backward, "dcnv3_backward"); +} diff --git a/detection/ops_dcnv3/test.py b/detection/ops_dcnv3/test.py new file mode 100644 index 0000000..467f952 --- /dev/null +++ b/detection/ops_dcnv3/test.py @@ -0,0 +1,263 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + +from __future__ import absolute_import +from __future__ import print_function +from __future__ import division + +import time +import torch +import torch.nn as nn +import math +from torch.autograd import gradcheck + +from functions.dcnv3_func import DCNv3Function, dcnv3_core_pytorch + +H_in, W_in = 8, 8 +N, M, D = 2, 4, 16 +Kh, Kw = 3, 3 +P = Kh * Kw +offset_scale = 2.0 +pad = 1 +dilation = 1 +stride = 1 +H_out = (H_in + 2 * pad - (dilation * (Kh - 1) + 1)) // stride + 1 +W_out = (W_in + 2 * pad - (dilation * (Kw - 1) + 1)) // stride + 1 + +torch.manual_seed(3) + + +@torch.no_grad() +def check_forward_equal_with_pytorch_double(): + input = torch.rand(N, H_in, W_in, M*D).cuda() * 0.01 + offset = torch.rand(N, H_out, W_out, M*P*2).cuda() * 10 + mask = torch.rand(N, H_out, W_out, M, P).cuda() + 1e-5 + mask /= mask.sum(-1, keepdim=True) + mask = mask.reshape(N, H_out, W_out, M*P) + + output_pytorch = dcnv3_core_pytorch( + input.double(), + offset.double(), + mask.double(), + Kh, Kw, stride, stride, Kh // 2, Kw // 2, dilation, dilation, M, D, offset_scale).detach().cpu() + + im2col_step = 2 + output_cuda = DCNv3Function.apply( + input.double(), + offset.double(), + mask.double(), + Kh, Kw, stride, stride, Kh // 2, Kw // 2, dilation, dilation, M, D, offset_scale, + im2col_step).detach().cpu() + + fwdok = torch.allclose(output_cuda, output_pytorch) + max_abs_err = (output_cuda - output_pytorch).abs().max() + max_rel_err = ((output_cuda - output_pytorch).abs() / + output_pytorch.abs()).max() + print('>>> forward double') + print(f'* {fwdok} check_forward_equal_with_pytorch_double: max_abs_err {max_abs_err:.2e} max_rel_err {max_rel_err:.2e}') + + +@torch.no_grad() +def check_forward_equal_with_pytorch_float(): + input = torch.rand(N, H_in, W_in, M*D).cuda() * 0.01 + offset = torch.rand(N, H_out, W_out, M*P*2).cuda() * 10 + mask = torch.rand(N, H_out, W_out, M, P).cuda() + 1e-5 + mask /= mask.sum(-1, keepdim=True) + mask = mask.reshape(N, H_out, W_out, M*P) + + output_pytorch = dcnv3_core_pytorch( + input, + offset, + mask, + Kh, Kw, stride, stride, Kh // 2, Kw // 2, dilation, dilation, M, D, offset_scale).detach().cpu() + + im2col_step = 2 + output_cuda = DCNv3Function.apply( + input, + offset, + mask, + Kh, Kw, stride, stride, Kh // 2, Kw // 2, dilation, dilation, M, D, offset_scale, + im2col_step).detach().cpu() + + fwdok = torch.allclose(output_cuda, output_pytorch, rtol=1e-2, atol=1e-3) + max_abs_err = (output_cuda - output_pytorch).abs().max() + max_rel_err = ((output_cuda - output_pytorch).abs() / + output_pytorch.abs()).max() + print('>>> forward float') + print(f'* {fwdok} check_forward_equal_with_pytorch_float: max_abs_err {max_abs_err:.2e} max_rel_err {max_rel_err:.2e}') + + +def check_backward_equal_with_pytorch_double(channels=4, grad_input=True, grad_offset=True, grad_mask=True): + # H_in, W_in = 4, 4 + N = 2 + M = 2 + H_out = (H_in + 2 * pad - (dilation * (Kh - 1) + 1)) // stride + 1 + W_out = (W_in + 2 * pad - (dilation * (Kw - 1) + 1)) // stride + 1 + + D = channels + input0 = torch.rand(N, H_in, W_in, M*D).cuda() * 0.01 + offset0 = torch.rand(N, H_out, W_out, M*P*2).cuda() * 10 + mask0 = torch.rand(N, H_out, W_out, M, P).cuda() + 1e-5 + mask0 /= mask0.sum(-1, keepdim=True) + mask0 = mask0.reshape(N, H_out, W_out, M*P) + input0.requires_grad = grad_input + offset0.requires_grad = grad_offset + mask0.requires_grad = grad_mask + + output_pytorch = dcnv3_core_pytorch( + input0.double(), + offset0.double(), + mask0.double(), + Kh, Kw, stride, stride, Kh // 2, Kw // 2, dilation, dilation, M, D, offset_scale) + output_pytorch.sum().backward() + + input1 = input0.detach() + offset1 = offset0.detach() + mask1 = mask0.detach() + input1.requires_grad = grad_input + offset1.requires_grad = grad_offset + mask1.requires_grad = grad_mask + + im2col_step = 2 + output_cuda = DCNv3Function.apply( + input1.double(), + offset1.double(), + mask1.double(), + Kh, Kw, stride, stride, Kh // 2, Kw // 2, dilation, dilation, M, D, offset_scale, + im2col_step) + output_cuda.sum().backward() + + print(f'>>> backward double: channels {D}') + bwdok = torch.allclose(input0.grad, input1.grad, rtol=1e-2, atol=1e-3) + max_abs_err = (input0.grad - input1.grad).abs().max() + max_rel_err = ((input0.grad - input1.grad).abs() / + input0.grad.abs()).max() + print( + f'* {bwdok} input_grad check_backward_equal_with_pytorch_double: max_abs_err {max_abs_err:.2e} max_rel_err {max_rel_err:.2e}') + + bwdok = torch.allclose(offset0.grad, offset1.grad, rtol=1e-2, atol=1e-3) + max_abs_err = (offset0.grad - offset1.grad).abs().max() + max_rel_err = ((offset0.grad - offset1.grad).abs() / + offset0.grad.abs()).max() + print( + f'* {bwdok} offset_grad check_backward_equal_with_pytorch_double: max_abs_err {max_abs_err:.2e} max_rel_err {max_rel_err:.2e}') + + bwdok = torch.allclose(mask0.grad, mask1.grad, rtol=1e-2, atol=1e-3) + max_abs_err = (mask0.grad - mask1.grad).abs().max() + max_rel_err = ((mask0.grad - mask1.grad).abs() / + mask0.grad.abs()).max() + print( + f'* {bwdok} mask_grad check_backward_equal_with_pytorch_double: max_abs_err {max_abs_err:.2e} max_rel_err {max_rel_err:.2e}') + + +def check_backward_equal_with_pytorch_float(channels=4, grad_input=True, grad_offset=True, grad_mask=True): + # H_in, W_in = 4, 4 + N = 2 + M = 2 + H_out = (H_in + 2 * pad - (dilation * (Kh - 1) + 1)) // stride + 1 + W_out = (W_in + 2 * pad - (dilation * (Kw - 1) + 1)) // stride + 1 + + D = channels + input0 = torch.rand(N, H_in, W_in, M*D).cuda() * 0.01 + offset0 = torch.rand(N, H_out, W_out, M*P*2).cuda() * 10 + mask0 = torch.rand(N, H_out, W_out, M, P).cuda() + 1e-5 + mask0 /= mask0.sum(-1, keepdim=True) + mask0 = mask0.reshape(N, H_out, W_out, M*P) + input0.requires_grad = grad_input + offset0.requires_grad = grad_offset + mask0.requires_grad = grad_mask + + output_pytorch = dcnv3_core_pytorch( + input0, + offset0, + mask0, + Kh, Kw, stride, stride, Kh // 2, Kw // 2, dilation, dilation, M, D, offset_scale) + output_pytorch.sum().backward() + + input1 = input0.detach() + offset1 = offset0.detach() + mask1 = mask0.detach() + input1.requires_grad = grad_input + offset1.requires_grad = grad_offset + mask1.requires_grad = grad_mask + + im2col_step = 2 + output_cuda = DCNv3Function.apply( + input1, + offset1, + mask1, + Kh, Kw, stride, stride, Kh // 2, Kw // 2, dilation, dilation, M, D, offset_scale, + im2col_step) + output_cuda.sum().backward() + + print(f'>>> backward float: channels {D}') + bwdok = torch.allclose(input0.grad, input1.grad, rtol=1e-2, atol=1e-3) + max_abs_err = (input0.grad - input1.grad).abs().max() + max_rel_err = ((input0.grad - input1.grad).abs() / + input0.grad.abs()).max() + print( + f'* {bwdok} input_grad check_backward_equal_with_pytorch_float: max_abs_err {max_abs_err:.2e} max_rel_err {max_rel_err:.2e}') + + bwdok = torch.allclose(offset0.grad, offset1.grad, rtol=1e-2, atol=1e-3) + max_abs_err = (offset0.grad - offset1.grad).abs().max() + max_rel_err = ((offset0.grad - offset1.grad).abs() / + offset0.grad.abs()).max() + print( + f'* {bwdok} offset_grad check_backward_equal_with_pytorch_float: max_abs_err {max_abs_err:.2e} max_rel_err {max_rel_err:.2e}') + + bwdok = torch.allclose(mask0.grad, mask1.grad, rtol=1e-2, atol=1e-3) + max_abs_err = (mask0.grad - mask1.grad).abs().max() + max_rel_err = ((mask0.grad - mask1.grad).abs() / + mask0.grad.abs()).max() + print( + f'* {bwdok} mask_grad check_backward_equal_with_pytorch_float: max_abs_err {max_abs_err:.2e} max_rel_err {max_rel_err:.2e}') + + +@torch.no_grad() +def check_time_cost(im2col_step=128): + N = 512 + H_in, W_in = 64, 64 + H_out = (H_in + 2 * pad - (dilation * (Kh - 1) + 1)) // stride + 1 + W_out = (W_in + 2 * pad - (dilation * (Kw - 1) + 1)) // stride + 1 + + input = torch.rand(N, H_in, W_in, M*D).cuda() * 0.01 + offset = torch.rand(N, H_out, W_out, M*P*2).cuda() * 10 + mask = torch.rand(N, H_out, W_out, M, P).cuda() + 1e-5 + mask /= mask.sum(-1, keepdim=True) + mask = mask.reshape(N, H_out, W_out, M*P) + print( + f'>>> time cost: im2col_step {im2col_step}; input {input.shape}; points {P} ') + repeat = 100 + for i in range(repeat): + output_cuda = DCNv3Function.apply( + input, + offset, + mask, + Kh, Kw, stride, stride, Kh // 2, Kw // 2, dilation, dilation, M, D, 1.0, + im2col_step) + torch.cuda.synchronize() + start = time.time() + for i in range(repeat): + output_cuda = DCNv3Function.apply( + input, + offset, + mask, + Kh, Kw, stride, stride, Kh // 2, Kw // 2, dilation, dilation, M, D, 1.0, + im2col_step) + torch.cuda.synchronize() + print(f'foward time cost: {(time.time() - start) / repeat}') + + +if __name__ == '__main__': + check_forward_equal_with_pytorch_double() + check_forward_equal_with_pytorch_float() + for channels in [1, 16, 30, 32, 64, 71, 1025]: + check_backward_equal_with_pytorch_double(channels, True, True, True) + for channels in [1, 16, 30, 32, 64, 71, 1025]: + check_backward_equal_with_pytorch_float(channels, True, True, True) + for i in range(3): + im2col_step = 128 * (2 ** i) + check_time_cost(im2col_step) diff --git a/detection/slurm_test.sh b/detection/slurm_test.sh new file mode 100644 index 0000000..06d6a91 --- /dev/null +++ b/detection/slurm_test.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +set -x + +PARTITION=$1 +JOB_NAME=$2 +CONFIG=$3 +CHECKPOINT=$4 +GPUS=${GPUS:-8} +GPUS_PER_NODE=${GPUS_PER_NODE:-8} +CPUS_PER_TASK=${CPUS_PER_TASK:-5} +PY_ARGS=${@:5} +SRUN_ARGS=${SRUN_ARGS:-""} + +PYTHONPATH="$(dirname $0)/..":$PYTHONPATH \ +srun -p ${PARTITION} \ + --job-name=${JOB_NAME} \ + --gres=gpu:${GPUS_PER_NODE} \ + --ntasks=${GPUS} \ + --ntasks-per-node=${GPUS_PER_NODE} \ + --cpus-per-task=${CPUS_PER_TASK} \ + --kill-on-bad-exit=1 \ + --quotatype=spot \ + ${SRUN_ARGS} \ + python -u test.py ${CONFIG} ${CHECKPOINT} --launcher="slurm" ${PY_ARGS} diff --git a/detection/slurm_train.sh b/detection/slurm_train.sh new file mode 100755 index 0000000..165081a --- /dev/null +++ b/detection/slurm_train.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +set -x + +PARTITION=$1 +JOB_NAME=$2 +CONFIG=$3 +WORK_DIR=$4 +GPUS=${GPUS:-8} +GPUS_PER_NODE=${GPUS_PER_NODE:-8} +CPUS_PER_TASK=${CPUS_PER_TASK:-10} +SRUN_ARGS=${SRUN_ARGS:-""} +PY_ARGS=${@:5} + +PYTHONPATH="$(dirname $0)/..":$PYTHONPATH \ +srun -p ${PARTITION} \ + --job-name=${JOB_NAME} \ + --gres=gpu:${GPUS_PER_NODE} \ + --ntasks=${GPUS} \ + --ntasks-per-node=${GPUS_PER_NODE} \ + --cpus-per-task=${CPUS_PER_TASK} \ + --kill-on-bad-exit=1 \ + --quotatype=reserved \ + ${SRUN_ARGS} \ + python -u train.py ${CONFIG} --work-dir=${WORK_DIR} --launcher="slurm" ${PY_ARGS} \ No newline at end of file diff --git a/detection/test.py b/detection/test.py new file mode 100644 index 0000000..866a234 --- /dev/null +++ b/detection/test.py @@ -0,0 +1,265 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + + +import argparse +import os +import os.path as osp +import time +import warnings + +import mmcv +import torch +from mmcv import Config, DictAction +from mmcv.cnn import fuse_conv_bn +from mmcv.parallel import MMDataParallel, MMDistributedDataParallel +from mmcv.runner import (get_dist_info, init_dist, load_checkpoint, + wrap_fp16_model) +from mmdet.apis import multi_gpu_test, single_gpu_test +from mmdet.datasets import (build_dataloader, build_dataset, + replace_ImageToTensor) +from mmdet.models import build_detector +import mmdet_custom # noqa: F401,F403 +import mmcv_custom # noqa: F401,F403 + + +def parse_args(): + parser = argparse.ArgumentParser( + description='MMDet test (and eval) a model') + parser.add_argument('config', help='test config file path') + parser.add_argument('checkpoint', help='checkpoint file') + parser.add_argument( + '--work-dir', + help='the directory to save the file containing evaluation metrics') + parser.add_argument('--out', help='output result file in pickle format') + parser.add_argument( + '--fuse-conv-bn', + action='store_true', + help='Whether to fuse conv and bn, this will slightly increase' + 'the inference speed') + parser.add_argument('--gpu-ids', + type=int, + nargs='+', + help='ids of gpus to use ' + '(only applicable to non-distributed testing)') + parser.add_argument( + '--format-only', + action='store_true', + help='Format the output results without perform evaluation. It is' + 'useful when you want to format the result to a specific format and ' + 'submit it to the test server') + parser.add_argument( + '--eval', + type=str, + nargs='+', + help='evaluation metrics, which depends on the dataset, e.g., "bbox",' + ' "segm", "proposal" for COCO, and "mAP", "recall" for PASCAL VOC') + parser.add_argument('--show', action='store_true', help='show results') + parser.add_argument('--show-dir', + help='directory where painted images will be saved') + parser.add_argument('--show-score-thr', + type=float, + default=0.3, + help='score threshold (default: 0.3)') + parser.add_argument('--gpu-collect', + action='store_true', + help='whether to use gpu to collect results.') + parser.add_argument( + '--tmpdir', + help='tmp directory used for collecting results from multiple ' + 'workers, available when gpu-collect is not specified') + parser.add_argument( + '--cfg-options', + nargs='+', + action=DictAction, + help='override some settings in the used config, the key-value pair ' + 'in xxx=yyy format will be merged into config file. If the value to ' + 'be overwritten is a list, it should be like key="[a,b]" or key=a,b ' + 'It also allows nested list/tuple values, e.g. key="[(a,b),(c,d)]" ' + 'Note that the quotation marks are necessary and that no white space ' + 'is allowed.') + parser.add_argument( + '--options', + nargs='+', + action=DictAction, + help='custom options for evaluation, the key-value pair in xxx=yyy ' + 'format will be kwargs for dataset.evaluate() function (deprecate), ' + 'change to --eval-options instead.') + parser.add_argument( + '--eval-options', + nargs='+', + action=DictAction, + help='custom options for evaluation, the key-value pair in xxx=yyy ' + 'format will be kwargs for dataset.evaluate() function') + parser.add_argument('--launcher', + choices=['none', 'pytorch', 'slurm', 'mpi'], + default='none', + help='job launcher') + parser.add_argument('--local_rank', type=int, default=0) + args = parser.parse_args() + if 'LOCAL_RANK' not in os.environ: + os.environ['LOCAL_RANK'] = str(args.local_rank) + + if args.options and args.eval_options: + raise ValueError( + '--options and --eval-options cannot be both ' + 'specified, --options is deprecated in favor of --eval-options') + if args.options: + warnings.warn('--options is deprecated in favor of --eval-options') + args.eval_options = args.options + return args + + +def main(): + + args = parse_args() + + assert args.out or args.eval or args.format_only or args.show \ + or args.show_dir, \ + ('Please specify at least one operation (save/eval/format/show the ' + 'results / save the results) with the argument "--out", "--eval"' + ', "--format-only", "--show" or "--show-dir"') + + if args.eval and args.format_only: + raise ValueError('--eval and --format_only cannot be both specified') + + if args.out is not None and not args.out.endswith(('.pkl', '.pickle')): + raise ValueError('The output file must be a pkl file.') + + cfg = Config.fromfile(args.config) + if args.cfg_options is not None: + cfg.merge_from_dict(args.cfg_options) + # set cudnn_benchmark + if cfg.get('cudnn_benchmark', False): + torch.backends.cudnn.benchmark = True + + cfg.model.pretrained = None + if cfg.model.get('neck'): + if isinstance(cfg.model.neck, list): + for neck_cfg in cfg.model.neck: + if neck_cfg.get('rfp_backbone'): + if neck_cfg.rfp_backbone.get('pretrained'): + neck_cfg.rfp_backbone.pretrained = None + elif cfg.model.neck.get('rfp_backbone'): + if cfg.model.neck.rfp_backbone.get('pretrained'): + cfg.model.neck.rfp_backbone.pretrained = None + + # in case the test dataset is concatenated + samples_per_gpu = 1 + if isinstance(cfg.data.test, dict): + cfg.data.test.test_mode = True + samples_per_gpu = cfg.data.test.pop('samples_per_gpu', 1) + if samples_per_gpu > 1: + # Replace 'ImageToTensor' to 'DefaultFormatBundle' + cfg.data.test.pipeline = replace_ImageToTensor( + cfg.data.test.pipeline) + elif isinstance(cfg.data.test, list): + for ds_cfg in cfg.data.test: + ds_cfg.test_mode = True + samples_per_gpu = max( + [ds_cfg.pop('samples_per_gpu', 1) for ds_cfg in cfg.data.test]) + if samples_per_gpu > 1: + for ds_cfg in cfg.data.test: + ds_cfg.pipeline = replace_ImageToTensor(ds_cfg.pipeline) + + if args.gpu_ids is not None: + cfg.gpu_ids = args.gpu_ids + else: + cfg.gpu_ids = range(1) + + + # init distributed env first, since logger depends on the dist info. + if args.launcher == 'none': + distributed = False + if len(cfg.gpu_ids) > 1: + warnings.warn( + f'We treat {cfg.gpu_ids} as gpu-ids, and reset to ' + f'{cfg.gpu_ids[0:1]} as gpu-ids to avoid potential error in ' + 'non-distribute testing time.') + cfg.gpu_ids = cfg.gpu_ids[0:1] + else: + distributed = True + init_dist(args.launcher, **cfg.dist_params) + + + rank, _ = get_dist_info() + # allows not to create + if args.work_dir is not None and rank == 0: + mmcv.mkdir_or_exist(osp.abspath(args.work_dir)) + timestamp = time.strftime('%Y%m%d_%H%M%S', time.localtime()) + json_file = osp.join(args.work_dir, f'eval_{timestamp}.json') + + # build the dataloader + dataset = build_dataset(cfg.data.test) + data_loader = build_dataloader(dataset, + samples_per_gpu=samples_per_gpu, + workers_per_gpu=cfg.data.workers_per_gpu, + dist=distributed, + shuffle=False) + + # build the model and load checkpoint + cfg.model.train_cfg = None + model = build_detector(cfg.model, test_cfg=cfg.get('test_cfg')) + print(model) + model_without_ddp = model + n_parameters = sum(p.numel() for p in model.parameters() + if p.requires_grad) + print(f"number of params: {n_parameters}") + + + fp16_cfg = cfg.get('fp16', None) + if fp16_cfg is not None: + wrap_fp16_model(model) + checkpoint = load_checkpoint(model, args.checkpoint, map_location='cpu') + + if args.fuse_conv_bn: + model = fuse_conv_bn(model) + # old versions did not save class info in checkpoints, this walkaround is + # for backward compatibility + if 'CLASSES' in checkpoint.get('meta', {}): + model.CLASSES = checkpoint['meta']['CLASSES'] + else: + model.CLASSES = dataset.CLASSES + + if not distributed: + model = MMDataParallel(model, device_ids=cfg.gpu_ids) + outputs = single_gpu_test(model, data_loader, args.show, args.show_dir, + args.show_score_thr) + else: + model = MMDistributedDataParallel( + model.cuda(), + device_ids=[torch.cuda.current_device()], + broadcast_buffers=False) + outputs = multi_gpu_test(model, data_loader, args.tmpdir, + args.gpu_collect) + + + rank, _ = get_dist_info() + if rank == 0: + if args.out: + print(f'\nwriting results to {args.out}') + mmcv.dump(outputs, args.out) + kwargs = {} if args.eval_options is None else args.eval_options + if args.format_only: + dataset.format_results(outputs, **kwargs) + if args.eval: + eval_kwargs = cfg.get('evaluation', {}).copy() + # hard-code way to remove EvalHook args + for key in [ + 'interval', 'tmpdir', 'start', 'gpu_collect', 'save_best', + 'rule', 'dynamic_intervals' + ]: + eval_kwargs.pop(key, None) + eval_kwargs.update(dict(metric=args.eval, **kwargs)) + metric = dataset.evaluate(outputs, **eval_kwargs) + print(metric) + metric_dict = dict(config=args.config, metric=metric) + if args.work_dir is not None and rank == 0: + mmcv.dump(metric_dict, json_file) + + +if __name__ == '__main__': + main() diff --git a/detection/tools/create_crowd_anno.py b/detection/tools/create_crowd_anno.py new file mode 100644 index 0000000..e363992 --- /dev/null +++ b/detection/tools/create_crowd_anno.py @@ -0,0 +1,95 @@ +import argparse +import os +import pickle as pkl +import numpy as np +import random +from PIL import Image +import concurrent.futures +import json +import mmcv + +def parse_args(): + parser = argparse.ArgumentParser(description='Generate MMDetection Annotations for Crowdhuman-like dataset') + parser.add_argument('--dataset', help='dataset name', type=str) + parser.add_argument('--dataset-split', help='dataset split, e.g. train, val', type=str) + + args = parser.parse_args() + return args.dataset, args.dataset_split + +def load_func(fpath): + assert os.path.exists(fpath) + with open(fpath, 'r') as fid: + lines = fid.readlines() + records = [json.loads(line.strip('\n')) for line in lines] + return records + +def decode_annotations(records, dataset_path): + rec_ids = list(range(len(records))) + img_list = [] + ann_list = [] + ann_id = 1 + for idx, rec_id in enumerate(rec_ids): + img_id = records[rec_id]['ID'] + img_url = dataset_path + 'Images/' + img_id + '.jpg' + assert os.path.exists(img_url) + im = Image.open(img_url) + im_w, im_h = im.width, im.height + + gt_box = records[rec_id]['gtboxes'] + gt_box_len = len(gt_box) + img_dict = dict( + file_name=img_id + '.jpg', + height=im_h, + width=im_w, + id=idx + ) + img_list.append(img_dict) + for ii in range(gt_box_len): + each_data = gt_box[ii] + x, y, w, h = each_data['fbox'] + + if w <= 0 or h <= 0: + continue + # x1 = x; y1 = y; x2 = x + w; y2 = y + h + + valid_bbox = [x, y, w, h] + if each_data['tag'] == 'person': + tag = 1 + else: + tag = -2 + if 'extra' in each_data: + if 'ignore' in each_data['extra']: + if each_data['extra']['ignore'] != 0: + tag = -2 + ann_dict = dict( + area=w * h, + iscrowd=1 if tag == -2 else 0, + image_id=idx, + bbox=[x, y, w, h], + category_id=1, + id=ann_id, + # ignore=1 if tag == -2 else 1, + ) + ann_id += 1 + ann_list.append(ann_dict) + cate_list = [{'supercategory': 'none', 'id': 1, 'name': 'person'}] + json_dict = dict( + images=img_list, + annotations=ann_list, + categories=cate_list + ) + return json_dict + +if __name__ == "__main__": + dataset_name, dataset_type = parse_args() + dataset_path = 'data/%s/' % dataset_name + ch_file_path = dataset_path + 'annotations/annotation_%s.odgt' % dataset_type + json_file_path = dataset_path + 'annotations/annotation_%s.json' % dataset_type + + records = load_func(ch_file_path) + print("Loading Annotations Done") + + json_dict = decode_annotations(records, dataset_path) + + print("Parsing Bbox Number: %d" % len(json_dict['annotations'])) + mmcv.dump(json_dict, json_file_path) diff --git a/detection/tools/evaluate/__init__.py b/detection/tools/evaluate/__init__.py new file mode 100644 index 0000000..9910d1e --- /dev/null +++ b/detection/tools/evaluate/__init__.py @@ -0,0 +1,2 @@ +from .compute_APMR import compute_APMR +from .compute_JI import compute_JI_with_ignore \ No newline at end of file diff --git a/detection/train.py b/detection/train.py new file mode 100644 index 0000000..afcb30e --- /dev/null +++ b/detection/train.py @@ -0,0 +1,249 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + + +import argparse +import copy +import os +import os.path as osp +import time +import warnings + +import mmcv +import torch +import torch.distributed as dist +from mmcv import Config, DictAction +from mmcv.runner import get_dist_info, init_dist +from mmcv.utils import get_git_hash + +from mmdet import __version__ +from mmdet.apis import init_random_seed, set_random_seed, train_detector +from mmdet.datasets import build_dataset +from mmdet.models import build_detector +from mmdet.utils import (collect_env, get_device, get_root_logger, + replace_cfg_vals, setup_multi_processes, + update_data_root) +import mmcv_custom # noqa: F401,F403 +import mmdet_custom # noqa: F401,F403 + + +def parse_args(): + parser = argparse.ArgumentParser(description='Train a detector') + parser.add_argument('config', help='train config file path') + parser.add_argument('--work-dir', help='the dir to save logs and models') + parser.add_argument('--resume-from', + help='the checkpoint file to resume from') + parser.add_argument('--auto-resume', + action='store_true', + help='resume from the latest checkpoint automatically') + parser.add_argument( + '--no-validate', + action='store_true', + help='whether not to evaluate the checkpoint during training') + group_gpus = parser.add_mutually_exclusive_group() + group_gpus.add_argument( + '--gpus', + type=int, + help='(Deprecated, please use --gpu-id) number of gpus to use ' + '(only applicable to non-distributed training)') + group_gpus.add_argument( + '--gpu-ids', + type=int, + nargs='+', + help='(Deprecated, please use --gpu-id) ids of gpus to use ' + '(only applicable to non-distributed training)') + group_gpus.add_argument('--gpu-id', + type=int, + default=0, + help='id of gpu to use ' + '(only applicable to non-distributed training)') + parser.add_argument('--seed', type=int, default=None, help='random seed') + parser.add_argument( + '--diff-seed', + action='store_true', + help='Whether or not set different seeds for different ranks') + parser.add_argument( + '--deterministic', + action='store_true', + help='whether to set deterministic options for CUDNN backend.') + parser.add_argument( + '--options', + nargs='+', + action=DictAction, + help='override some settings in the used config, the key-value pair ' + 'in xxx=yyy format will be merged into config file (deprecate), ' + 'change to --cfg-options instead.') + parser.add_argument( + '--cfg-options', + nargs='+', + action=DictAction, + help='override some settings in the used config, the key-value pair ' + 'in xxx=yyy format will be merged into config file. If the value to ' + 'be overwritten is a list, it should be like key="[a,b]" or key=a,b ' + 'It also allows nested list/tuple values, e.g. key="[(a,b),(c,d)]" ' + 'Note that the quotation marks are necessary and that no white space ' + 'is allowed.') + parser.add_argument('--launcher', + choices=['none', 'pytorch', 'slurm', 'mpi'], + default='none', + help='job launcher') + parser.add_argument('--local_rank', type=int, default=0) + parser.add_argument('--auto-scale-lr', + action='store_true', + help='enable automatically scaling LR.') + args = parser.parse_args() + if 'LOCAL_RANK' not in os.environ: + os.environ['LOCAL_RANK'] = str(args.local_rank) + + if args.options and args.cfg_options: + raise ValueError( + '--options and --cfg-options cannot be both ' + 'specified, --options is deprecated in favor of --cfg-options') + if args.options: + warnings.warn('--options is deprecated in favor of --cfg-options') + args.cfg_options = args.options + + return args + + +def main(): + args = parse_args() + + cfg = Config.fromfile(args.config) + + # replace the ${key} with the value of cfg.key + cfg = replace_cfg_vals(cfg) + + # update data root according to MMDET_DATASETS + update_data_root(cfg) + + if args.cfg_options is not None: + cfg.merge_from_dict(args.cfg_options) + + if args.auto_scale_lr: + if 'auto_scale_lr' in cfg and \ + 'enable' in cfg.auto_scale_lr and \ + 'base_batch_size' in cfg.auto_scale_lr: + cfg.auto_scale_lr.enable = True + else: + warnings.warn('Can not find "auto_scale_lr" or ' + '"auto_scale_lr.enable" or ' + '"auto_scale_lr.base_batch_size" in your' + ' configuration file. Please update all the ' + 'configuration files to mmdet >= 2.24.1.') + + # set multi-process settings + setup_multi_processes(cfg) + + # set cudnn_benchmark + if cfg.get('cudnn_benchmark', False): + torch.backends.cudnn.benchmark = True + + # work_dir is determined in this priority: CLI > segment in file > filename + if args.work_dir is not None: + # update configs according to CLI args if args.work_dir is not None + cfg.work_dir = args.work_dir + elif cfg.get('work_dir', None) is None: + # use config filename as default work_dir if cfg.work_dir is None + cfg.work_dir = osp.join('./work_dirs', + osp.splitext(osp.basename(args.config))[0]) + + if args.resume_from is not None: + cfg.resume_from = args.resume_from + cfg.auto_resume = args.auto_resume + if args.gpus is not None: + cfg.gpu_ids = range(1) + warnings.warn('`--gpus` is deprecated because we only support ' + 'single GPU mode in non-distributed training. ' + 'Use `gpus=1` now.') + if args.gpu_ids is not None: + cfg.gpu_ids = args.gpu_ids[0:1] + warnings.warn('`--gpu-ids` is deprecated, please use `--gpu-id`. ' + 'Because we only support single GPU mode in ' + 'non-distributed training. Use the first GPU ' + 'in `gpu_ids` now.') + if args.gpus is None and args.gpu_ids is None: + cfg.gpu_ids = [args.gpu_id] + + # init distributed env first, since logger depends on the dist info. + if args.launcher == 'none': + distributed = False + else: + distributed = True + init_dist(args.launcher, **cfg.dist_params) + # re-set gpu_ids with distributed training mode + _, world_size = get_dist_info() + cfg.gpu_ids = range(world_size) + + # create work_dir + mmcv.mkdir_or_exist(osp.abspath(cfg.work_dir)) + # dump config + cfg.dump(osp.join(cfg.work_dir, osp.basename(args.config))) + # init the logger before other steps + timestamp = time.strftime('%Y%m%d_%H%M%S', time.localtime()) + log_file = osp.join(cfg.work_dir, f'{timestamp}.log') + logger = get_root_logger(log_file=log_file, log_level=cfg.log_level) + + # init the meta dict to record some important information such as + # environment info and seed, which will be logged + meta = dict() + # log env info + env_info_dict = collect_env() + env_info = '\n'.join([(f'{k}: {v}') for k, v in env_info_dict.items()]) + dash_line = '-' * 60 + '\n' + logger.info('Environment info:\n' + dash_line + env_info + '\n' + + dash_line) + meta['env_info'] = env_info + meta['config'] = cfg.pretty_text + # log some basic info + logger.info(f'Distributed training: {distributed}') + logger.info(f'Config:\n{cfg.pretty_text}') + + cfg.device = get_device() + # set random seeds + seed = init_random_seed(args.seed, device=cfg.device) + seed = seed + dist.get_rank() if args.diff_seed else seed + logger.info(f'Set random seed to {seed}, ' + f'deterministic: {args.deterministic}') + set_random_seed(seed, deterministic=args.deterministic) + cfg.seed = seed + meta['seed'] = seed + meta['exp_name'] = osp.basename(args.config) + + model = build_detector(cfg.model, + train_cfg=cfg.get('train_cfg'), + test_cfg=cfg.get('test_cfg')) + model.init_weights() + logger.info(model) + + datasets = [build_dataset(cfg.data.train)] + if len(cfg.workflow) == 2: + val_dataset = copy.deepcopy(cfg.data.val) + val_dataset.pipeline = cfg.data.train.pipeline + datasets.append(build_dataset(val_dataset)) + if cfg.checkpoint_config is not None: + # save mmdet version, config file content and class names in + # checkpoints as meta data + cfg.checkpoint_config.meta = dict(mmdet_version=__version__ + + get_git_hash()[:7], + CLASSES=datasets[0].CLASSES) + # add an attribute for visualization convenience + model.CLASSES = datasets[0].CLASSES + + torch.backends.cuda.matmul.allow_tf32 = True + torch.backends.cudnn.allow_tf32 = True + + train_detector(model, + datasets, + cfg, + distributed=distributed, + validate=(not args.no_validate), + timestamp=timestamp, + meta=meta) + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/segmentation/README.md b/segmentation/README.md new file mode 100644 index 0000000..213a140 --- /dev/null +++ b/segmentation/README.md @@ -0,0 +1,96 @@ +# FlashInternImage for Semantic Segmentation + +This folder contains the implementation of the InternImage for semantic segmentation. + +Our segmentation code is developed on top of [MMSegmentation v0.27.0](https://github.com/open-mmlab/mmsegmentation/tree/v0.27.0). + +## Usage + +### Install + +- Clone this repo: + +```bash +git clone https://github.com/OpenGVLab/DCNv4.git +cd DCNv4 +``` + +- Create a conda virtual environment and activate it: + +```bash +conda create -n dcnv4 python=3.7 -y +conda activate dcnv4 +``` + +- Install `CUDA>=10.2` with `cudnn>=7` following + the [official installation instructions](https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html) +- Install `PyTorch>=1.10.0` and `torchvision>=0.9.0` with `CUDA>=10.2`: + +For examples, to install torch==1.11 with CUDA==11.3 and nvcc: +```bash +conda install pytorch==1.11.0 torchvision==0.12.0 torchaudio==0.11.0 cudatoolkit=11.3 -c pytorch -y +conda install -c conda-forge cudatoolkit-dev=11.3 -y # to install nvcc +``` + +- Install other requirements: + + note: conda opencv will break torchvision as not to support GPU, so we need to install opencv using pip. + +```bash +conda install -c conda-forge termcolor yacs pyyaml scipy pip -y +pip install opencv-python +``` + +- Install `timm` and `mmcv-full` and `mmsegmentation': + +```bash +pip install -U openmim +mim install mmcv-full==1.5.0 +mim install mmsegmentation==0.27.0 +pip install timm==0.6.11 mmdet==2.28.1 +``` + +- Install DCNv4 +```bash +pip install DCNv4 +``` + +### Data Preparation + +Prepare datasets according to the [guidelines](https://github.com/open-mmlab/mmsegmentation/blob/master/docs/en/dataset_prepare.md#prepare-datasets) in MMSegmentation. + + +### Evaluation + +To evaluate our `FlashInternImage` on ADE20K val, run: + +```bash +sh dist_test.sh --eval mIoU +``` +You can download checkpoint files from [here](https://huggingface.co/OpenGVLab/DCNv4). Then place it to segmentation/checkpoint_dir/seg. + +For example, to evaluate the `FlashInternImage-T` with a single GPU: + +```bash +python test.py configs/ade20k/upernet_flash_internimage_t_512_160k_ade20k.py checkpoint_dir/seg/upernet_flash_internimage_t_512_160k_ade20k.pth --eval mIoU +``` + +For example, to evaluate the `FlashInternImage-B` with a single node with 8 GPUs: + +```bash +sh dist_test.sh configs/ade20k/upernet_flash_internimage_b_512_160k_ade20k.py checkpoint_dir/seg/upernet_flash_internimage_b_512_160k_ade20k.pth 8 --eval mIoU +``` + +### Training + +To train an `FlashInternImage` on ADE20K, run: + +```bash +sh dist_train.sh +``` + +For example, to train `FlashInternImage-T` with 8 GPU on 1 node (total batch size 16), run: + +```bash +sh dist_train.sh configs/ade20k/upernet_flash_internimage_t_512_160k_ade20k.py 8 +``` diff --git a/segmentation/configs/_base_/datasets/ade20k.py b/segmentation/configs/_base_/datasets/ade20k.py new file mode 100644 index 0000000..30f501a --- /dev/null +++ b/segmentation/configs/_base_/datasets/ade20k.py @@ -0,0 +1,54 @@ +# dataset settings +dataset_type = 'ADE20KDataset' +data_root = 'data/ADEChallengeData2016' +img_norm_cfg = dict( + mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) +crop_size = (512, 512) +train_pipeline = [ + dict(type='LoadImageFromFile'), + dict(type='LoadAnnotations', reduce_zero_label=True), + dict(type='Resize', img_scale=(2048, 512), ratio_range=(0.5, 2.0)), + dict(type='RandomCrop', crop_size=crop_size, cat_max_ratio=0.75), + dict(type='RandomFlip', prob=0.5), + dict(type='PhotoMetricDistortion'), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size=crop_size, pad_val=0, seg_pad_val=255), + dict(type='DefaultFormatBundle'), + dict(type='Collect', keys=['img', 'gt_semantic_seg']), +] +test_pipeline = [ + dict(type='LoadImageFromFile'), + dict( + type='MultiScaleFlipAug', + img_scale=(2048, 512), + # img_ratios=[0.5, 0.75, 1.0, 1.25, 1.5, 1.75], + flip=False, + transforms=[ + dict(type='Resize', keep_ratio=True), + dict(type='RandomFlip'), + dict(type='Normalize', **img_norm_cfg), + dict(type='ImageToTensor', keys=['img']), + dict(type='Collect', keys=['img']), + ]) +] +data = dict( + samples_per_gpu=4, + workers_per_gpu=4, + train=dict( + type=dataset_type, + data_root=data_root, + img_dir='images/training', + ann_dir='annotations/training', + pipeline=train_pipeline), + val=dict( + type=dataset_type, + data_root=data_root, + img_dir='images/validation', + ann_dir='annotations/validation', + pipeline=test_pipeline), + test=dict( + type=dataset_type, + data_root=data_root, + img_dir='images/validation', + ann_dir='annotations/validation', + pipeline=test_pipeline)) diff --git a/segmentation/configs/_base_/datasets/ade20k_640x640.py b/segmentation/configs/_base_/datasets/ade20k_640x640.py new file mode 100644 index 0000000..f53e81a --- /dev/null +++ b/segmentation/configs/_base_/datasets/ade20k_640x640.py @@ -0,0 +1,54 @@ +# dataset settings +dataset_type = 'ADE20KDataset' +data_root = 'data/ADEChallengeData2016' +img_norm_cfg = dict( + mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) +crop_size = (640, 640) +train_pipeline = [ + dict(type='LoadImageFromFile'), + dict(type='LoadAnnotations', reduce_zero_label=True), + dict(type='Resize', img_scale=(2560, 640), ratio_range=(0.5, 2.0)), + dict(type='RandomCrop', crop_size=crop_size, cat_max_ratio=0.75), + dict(type='RandomFlip', prob=0.5), + dict(type='PhotoMetricDistortion'), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size=crop_size, pad_val=0, seg_pad_val=255), + dict(type='DefaultFormatBundle'), + dict(type='Collect', keys=['img', 'gt_semantic_seg']), +] +test_pipeline = [ + dict(type='LoadImageFromFile'), + dict( + type='MultiScaleFlipAug', + img_scale=(2560, 640), + # img_ratios=[0.5, 0.75, 1.0, 1.25, 1.5, 1.75], + flip=False, + transforms=[ + dict(type='Resize', keep_ratio=True), + dict(type='RandomFlip'), + dict(type='Normalize', **img_norm_cfg), + dict(type='ImageToTensor', keys=['img']), + dict(type='Collect', keys=['img']), + ]) +] +data = dict( + samples_per_gpu=4, + workers_per_gpu=4, + train=dict( + type=dataset_type, + data_root=data_root, + img_dir='images/training', + ann_dir='annotations/training', + pipeline=train_pipeline), + val=dict( + type=dataset_type, + data_root=data_root, + img_dir='images/validation', + ann_dir='annotations/validation', + pipeline=test_pipeline), + test=dict( + type=dataset_type, + data_root=data_root, + img_dir='images/validation', + ann_dir='annotations/validation', + pipeline=test_pipeline)) diff --git a/segmentation/configs/_base_/datasets/chase_db1.py b/segmentation/configs/_base_/datasets/chase_db1.py new file mode 100644 index 0000000..298594e --- /dev/null +++ b/segmentation/configs/_base_/datasets/chase_db1.py @@ -0,0 +1,59 @@ +# dataset settings +dataset_type = 'ChaseDB1Dataset' +data_root = 'data/CHASE_DB1' +img_norm_cfg = dict( + mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) +img_scale = (960, 999) +crop_size = (128, 128) +train_pipeline = [ + dict(type='LoadImageFromFile'), + dict(type='LoadAnnotations'), + dict(type='Resize', img_scale=img_scale, ratio_range=(0.5, 2.0)), + dict(type='RandomCrop', crop_size=crop_size, cat_max_ratio=0.75), + dict(type='RandomFlip', prob=0.5), + dict(type='PhotoMetricDistortion'), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size=crop_size, pad_val=0, seg_pad_val=255), + dict(type='DefaultFormatBundle'), + dict(type='Collect', keys=['img', 'gt_semantic_seg']) +] +test_pipeline = [ + dict(type='LoadImageFromFile'), + dict( + type='MultiScaleFlipAug', + img_scale=img_scale, + # img_ratios=[0.5, 0.75, 1.0, 1.25, 1.5, 1.75, 2.0], + flip=False, + transforms=[ + dict(type='Resize', keep_ratio=True), + dict(type='RandomFlip'), + dict(type='Normalize', **img_norm_cfg), + dict(type='ImageToTensor', keys=['img']), + dict(type='Collect', keys=['img']) + ]) +] + +data = dict( + samples_per_gpu=4, + workers_per_gpu=4, + train=dict( + type='RepeatDataset', + times=40000, + dataset=dict( + type=dataset_type, + data_root=data_root, + img_dir='images/training', + ann_dir='annotations/training', + pipeline=train_pipeline)), + val=dict( + type=dataset_type, + data_root=data_root, + img_dir='images/validation', + ann_dir='annotations/validation', + pipeline=test_pipeline), + test=dict( + type=dataset_type, + data_root=data_root, + img_dir='images/validation', + ann_dir='annotations/validation', + pipeline=test_pipeline)) diff --git a/segmentation/configs/_base_/datasets/cityscapes.py b/segmentation/configs/_base_/datasets/cityscapes.py new file mode 100644 index 0000000..f21867c --- /dev/null +++ b/segmentation/configs/_base_/datasets/cityscapes.py @@ -0,0 +1,54 @@ +# dataset settings +dataset_type = 'CityscapesDataset' +data_root = 'data/cityscapes/' +img_norm_cfg = dict( + mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) +crop_size = (512, 1024) +train_pipeline = [ + dict(type='LoadImageFromFile'), + dict(type='LoadAnnotations'), + dict(type='Resize', img_scale=(2048, 1024), ratio_range=(0.5, 2.0)), + dict(type='RandomCrop', crop_size=crop_size, cat_max_ratio=0.75), + dict(type='RandomFlip', prob=0.5), + dict(type='PhotoMetricDistortion'), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size=crop_size, pad_val=0, seg_pad_val=255), + dict(type='DefaultFormatBundle'), + dict(type='Collect', keys=['img', 'gt_semantic_seg']), +] +test_pipeline = [ + dict(type='LoadImageFromFile'), + dict( + type='MultiScaleFlipAug', + img_scale=(2048, 1024), + # img_ratios=[0.5, 0.75, 1.0, 1.25, 1.5, 1.75], + flip=False, + transforms=[ + dict(type='Resize', keep_ratio=True), + dict(type='RandomFlip'), + dict(type='Normalize', **img_norm_cfg), + dict(type='ImageToTensor', keys=['img']), + dict(type='Collect', keys=['img']), + ]) +] +data = dict( + samples_per_gpu=2, + workers_per_gpu=2, + train=dict( + type=dataset_type, + data_root=data_root, + img_dir='leftImg8bit/train', + ann_dir='gtFine/train', + pipeline=train_pipeline), + val=dict( + type=dataset_type, + data_root=data_root, + img_dir='leftImg8bit/val', + ann_dir='gtFine/val', + pipeline=test_pipeline), + test=dict( + type=dataset_type, + data_root=data_root, + img_dir='leftImg8bit/val', + ann_dir='gtFine/val', + pipeline=test_pipeline)) diff --git a/segmentation/configs/_base_/datasets/cityscapes_1024x1024.py b/segmentation/configs/_base_/datasets/cityscapes_1024x1024.py new file mode 100644 index 0000000..f98d929 --- /dev/null +++ b/segmentation/configs/_base_/datasets/cityscapes_1024x1024.py @@ -0,0 +1,35 @@ +_base_ = './cityscapes.py' +img_norm_cfg = dict( + mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) +crop_size = (1024, 1024) +train_pipeline = [ + dict(type='LoadImageFromFile'), + dict(type='LoadAnnotations'), + dict(type='Resize', img_scale=(2048, 1024), ratio_range=(0.5, 2.0)), + dict(type='RandomCrop', crop_size=crop_size, cat_max_ratio=0.75), + dict(type='RandomFlip', prob=0.5), + dict(type='PhotoMetricDistortion'), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size=crop_size, pad_val=0, seg_pad_val=255), + dict(type='DefaultFormatBundle'), + dict(type='Collect', keys=['img', 'gt_semantic_seg']), +] +test_pipeline = [ + dict(type='LoadImageFromFile'), + dict( + type='MultiScaleFlipAug', + img_scale=(2048, 1024), + # img_ratios=[0.5, 0.75, 1.0, 1.25, 1.5, 1.75], + flip=False, + transforms=[ + dict(type='Resize', keep_ratio=True), + dict(type='RandomFlip'), + dict(type='Normalize', **img_norm_cfg), + dict(type='ImageToTensor', keys=['img']), + dict(type='Collect', keys=['img']), + ]) +] +data = dict( + train=dict(pipeline=train_pipeline), + val=dict(pipeline=test_pipeline), + test=dict(pipeline=test_pipeline)) diff --git a/segmentation/configs/_base_/datasets/cityscapes_extra.py b/segmentation/configs/_base_/datasets/cityscapes_extra.py new file mode 100644 index 0000000..031df1b --- /dev/null +++ b/segmentation/configs/_base_/datasets/cityscapes_extra.py @@ -0,0 +1,54 @@ +# dataset settings +dataset_type = 'CityscapesDataset' +data_root = 'data/cityscapes/' +img_norm_cfg = dict( + mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) +crop_size = (512, 1024) +train_pipeline = [ + dict(type='LoadImageFromFile'), + dict(type='LoadAnnotations'), + dict(type='Resize', img_scale=(2048, 1024), ratio_range=(0.5, 2.0)), + dict(type='RandomCrop', crop_size=crop_size, cat_max_ratio=0.75), + dict(type='RandomFlip', prob=0.5), + dict(type='PhotoMetricDistortion'), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size=crop_size, pad_val=0, seg_pad_val=255), + dict(type='DefaultFormatBundle'), + dict(type='Collect', keys=['img', 'gt_semantic_seg']), +] +test_pipeline = [ + dict(type='LoadImageFromFile'), + dict( + type='MultiScaleFlipAug', + img_scale=(2048, 1024), + # img_ratios=[0.5, 0.75, 1.0, 1.25, 1.5, 1.75], + flip=False, + transforms=[ + dict(type='Resize', keep_ratio=True), + dict(type='RandomFlip'), + dict(type='Normalize', **img_norm_cfg), + dict(type='ImageToTensor', keys=['img']), + dict(type='Collect', keys=['img']), + ]) +] +data = dict( + samples_per_gpu=2, + workers_per_gpu=2, + train=dict( + type=dataset_type, + data_root=data_root, + img_dir=['leftImg8bit/train', 'leftImg8bit/train_extra'], + ann_dir=['gtFine/train', 'refinement_final/train_extra'], + pipeline=train_pipeline), + val=dict( + type=dataset_type, + data_root=data_root, + img_dir='leftImg8bit/val', + ann_dir='gtFine/val', + pipeline=test_pipeline), + test=dict( + type=dataset_type, + data_root=data_root, + img_dir='leftImg8bit/val', + ann_dir='gtFine/val', + pipeline=test_pipeline)) diff --git a/segmentation/configs/_base_/datasets/coco-stuff10k.py b/segmentation/configs/_base_/datasets/coco-stuff10k.py new file mode 100644 index 0000000..ec04969 --- /dev/null +++ b/segmentation/configs/_base_/datasets/coco-stuff10k.py @@ -0,0 +1,57 @@ +# dataset settings +dataset_type = 'COCOStuffDataset' +data_root = 'data/coco_stuff10k' +img_norm_cfg = dict( + mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) +crop_size = (512, 512) +train_pipeline = [ + dict(type='LoadImageFromFile'), + dict(type='LoadAnnotations', reduce_zero_label=True), + dict(type='Resize', img_scale=(2048, 512), ratio_range=(0.5, 2.0)), + dict(type='RandomCrop', crop_size=crop_size, cat_max_ratio=0.75), + dict(type='RandomFlip', prob=0.5), + dict(type='PhotoMetricDistortion'), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size=crop_size, pad_val=0, seg_pad_val=255), + dict(type='DefaultFormatBundle'), + dict(type='Collect', keys=['img', 'gt_semantic_seg']), +] +test_pipeline = [ + dict(type='LoadImageFromFile'), + dict( + type='MultiScaleFlipAug', + img_scale=(2048, 512), + # img_ratios=[0.5, 0.75, 1.0, 1.25, 1.5, 1.75], + flip=False, + transforms=[ + dict(type='Resize', keep_ratio=True), + dict(type='RandomFlip'), + dict(type='Normalize', **img_norm_cfg), + dict(type='ImageToTensor', keys=['img']), + dict(type='Collect', keys=['img']), + ]) +] +data = dict( + samples_per_gpu=4, + workers_per_gpu=4, + train=dict( + type=dataset_type, + data_root=data_root, + reduce_zero_label=True, + img_dir='images/train2014', + ann_dir='annotations/train2014', + pipeline=train_pipeline), + val=dict( + type=dataset_type, + data_root=data_root, + reduce_zero_label=True, + img_dir='images/test2014', + ann_dir='annotations/test2014', + pipeline=test_pipeline), + test=dict( + type=dataset_type, + data_root=data_root, + reduce_zero_label=True, + img_dir='images/test2014', + ann_dir='annotations/test2014', + pipeline=test_pipeline)) diff --git a/segmentation/configs/_base_/datasets/coco-stuff164k.py b/segmentation/configs/_base_/datasets/coco-stuff164k.py new file mode 100644 index 0000000..a6a38f2 --- /dev/null +++ b/segmentation/configs/_base_/datasets/coco-stuff164k.py @@ -0,0 +1,54 @@ +# dataset settings +dataset_type = 'COCOStuffDataset' +data_root = 'data/coco_stuff164k' +img_norm_cfg = dict( + mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) +crop_size = (512, 512) +train_pipeline = [ + dict(type='LoadImageFromFile'), + dict(type='LoadAnnotations'), + dict(type='Resize', img_scale=(2048, 512), ratio_range=(0.5, 2.0)), + dict(type='RandomCrop', crop_size=crop_size, cat_max_ratio=0.75), + dict(type='RandomFlip', prob=0.5), + dict(type='PhotoMetricDistortion'), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size=crop_size, pad_val=0, seg_pad_val=255), + dict(type='DefaultFormatBundle'), + dict(type='Collect', keys=['img', 'gt_semantic_seg']), +] +test_pipeline = [ + dict(type='LoadImageFromFile'), + dict( + type='MultiScaleFlipAug', + img_scale=(2048, 512), + # img_ratios=[0.5, 0.75, 1.0, 1.25, 1.5, 1.75], + flip=False, + transforms=[ + dict(type='Resize', keep_ratio=True), + dict(type='RandomFlip'), + dict(type='Normalize', **img_norm_cfg), + dict(type='ImageToTensor', keys=['img']), + dict(type='Collect', keys=['img']), + ]) +] +data = dict( + samples_per_gpu=4, + workers_per_gpu=4, + train=dict( + type=dataset_type, + data_root=data_root, + img_dir='images/train2017', + ann_dir='annotations/train2017', + pipeline=train_pipeline), + val=dict( + type=dataset_type, + data_root=data_root, + img_dir='images/val2017', + ann_dir='annotations/val2017', + pipeline=test_pipeline), + test=dict( + type=dataset_type, + data_root=data_root, + img_dir='images/val2017', + ann_dir='annotations/val2017', + pipeline=test_pipeline)) diff --git a/segmentation/configs/_base_/datasets/drive.py b/segmentation/configs/_base_/datasets/drive.py new file mode 100644 index 0000000..06e8ff6 --- /dev/null +++ b/segmentation/configs/_base_/datasets/drive.py @@ -0,0 +1,59 @@ +# dataset settings +dataset_type = 'DRIVEDataset' +data_root = 'data/DRIVE' +img_norm_cfg = dict( + mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) +img_scale = (584, 565) +crop_size = (64, 64) +train_pipeline = [ + dict(type='LoadImageFromFile'), + dict(type='LoadAnnotations'), + dict(type='Resize', img_scale=img_scale, ratio_range=(0.5, 2.0)), + dict(type='RandomCrop', crop_size=crop_size, cat_max_ratio=0.75), + dict(type='RandomFlip', prob=0.5), + dict(type='PhotoMetricDistortion'), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size=crop_size, pad_val=0, seg_pad_val=255), + dict(type='DefaultFormatBundle'), + dict(type='Collect', keys=['img', 'gt_semantic_seg']) +] +test_pipeline = [ + dict(type='LoadImageFromFile'), + dict( + type='MultiScaleFlipAug', + img_scale=img_scale, + # img_ratios=[0.5, 0.75, 1.0, 1.25, 1.5, 1.75, 2.0], + flip=False, + transforms=[ + dict(type='Resize', keep_ratio=True), + dict(type='RandomFlip'), + dict(type='Normalize', **img_norm_cfg), + dict(type='ImageToTensor', keys=['img']), + dict(type='Collect', keys=['img']) + ]) +] + +data = dict( + samples_per_gpu=4, + workers_per_gpu=4, + train=dict( + type='RepeatDataset', + times=40000, + dataset=dict( + type=dataset_type, + data_root=data_root, + img_dir='images/training', + ann_dir='annotations/training', + pipeline=train_pipeline)), + val=dict( + type=dataset_type, + data_root=data_root, + img_dir='images/validation', + ann_dir='annotations/validation', + pipeline=test_pipeline), + test=dict( + type=dataset_type, + data_root=data_root, + img_dir='images/validation', + ann_dir='annotations/validation', + pipeline=test_pipeline)) diff --git a/segmentation/configs/_base_/datasets/hrf.py b/segmentation/configs/_base_/datasets/hrf.py new file mode 100644 index 0000000..242d790 --- /dev/null +++ b/segmentation/configs/_base_/datasets/hrf.py @@ -0,0 +1,59 @@ +# dataset settings +dataset_type = 'HRFDataset' +data_root = 'data/HRF' +img_norm_cfg = dict( + mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) +img_scale = (2336, 3504) +crop_size = (256, 256) +train_pipeline = [ + dict(type='LoadImageFromFile'), + dict(type='LoadAnnotations'), + dict(type='Resize', img_scale=img_scale, ratio_range=(0.5, 2.0)), + dict(type='RandomCrop', crop_size=crop_size, cat_max_ratio=0.75), + dict(type='RandomFlip', prob=0.5), + dict(type='PhotoMetricDistortion'), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size=crop_size, pad_val=0, seg_pad_val=255), + dict(type='DefaultFormatBundle'), + dict(type='Collect', keys=['img', 'gt_semantic_seg']) +] +test_pipeline = [ + dict(type='LoadImageFromFile'), + dict( + type='MultiScaleFlipAug', + img_scale=img_scale, + # img_ratios=[0.5, 0.75, 1.0, 1.25, 1.5, 1.75, 2.0], + flip=False, + transforms=[ + dict(type='Resize', keep_ratio=True), + dict(type='RandomFlip'), + dict(type='Normalize', **img_norm_cfg), + dict(type='ImageToTensor', keys=['img']), + dict(type='Collect', keys=['img']) + ]) +] + +data = dict( + samples_per_gpu=4, + workers_per_gpu=4, + train=dict( + type='RepeatDataset', + times=40000, + dataset=dict( + type=dataset_type, + data_root=data_root, + img_dir='images/training', + ann_dir='annotations/training', + pipeline=train_pipeline)), + val=dict( + type=dataset_type, + data_root=data_root, + img_dir='images/validation', + ann_dir='annotations/validation', + pipeline=test_pipeline), + test=dict( + type=dataset_type, + data_root=data_root, + img_dir='images/validation', + ann_dir='annotations/validation', + pipeline=test_pipeline)) diff --git a/segmentation/configs/_base_/datasets/loveda.py b/segmentation/configs/_base_/datasets/loveda.py new file mode 100644 index 0000000..e553356 --- /dev/null +++ b/segmentation/configs/_base_/datasets/loveda.py @@ -0,0 +1,54 @@ +# dataset settings +dataset_type = 'LoveDADataset' +data_root = 'data/loveDA' +img_norm_cfg = dict( + mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) +crop_size = (512, 512) +train_pipeline = [ + dict(type='LoadImageFromFile'), + dict(type='LoadAnnotations', reduce_zero_label=True), + dict(type='Resize', img_scale=(2048, 512), ratio_range=(0.5, 2.0)), + dict(type='RandomCrop', crop_size=crop_size, cat_max_ratio=0.75), + dict(type='RandomFlip', prob=0.5), + dict(type='PhotoMetricDistortion'), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size=crop_size, pad_val=0, seg_pad_val=255), + dict(type='DefaultFormatBundle'), + dict(type='Collect', keys=['img', 'gt_semantic_seg']), +] +test_pipeline = [ + dict(type='LoadImageFromFile'), + dict( + type='MultiScaleFlipAug', + img_scale=(1024, 1024), + # img_ratios=[0.5, 0.75, 1.0, 1.25, 1.5, 1.75], + flip=False, + transforms=[ + dict(type='Resize', keep_ratio=True), + dict(type='RandomFlip'), + dict(type='Normalize', **img_norm_cfg), + dict(type='ImageToTensor', keys=['img']), + dict(type='Collect', keys=['img']), + ]) +] +data = dict( + samples_per_gpu=4, + workers_per_gpu=4, + train=dict( + type=dataset_type, + data_root=data_root, + img_dir='img_dir/train', + ann_dir='ann_dir/train', + pipeline=train_pipeline), + val=dict( + type=dataset_type, + data_root=data_root, + img_dir='img_dir/val', + ann_dir='ann_dir/val', + pipeline=test_pipeline), + test=dict( + type=dataset_type, + data_root=data_root, + img_dir='img_dir/val', + ann_dir='ann_dir/val', + pipeline=test_pipeline)) diff --git a/segmentation/configs/_base_/datasets/mapillary.py b/segmentation/configs/_base_/datasets/mapillary.py new file mode 100644 index 0000000..3e62497 --- /dev/null +++ b/segmentation/configs/_base_/datasets/mapillary.py @@ -0,0 +1,55 @@ +# dataset settings +dataset_type = 'MapillaryDataset' +data_root = 'data/Mapillary/' +img_norm_cfg = dict( + mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) +crop_size = (512, 1024) +train_pipeline = [ + dict(type='LoadImageFromFile'), + dict(type='LoadAnnotations'), + dict(type='MapillaryHack'), + dict(type='Resize', img_scale=(2048, 1024), ratio_range=(0.5, 1.0)), + dict(type='RandomCrop', crop_size=crop_size, cat_max_ratio=0.75), + dict(type='RandomFlip', prob=0.5), + dict(type='PhotoMetricDistortion'), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size=crop_size, pad_val=0, seg_pad_val=255), + dict(type='DefaultFormatBundle'), + dict(type='Collect', keys=['img', 'gt_semantic_seg']), +] +test_pipeline = [ + dict(type='LoadImageFromFile'), + dict( + type='MultiScaleFlipAug', + img_scale=(2048, 1024), + # img_ratios=[0.5, 0.75, 1.0, 1.25, 1.5, 1.75], + flip=False, + transforms=[ + dict(type='Resize', keep_ratio=True), + dict(type='RandomFlip'), + dict(type='Normalize', **img_norm_cfg), + dict(type='ImageToTensor', keys=['img']), + dict(type='Collect', keys=['img']), + ]) +] +data = dict( + samples_per_gpu=2, + workers_per_gpu=2, + train=dict( + type=dataset_type, + data_root='data/Mapillary/', + img_dir=['training/images', 'validation/images'], + ann_dir=['training/labels', 'validation/labels'], + pipeline=train_pipeline), + val=dict( + type='CityscapesDataset', + data_root='data/cityscapes/', + img_dir='leftImg8bit/val', + ann_dir='gtFine/val', + pipeline=test_pipeline), + test=dict( + type='CityscapesDataset', + data_root='data/cityscapes/', + img_dir='leftImg8bit/val', + ann_dir='gtFine/val', + pipeline=test_pipeline)) diff --git a/segmentation/configs/_base_/datasets/mapillary_1024x1024.py b/segmentation/configs/_base_/datasets/mapillary_1024x1024.py new file mode 100644 index 0000000..1c81ea2 --- /dev/null +++ b/segmentation/configs/_base_/datasets/mapillary_1024x1024.py @@ -0,0 +1,55 @@ +# dataset settings +dataset_type = 'MapillaryDataset' +data_root = 'data/Mapillary/' +img_norm_cfg = dict( + mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) +crop_size = (1024, 1024) +train_pipeline = [ + dict(type='LoadImageFromFile'), + dict(type='LoadAnnotations'), + dict(type='MapillaryHack'), + dict(type='Resize', img_scale=(2048, 1024), ratio_range=(0.5, 1.0)), + dict(type='RandomCrop', crop_size=crop_size, cat_max_ratio=0.75), + dict(type='RandomFlip', prob=0.5), + dict(type='PhotoMetricDistortion'), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size=crop_size, pad_val=0, seg_pad_val=255), + dict(type='DefaultFormatBundle'), + dict(type='Collect', keys=['img', 'gt_semantic_seg']), +] +test_pipeline = [ + dict(type='LoadImageFromFile'), + dict( + type='MultiScaleFlipAug', + img_scale=(2048, 1024), + # img_ratios=[0.5, 0.75, 1.0, 1.25, 1.5, 1.75], + flip=False, + transforms=[ + dict(type='Resize', keep_ratio=True), + dict(type='RandomFlip'), + dict(type='Normalize', **img_norm_cfg), + dict(type='ImageToTensor', keys=['img']), + dict(type='Collect', keys=['img']), + ]) +] +data = dict( + samples_per_gpu=2, + workers_per_gpu=2, + train=dict( + type=dataset_type, + data_root='data/Mapillary/', + img_dir=['training/images', 'validation/images'], + ann_dir=['training/labels', 'validation/labels'], + pipeline=train_pipeline), + val=dict( + type='CityscapesDataset', + data_root='data/cityscapes/', + img_dir='leftImg8bit/val', + ann_dir='gtFine/val', + pipeline=test_pipeline), + test=dict( + type='CityscapesDataset', + data_root='data/cityscapes/', + img_dir='leftImg8bit/val', + ann_dir='gtFine/val', + pipeline=test_pipeline)) diff --git a/segmentation/configs/_base_/datasets/nyu_depth_v2.py b/segmentation/configs/_base_/datasets/nyu_depth_v2.py new file mode 100644 index 0000000..bac9443 --- /dev/null +++ b/segmentation/configs/_base_/datasets/nyu_depth_v2.py @@ -0,0 +1,59 @@ +# dataset settings +dataset_type = 'NYUDepthV2Dataset' +data_root = 'data/nyu_depth_v2/' +img_norm_cfg = dict( + mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) + +crop_size = (480, 480) + +train_pipeline = [ + dict(type='LoadImageFromFile'), + dict(type='LoadAnnotations', reduce_zero_label=True), + dict(type='Resize', img_scale=(640, 480), ratio_range=(0.5, 2.0)), + dict(type='RandomCrop', crop_size=crop_size, cat_max_ratio=0.75), + dict(type='RandomFlip', prob=0.5), + dict(type='PhotoMetricDistortion'), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size=crop_size, pad_val=0, seg_pad_val=255), + dict(type='DefaultFormatBundle'), + dict(type='Collect', keys=['img', 'gt_semantic_seg']), +] +test_pipeline = [ + dict(type='LoadImageFromFile'), + dict( + type='MultiScaleFlipAug', + img_scale=(640, 480), + # img_ratios=[0.5, 0.75, 1.0, 1.25, 1.5, 1.75], + flip=False, + transforms=[ + dict(type='Resize', keep_ratio=True), + dict(type='RandomFlip'), + dict(type='Normalize', **img_norm_cfg), + dict(type='ImageToTensor', keys=['img']), + dict(type='Collect', keys=['img']), + ]) +] +data = dict( + samples_per_gpu=4, + workers_per_gpu=4, + train=dict( + type=dataset_type, + data_root=data_root, + img_dir='image', + ann_dir='label40', + split='train.txt', + pipeline=train_pipeline), + val=dict( + type=dataset_type, + data_root=data_root, + img_dir='image', + ann_dir='label40', + split='test.txt', + pipeline=test_pipeline), + test=dict( + type=dataset_type, + data_root=data_root, + img_dir='image', + ann_dir='label40', + split='test.txt', + pipeline=test_pipeline)) diff --git a/segmentation/configs/_base_/datasets/pascal_context.py b/segmentation/configs/_base_/datasets/pascal_context.py new file mode 100644 index 0000000..ff65bad --- /dev/null +++ b/segmentation/configs/_base_/datasets/pascal_context.py @@ -0,0 +1,60 @@ +# dataset settings +dataset_type = 'PascalContextDataset' +data_root = 'data/VOCdevkit/VOC2010/' +img_norm_cfg = dict( + mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) + +img_scale = (520, 520) +crop_size = (480, 480) + +train_pipeline = [ + dict(type='LoadImageFromFile'), + dict(type='LoadAnnotations'), + dict(type='Resize', img_scale=img_scale, ratio_range=(0.5, 2.0)), + dict(type='RandomCrop', crop_size=crop_size, cat_max_ratio=0.75), + dict(type='RandomFlip', prob=0.5), + dict(type='PhotoMetricDistortion'), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size=crop_size, pad_val=0, seg_pad_val=255), + dict(type='DefaultFormatBundle'), + dict(type='Collect', keys=['img', 'gt_semantic_seg']), +] +test_pipeline = [ + dict(type='LoadImageFromFile'), + dict( + type='MultiScaleFlipAug', + img_scale=img_scale, + # img_ratios=[0.5, 0.75, 1.0, 1.25, 1.5, 1.75], + flip=False, + transforms=[ + dict(type='Resize', keep_ratio=True), + dict(type='RandomFlip'), + dict(type='Normalize', **img_norm_cfg), + dict(type='ImageToTensor', keys=['img']), + dict(type='Collect', keys=['img']), + ]) +] +data = dict( + samples_per_gpu=4, + workers_per_gpu=4, + train=dict( + type=dataset_type, + data_root=data_root, + img_dir='JPEGImages', + ann_dir='SegmentationClassContext', + split='ImageSets/SegmentationContext/train.txt', + pipeline=train_pipeline), + val=dict( + type=dataset_type, + data_root=data_root, + img_dir='JPEGImages', + ann_dir='SegmentationClassContext', + split='ImageSets/SegmentationContext/val.txt', + pipeline=test_pipeline), + test=dict( + type=dataset_type, + data_root=data_root, + img_dir='JPEGImages', + ann_dir='SegmentationClassContext', + split='ImageSets/SegmentationContext/val.txt', + pipeline=test_pipeline)) diff --git a/segmentation/configs/_base_/datasets/pascal_context_59.py b/segmentation/configs/_base_/datasets/pascal_context_59.py new file mode 100644 index 0000000..37585ab --- /dev/null +++ b/segmentation/configs/_base_/datasets/pascal_context_59.py @@ -0,0 +1,60 @@ +# dataset settings +dataset_type = 'PascalContextDataset59' +data_root = 'data/VOCdevkit/VOC2010/' +img_norm_cfg = dict( + mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) + +img_scale = (520, 520) +crop_size = (480, 480) + +train_pipeline = [ + dict(type='LoadImageFromFile'), + dict(type='LoadAnnotations', reduce_zero_label=True), + dict(type='Resize', img_scale=img_scale, ratio_range=(0.5, 2.0)), + dict(type='RandomCrop', crop_size=crop_size, cat_max_ratio=0.75), + dict(type='RandomFlip', prob=0.5), + dict(type='PhotoMetricDistortion'), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size=crop_size, pad_val=0, seg_pad_val=255), + dict(type='DefaultFormatBundle'), + dict(type='Collect', keys=['img', 'gt_semantic_seg']), +] +test_pipeline = [ + dict(type='LoadImageFromFile'), + dict( + type='MultiScaleFlipAug', + img_scale=img_scale, + # img_ratios=[0.5, 0.75, 1.0, 1.25, 1.5, 1.75], + flip=False, + transforms=[ + dict(type='Resize', keep_ratio=True), + dict(type='RandomFlip'), + dict(type='Normalize', **img_norm_cfg), + dict(type='ImageToTensor', keys=['img']), + dict(type='Collect', keys=['img']), + ]) +] +data = dict( + samples_per_gpu=4, + workers_per_gpu=4, + train=dict( + type=dataset_type, + data_root=data_root, + img_dir='JPEGImages', + ann_dir='SegmentationClassContext', + split='ImageSets/SegmentationContext/train.txt', + pipeline=train_pipeline), + val=dict( + type=dataset_type, + data_root=data_root, + img_dir='JPEGImages', + ann_dir='SegmentationClassContext', + split='ImageSets/SegmentationContext/val.txt', + pipeline=test_pipeline), + test=dict( + type=dataset_type, + data_root=data_root, + img_dir='JPEGImages', + ann_dir='SegmentationClassContext', + split='ImageSets/SegmentationContext/val.txt', + pipeline=test_pipeline)) diff --git a/segmentation/configs/_base_/datasets/pascal_voc12.py b/segmentation/configs/_base_/datasets/pascal_voc12.py new file mode 100644 index 0000000..ba1d42d --- /dev/null +++ b/segmentation/configs/_base_/datasets/pascal_voc12.py @@ -0,0 +1,57 @@ +# dataset settings +dataset_type = 'PascalVOCDataset' +data_root = 'data/VOCdevkit/VOC2012' +img_norm_cfg = dict( + mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) +crop_size = (512, 512) +train_pipeline = [ + dict(type='LoadImageFromFile'), + dict(type='LoadAnnotations'), + dict(type='Resize', img_scale=(2048, 512), ratio_range=(0.5, 2.0)), + dict(type='RandomCrop', crop_size=crop_size, cat_max_ratio=0.75), + dict(type='RandomFlip', prob=0.5), + dict(type='PhotoMetricDistortion'), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size=crop_size, pad_val=0, seg_pad_val=255), + dict(type='DefaultFormatBundle'), + dict(type='Collect', keys=['img', 'gt_semantic_seg']), +] +test_pipeline = [ + dict(type='LoadImageFromFile'), + dict( + type='MultiScaleFlipAug', + img_scale=(2048, 512), + # img_ratios=[0.5, 0.75, 1.0, 1.25, 1.5, 1.75], + flip=False, + transforms=[ + dict(type='Resize', keep_ratio=True), + dict(type='RandomFlip'), + dict(type='Normalize', **img_norm_cfg), + dict(type='ImageToTensor', keys=['img']), + dict(type='Collect', keys=['img']), + ]) +] +data = dict( + samples_per_gpu=4, + workers_per_gpu=4, + train=dict( + type=dataset_type, + data_root=data_root, + img_dir='JPEGImages', + ann_dir='SegmentationClass', + split='ImageSets/Segmentation/train.txt', + pipeline=train_pipeline), + val=dict( + type=dataset_type, + data_root=data_root, + img_dir='JPEGImages', + ann_dir='SegmentationClass', + split='ImageSets/Segmentation/val.txt', + pipeline=test_pipeline), + test=dict( + type=dataset_type, + data_root=data_root, + img_dir='JPEGImages', + ann_dir='SegmentationClass', + split='ImageSets/Segmentation/val.txt', + pipeline=test_pipeline)) diff --git a/segmentation/configs/_base_/datasets/pascal_voc12_aug.py b/segmentation/configs/_base_/datasets/pascal_voc12_aug.py new file mode 100644 index 0000000..3f23b67 --- /dev/null +++ b/segmentation/configs/_base_/datasets/pascal_voc12_aug.py @@ -0,0 +1,9 @@ +_base_ = './pascal_voc12.py' +# dataset settings +data = dict( + train=dict( + ann_dir=['SegmentationClass', 'SegmentationClassAug'], + split=[ + 'ImageSets/Segmentation/train.txt', + 'ImageSets/Segmentation/aug.txt' + ])) diff --git a/segmentation/configs/_base_/datasets/potsdam.py b/segmentation/configs/_base_/datasets/potsdam.py new file mode 100644 index 0000000..e69de29 diff --git a/segmentation/configs/_base_/datasets/stare.py b/segmentation/configs/_base_/datasets/stare.py new file mode 100644 index 0000000..3f71b25 --- /dev/null +++ b/segmentation/configs/_base_/datasets/stare.py @@ -0,0 +1,59 @@ +# dataset settings +dataset_type = 'STAREDataset' +data_root = 'data/STARE' +img_norm_cfg = dict( + mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) +img_scale = (605, 700) +crop_size = (128, 128) +train_pipeline = [ + dict(type='LoadImageFromFile'), + dict(type='LoadAnnotations'), + dict(type='Resize', img_scale=img_scale, ratio_range=(0.5, 2.0)), + dict(type='RandomCrop', crop_size=crop_size, cat_max_ratio=0.75), + dict(type='RandomFlip', prob=0.5), + dict(type='PhotoMetricDistortion'), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size=crop_size, pad_val=0, seg_pad_val=255), + dict(type='DefaultFormatBundle'), + dict(type='Collect', keys=['img', 'gt_semantic_seg']) +] +test_pipeline = [ + dict(type='LoadImageFromFile'), + dict( + type='MultiScaleFlipAug', + img_scale=img_scale, + # img_ratios=[0.5, 0.75, 1.0, 1.25, 1.5, 1.75, 2.0], + flip=False, + transforms=[ + dict(type='Resize', keep_ratio=True), + dict(type='RandomFlip'), + dict(type='Normalize', **img_norm_cfg), + dict(type='ImageToTensor', keys=['img']), + dict(type='Collect', keys=['img']) + ]) +] + +data = dict( + samples_per_gpu=4, + workers_per_gpu=4, + train=dict( + type='RepeatDataset', + times=40000, + dataset=dict( + type=dataset_type, + data_root=data_root, + img_dir='images/training', + ann_dir='annotations/training', + pipeline=train_pipeline)), + val=dict( + type=dataset_type, + data_root=data_root, + img_dir='images/validation', + ann_dir='annotations/validation', + pipeline=test_pipeline), + test=dict( + type=dataset_type, + data_root=data_root, + img_dir='images/validation', + ann_dir='annotations/validation', + pipeline=test_pipeline)) diff --git a/segmentation/configs/_base_/default_runtime.py b/segmentation/configs/_base_/default_runtime.py new file mode 100644 index 0000000..b564cc4 --- /dev/null +++ b/segmentation/configs/_base_/default_runtime.py @@ -0,0 +1,14 @@ +# yapf:disable +log_config = dict( + interval=50, + hooks=[ + dict(type='TextLoggerHook', by_epoch=False), + # dict(type='TensorboardLoggerHook') + ]) +# yapf:enable +dist_params = dict(backend='nccl') +log_level = 'INFO' +load_from = None +resume_from = None +workflow = [('train', 1)] +cudnn_benchmark = True diff --git a/segmentation/configs/_base_/models/mask2former_beit.py b/segmentation/configs/_base_/models/mask2former_beit.py new file mode 100644 index 0000000..c81b466 --- /dev/null +++ b/segmentation/configs/_base_/models/mask2former_beit.py @@ -0,0 +1,138 @@ +# model_cfg +num_things_classes = 100 +num_stuff_classes = 50 +num_classes = num_things_classes + num_stuff_classes +norm_cfg = dict(type='SyncBN', requires_grad=True) +model = dict( + type='EncoderDecoderMask2Former', + pretrained=None, + backbone=dict( + type='XCiT', + patch_size=16, + embed_dim=384, + depth=12, + num_heads=8, + mlp_ratio=4, + qkv_bias=True, + use_abs_pos_emb=True, + use_rel_pos_bias=False, + ), + decode_head=dict( + type='Mask2FormerHead', + in_channels=[256, 512, 1024, 2048], # pass to pixel_decoder inside + # strides=[4, 8, 16, 32], + feat_channels=256, + out_channels=256, + in_index=[0, 1, 2, 3], + num_things_classes=num_things_classes, + num_stuff_classes=num_stuff_classes, + num_queries=100, + num_transformer_feat_level=3, + pixel_decoder=dict( + type='MSDeformAttnPixelDecoder', + num_outs=3, + norm_cfg=dict(type='GN', num_groups=32), + act_cfg=dict(type='ReLU'), + encoder=dict( + type='DetrTransformerEncoder', + num_layers=6, + transformerlayers=dict( + type='BaseTransformerLayer', + attn_cfgs=dict( + type='MultiScaleDeformableAttention', + embed_dims=256, + num_heads=8, + num_levels=3, + num_points=4, + im2col_step=64, + dropout=0.0, + batch_first=False, + norm_cfg=None, + init_cfg=None), + ffn_cfgs=dict( + type='FFN', + embed_dims=256, + feedforward_channels=1024, + num_fcs=2, + ffn_drop=0.0, + act_cfg=dict(type='ReLU', inplace=True)), + operation_order=('self_attn', 'norm', 'ffn', 'norm')), + init_cfg=None), + positional_encoding=dict( + type='SinePositionalEncoding', num_feats=128, normalize=True), + init_cfg=None), + enforce_decoder_input_project=False, + positional_encoding=dict( + type='SinePositionalEncoding', num_feats=128, normalize=True), + transformer_decoder=dict( + type='DetrTransformerDecoder', + return_intermediate=True, + num_layers=9, + transformerlayers=dict( + type='DetrTransformerDecoderLayer', + attn_cfgs=dict( + type='MultiheadAttention', + embed_dims=256, + num_heads=8, + attn_drop=0.0, + proj_drop=0.0, + dropout_layer=None, + batch_first=False), + ffn_cfgs=dict( + embed_dims=256, + feedforward_channels=2048, + num_fcs=2, + act_cfg=dict(type='ReLU', inplace=True), + ffn_drop=0.0, + dropout_layer=None, + add_identity=True), + feedforward_channels=2048, + operation_order=('cross_attn', 'norm', 'self_attn', 'norm', + 'ffn', 'norm')), + init_cfg=None), + loss_cls=dict( + type='CrossEntropyLoss', + use_sigmoid=False, + loss_weight=2.0, + reduction='mean', + class_weight=[1.0] * num_classes + [0.1]), + loss_mask=dict( + type='CrossEntropyLoss', + use_sigmoid=True, + reduction='mean', + loss_weight=5.0), + loss_dice=dict( + type='DiceLoss', + use_sigmoid=True, + activate=True, + reduction='mean', + naive_dice=True, + eps=1.0, + loss_weight=5.0)), + train_cfg=dict( + num_points=12544, + oversample_ratio=3.0, + importance_sample_ratio=0.75, + assigner=dict( + type='MaskHungarianAssigner', + cls_cost=dict(type='ClassificationCost', weight=2.0), + mask_cost=dict( + type='CrossEntropyLossCost', weight=5.0, use_sigmoid=True), + dice_cost=dict( + type='DiceCost', weight=5.0, pred_act=True, eps=1.0)), + sampler=dict(type='MaskPseudoSampler')), + test_cfg=dict( + panoptic_on=True, + # For now, the dataset does not support + # evaluating semantic segmentation metric. + semantic_on=False, + instance_on=True, + # max_per_image is for instance segmentation. + max_per_image=100, + iou_thr=0.8, + # In Mask2Former's panoptic postprocessing, + # it will filter mask area where score is less than 0.5 . + filter_low_score=True), + init_cfg=None) + +# find_unused_parameters = True \ No newline at end of file diff --git a/segmentation/configs/_base_/models/segformer_mit-b0.py b/segmentation/configs/_base_/models/segformer_mit-b0.py new file mode 100644 index 0000000..838b7c8 --- /dev/null +++ b/segmentation/configs/_base_/models/segformer_mit-b0.py @@ -0,0 +1,34 @@ +# model settings +norm_cfg = dict(type='SyncBN', requires_grad=True) +model = dict( + type='EncoderDecoder', + pretrained=None, + backbone=dict( + type='MixVisionTransformer', + in_channels=3, + embed_dims=32, + num_stages=4, + num_layers=[2, 2, 2, 2], + num_heads=[1, 2, 5, 8], + patch_sizes=[7, 3, 3, 3], + sr_ratios=[8, 4, 2, 1], + out_indices=(0, 1, 2, 3), + mlp_ratio=4, + qkv_bias=True, + drop_rate=0.0, + attn_drop_rate=0.0, + drop_path_rate=0.1), + decode_head=dict( + type='SegformerHead', + in_channels=[32, 64, 160, 256], + in_index=[0, 1, 2, 3], + channels=256, + dropout_ratio=0.1, + num_classes=19, + norm_cfg=norm_cfg, + align_corners=False, + loss_decode=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0)), + # model training and testing settings + train_cfg=dict(), + test_cfg=dict(mode='whole')) \ No newline at end of file diff --git a/segmentation/configs/_base_/models/upernet_convnext.py b/segmentation/configs/_base_/models/upernet_convnext.py new file mode 100644 index 0000000..419c1c0 --- /dev/null +++ b/segmentation/configs/_base_/models/upernet_convnext.py @@ -0,0 +1,46 @@ +norm_cfg = dict(type='SyncBN', requires_grad=True) +custom_imports = dict(imports='mmcls.models', allow_failed_imports=False) +# checkpoint_file = 'https://download.openmmlab.com/mmclassification/v0/convnext/downstream/convnext-base_3rdparty_32xb128-noema_in1k_20220301-2a0ee547.pth' # noqa +model = dict( + type='EncoderDecoder', + pretrained=None, + backbone=dict( + type='mmcls.ConvNeXt', + arch='base', + norm_cfg=dict(type='LN2dv2', eps=1e-6), + out_indices=[0, 1, 2, 3], + drop_path_rate=0.4, + layer_scale_init_value=1.0, + gap_before_final_norm=False, + # init_cfg=dict( + # type='Pretrained', checkpoint=checkpoint_file, + # prefix='backbone.') + ), + decode_head=dict( + type='UPerHead', + in_channels=[128, 256, 512, 1024], + in_index=[0, 1, 2, 3], + pool_scales=(1, 2, 3, 6), + channels=512, + dropout_ratio=0.1, + num_classes=19, + norm_cfg=norm_cfg, + align_corners=False, + loss_decode=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0)), + auxiliary_head=dict( + type='FCNHead', + in_channels=384, + in_index=2, + channels=256, + num_convs=1, + concat_input=False, + dropout_ratio=0.1, + num_classes=19, + norm_cfg=norm_cfg, + align_corners=False, + loss_decode=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=0.4)), + # model training and testing settings + train_cfg=dict(), + test_cfg=dict(mode='whole')) diff --git a/segmentation/configs/_base_/models/upernet_r50.py b/segmentation/configs/_base_/models/upernet_r50.py new file mode 100644 index 0000000..1097496 --- /dev/null +++ b/segmentation/configs/_base_/models/upernet_r50.py @@ -0,0 +1,44 @@ +# model settings +norm_cfg = dict(type='SyncBN', requires_grad=True) +model = dict( + type='EncoderDecoder', + pretrained='open-mmlab://resnet50_v1c', + backbone=dict( + type='ResNetV1c', + depth=50, + num_stages=4, + out_indices=(0, 1, 2, 3), + dilations=(1, 1, 1, 1), + strides=(1, 2, 2, 2), + norm_cfg=norm_cfg, + norm_eval=False, + style='pytorch', + contract_dilation=True), + decode_head=dict( + type='UPerHead', + in_channels=[256, 512, 1024, 2048], + in_index=[0, 1, 2, 3], + pool_scales=(1, 2, 3, 6), + channels=512, + dropout_ratio=0.1, + num_classes=19, + norm_cfg=norm_cfg, + align_corners=False, + loss_decode=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0)), + auxiliary_head=dict( + type='FCNHead', + in_channels=1024, + in_index=2, + channels=256, + num_convs=1, + concat_input=False, + dropout_ratio=0.1, + num_classes=19, + norm_cfg=norm_cfg, + align_corners=False, + loss_decode=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=0.4)), + # model training and testing settings + train_cfg=dict(), + test_cfg=dict(mode='whole')) diff --git a/segmentation/configs/_base_/models/upernet_swin.py b/segmentation/configs/_base_/models/upernet_swin.py new file mode 100644 index 0000000..71b5162 --- /dev/null +++ b/segmentation/configs/_base_/models/upernet_swin.py @@ -0,0 +1,54 @@ +# model settings +norm_cfg = dict(type='SyncBN', requires_grad=True) +backbone_norm_cfg = dict(type='LN', requires_grad=True) +model = dict( + type='EncoderDecoder', + pretrained=None, + backbone=dict( + type='SwinTransformer', + pretrain_img_size=224, + embed_dims=96, + patch_size=4, + window_size=7, + mlp_ratio=4, + depths=[2, 2, 6, 2], + num_heads=[3, 6, 12, 24], + strides=(4, 2, 2, 2), + out_indices=(0, 1, 2, 3), + qkv_bias=True, + qk_scale=None, + patch_norm=True, + drop_rate=0., + attn_drop_rate=0., + drop_path_rate=0.3, + use_abs_pos_embed=False, + act_cfg=dict(type='GELU'), + norm_cfg=backbone_norm_cfg), + decode_head=dict( + type='UPerHead', + in_channels=[96, 192, 384, 768], + in_index=[0, 1, 2, 3], + pool_scales=(1, 2, 3, 6), + channels=512, + dropout_ratio=0.1, + num_classes=19, + norm_cfg=norm_cfg, + align_corners=False, + loss_decode=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0)), + auxiliary_head=dict( + type='FCNHead', + in_channels=384, + in_index=2, + channels=256, + num_convs=1, + concat_input=False, + dropout_ratio=0.1, + num_classes=19, + norm_cfg=norm_cfg, + align_corners=False, + loss_decode=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=0.4)), + # model training and testing settings + train_cfg=dict(), + test_cfg=dict(mode='whole')) diff --git a/segmentation/configs/_base_/schedules/schedule_160k.py b/segmentation/configs/_base_/schedules/schedule_160k.py new file mode 100644 index 0000000..39630f2 --- /dev/null +++ b/segmentation/configs/_base_/schedules/schedule_160k.py @@ -0,0 +1,9 @@ +# optimizer +optimizer = dict(type='SGD', lr=0.01, momentum=0.9, weight_decay=0.0005) +optimizer_config = dict() +# learning policy +lr_config = dict(policy='poly', power=0.9, min_lr=1e-4, by_epoch=False) +# runtime settings +runner = dict(type='IterBasedRunner', max_iters=160000) +checkpoint_config = dict(by_epoch=False, interval=16000) +evaluation = dict(interval=16000, metric='mIoU', pre_eval=True) diff --git a/segmentation/configs/_base_/schedules/schedule_20k.py b/segmentation/configs/_base_/schedules/schedule_20k.py new file mode 100644 index 0000000..73c7021 --- /dev/null +++ b/segmentation/configs/_base_/schedules/schedule_20k.py @@ -0,0 +1,9 @@ +# optimizer +optimizer = dict(type='SGD', lr=0.01, momentum=0.9, weight_decay=0.0005) +optimizer_config = dict() +# learning policy +lr_config = dict(policy='poly', power=0.9, min_lr=1e-4, by_epoch=False) +# runtime settings +runner = dict(type='IterBasedRunner', max_iters=20000) +checkpoint_config = dict(by_epoch=False, interval=2000) +evaluation = dict(interval=2000, metric='mIoU', pre_eval=True) diff --git a/segmentation/configs/_base_/schedules/schedule_320k.py b/segmentation/configs/_base_/schedules/schedule_320k.py new file mode 100644 index 0000000..a0b2306 --- /dev/null +++ b/segmentation/configs/_base_/schedules/schedule_320k.py @@ -0,0 +1,9 @@ +# optimizer +optimizer = dict(type='SGD', lr=0.01, momentum=0.9, weight_decay=0.0005) +optimizer_config = dict() +# learning policy +lr_config = dict(policy='poly', power=0.9, min_lr=1e-4, by_epoch=False) +# runtime settings +runner = dict(type='IterBasedRunner', max_iters=320000) +checkpoint_config = dict(by_epoch=False, interval=32000) +evaluation = dict(interval=32000, metric='mIoU') diff --git a/segmentation/configs/_base_/schedules/schedule_40k.py b/segmentation/configs/_base_/schedules/schedule_40k.py new file mode 100644 index 0000000..d2c5023 --- /dev/null +++ b/segmentation/configs/_base_/schedules/schedule_40k.py @@ -0,0 +1,9 @@ +# optimizer +optimizer = dict(type='SGD', lr=0.01, momentum=0.9, weight_decay=0.0005) +optimizer_config = dict() +# learning policy +lr_config = dict(policy='poly', power=0.9, min_lr=1e-4, by_epoch=False) +# runtime settings +runner = dict(type='IterBasedRunner', max_iters=40000) +checkpoint_config = dict(by_epoch=False, interval=4000) +evaluation = dict(interval=4000, metric='mIoU', pre_eval=True) diff --git a/segmentation/configs/_base_/schedules/schedule_80k.py b/segmentation/configs/_base_/schedules/schedule_80k.py new file mode 100644 index 0000000..8365a87 --- /dev/null +++ b/segmentation/configs/_base_/schedules/schedule_80k.py @@ -0,0 +1,9 @@ +# optimizer +optimizer = dict(type='SGD', lr=0.01, momentum=0.9, weight_decay=0.0005) +optimizer_config = dict() +# learning policy +lr_config = dict(policy='poly', power=0.9, min_lr=1e-4, by_epoch=False) +# runtime settings +runner = dict(type='IterBasedRunner', max_iters=80000) +checkpoint_config = dict(by_epoch=False, interval=8000) +evaluation = dict(interval=8000, metric='mIoU', pre_eval=True) diff --git a/segmentation/configs/ade20k/README.md b/segmentation/configs/ade20k/README.md new file mode 100644 index 0000000..f051399 --- /dev/null +++ b/segmentation/configs/ade20k/README.md @@ -0,0 +1,33 @@ +# ADE20K + +Introduced by Zhou et al. in [Scene Parsing Through ADE20K Dataset](https://paperswithcode.com/paper/scene-parsing-through-ade20k-dataset). + +The ADE20K semantic segmentation dataset contains more than 20K scene-centric images exhaustively annotated with pixel-level objects and object parts labels. There are totally 150 semantic categories, which include stuffs like sky, road, grass, and discrete objects like person, car, bed. + + +## Model Zoo + +### UperNet + InternImage + + +| backbone | resolution | mIoU (ss/ms) | Config | Download | +|:--------------:|:----------:|:-----------:|:-----------:|:----------: +| FlashInternImage-T | 512x512 | 49.3 / 50.3 | [config](./upernet_flash_internimage_t_512_160k_ade20k.py) | [ckpt](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/upernet_flash_internimage_t_512_160k_ade20k.pth) \| [log](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/upernet_flash_internimage_t_512_160k_ade20k.log) | +| FlashInternImage-S | 512x512 | 50.6 / 51.6 | [config](./upernet_flash_internimage_s_512_160k_ade20k.py) | [ckpt](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/upernet_flash_internimage_s_512_160k_ade20k.pth) \| [log](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/upernet_flash_internimage_s_512_160k_ade20k.log) | +| FlashInternImage-B | 512x512 | 52.0 / 52.6 | [config](./upernet_flash_internimage_b_512_160k_ade20k.py) | [ckpt](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/upernet_flash_internimage_b_512_160k_ade20k.pth) \| [log](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/upernet_flash_internimage_s_512_160k_ade20k.log) | +| FlashInternImage-L | 640x640 | 55.6 / 56.0 | [config](./upernet_flash_internimage_l_640_160k_ade20k.py)| [ckpt](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/upernet_flash_internimage_l_640_160k_ade20k.pth) \| [log](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/upernet_flash_internimage_l_640_160k_ade20k.log) | + +- Training speed is measured with A100 GPU. +- Please set `with_cp=True` to save memory if you meet `out-of-memory` issues. +- The logs are our recent newly trained ones. There are slight differences between the results in logs and our paper. + + +### Mask2Former + InternImage + +| backbone | resolution | mIoU (ss) | Config | Download | +|:--------------:|:----------:|:-----------:|:-----------:|:----------: +| FlashInternImage-T | 512x512 | 51.2 | [config](./mask2former_flash_internimage_t_512_160k_ade20k_ss.py) | [ckpt](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/mask2former_flash_internimage_t_512_160k_ade20k_ss.pth) \| [log](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/mask2former_flash_internimage_t_512_160k_ade20k_ss.log) | +| FlashInternImage-S | 640x640 | 52.2 | [config](./mask2former_flash_internimage_s_640_160k_ade20k_ss.py) | [ckpt](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/mask2former_flash_internimage_s_640_160k_ade20k_ss.pth) \| [log](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/mask2former_flash_internimage_s_640_160k_ade20k_ss.log) | +| FlashInternImage-B | 640x640 | 53.4 | [config](./mask2former_flash_internimage_b_640_160k_ade20k_ss.py) | [ckpt](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/mask2former_flash_internimage_b_640_160k_ade20k_ss.pth) \| [log](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/mask2former_flash_internimage_b_640_160k_ade20k_ss.log) | +| FlashInternImage-L | 640x640 | 56.7 | [config](./mask2former_flash_internimage_l_640_160k_ade20k_ss.py)| [ckpt](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/mask2former_flash_internimage_l_640_160k_ade20k_ss.pth) \| [log](https://huggingface.co/OpenGVLab/DCNv4/resolve/main/mask2former_flash_internimage_l_640_160k_ade20k_ss.log) | + diff --git a/segmentation/configs/ade20k/mask2former_flash_internimage_b_640_160k_ade20k_ss.py b/segmentation/configs/ade20k/mask2former_flash_internimage_b_640_160k_ade20k_ss.py new file mode 100644 index 0000000..666947b --- /dev/null +++ b/segmentation/configs/ade20k/mask2former_flash_internimage_b_640_160k_ade20k_ss.py @@ -0,0 +1,159 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- +_base_ = [ + '../_base_/models/mask2former_beit.py', '../_base_/datasets/ade20k.py', + '../_base_/default_runtime.py', '../_base_/schedules/schedule_160k.py' +] +num_classes = 150 +pretrained = 'https://huggingface.co/OpenGVLab/DCNv4/resolve/main/flash_intern_image_b_1k_224.pth' +model = dict( + backbone=dict( + _delete_=True, + type='FlashInternImage', + core_op='DCNv4', + channels=112, + depths=[4, 4, 21, 4], + groups=[7, 14, 28, 56], + mlp_ratio=4., + drop_path_rate=0.4, + norm_layer='LN', + layer_scale=1.0, + offset_scale=0.5, + post_norm=True, + with_cp=False, + dw_kernel_size=3, + out_indices=(0, 1, 2, 3), + init_cfg=dict(type='Pretrained', checkpoint=pretrained)), + decode_head=dict( + in_channels=[112, 224, 448, 896], + feat_channels=256, + out_channels=256, + num_classes=num_classes, + num_queries=200, + pixel_decoder=dict( + type='MSDeformAttnPixelDecoder', + num_outs=3, + norm_cfg=dict(type='GN', num_groups=32), + act_cfg=dict(type='ReLU'), + encoder=dict( + type='DetrTransformerEncoder', + num_layers=6, + transformerlayers=dict( + type='BaseTransformerLayer', + attn_cfgs=dict( + type='MultiScaleDeformableAttention', + embed_dims=256, + num_heads=8, + num_levels=3, + num_points=4, + im2col_step=64, + dropout=0.0, + batch_first=False, + norm_cfg=None, + init_cfg=None), + ffn_cfgs=dict( + type='FFN', + embed_dims=256, + feedforward_channels=2048, + num_fcs=2, + ffn_drop=0.0, + with_cp=False, # set with_cp=True to save memory + act_cfg=dict(type='ReLU', inplace=True)), + operation_order=('self_attn', 'norm', 'ffn', 'norm')), + init_cfg=None), + positional_encoding=dict( + type='SinePositionalEncoding', num_feats=128, normalize=True), + init_cfg=None), + positional_encoding=dict( + type='SinePositionalEncoding', num_feats=128, normalize=True), + transformer_decoder=dict( + type='DetrTransformerDecoder', + return_intermediate=True, + num_layers=9, + transformerlayers=dict( + type='DetrTransformerDecoderLayer', + attn_cfgs=dict( + type='MultiheadAttention', + embed_dims=256, + num_heads=8, + attn_drop=0.0, + proj_drop=0.0, + dropout_layer=None, + batch_first=False), + ffn_cfgs=dict( + embed_dims=256, + feedforward_channels=2048, + num_fcs=2, + act_cfg=dict(type='ReLU', inplace=True), + ffn_drop=0.0, + dropout_layer=None, + with_cp=False, # set with_cp=True to save memory + add_identity=True), + feedforward_channels=2048, + operation_order=('cross_attn', 'norm', 'self_attn', 'norm', + 'ffn', 'norm')), + init_cfg=None), + loss_cls=dict( + type='CrossEntropyLoss', + use_sigmoid=False, + loss_weight=2.0, + reduction='mean', + class_weight=[1.0] * num_classes + [0.1]) + ), + test_cfg=dict(mode='whole')) +img_norm_cfg = dict( + mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) +crop_size = (640, 640) +train_pipeline = [ + dict(type='LoadImageFromFile'), + dict(type='LoadAnnotations', reduce_zero_label=True), + dict(type='Resize', img_scale=(2560, 640), ratio_range=(0.5, 2.0)), + dict(type='RandomCrop', crop_size=crop_size, cat_max_ratio=0.75), + dict(type='RandomFlip', prob=0.5), + dict(type='PhotoMetricDistortion'), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size=crop_size, pad_val=0, seg_pad_val=255), + dict(type='ToMask'), + dict(type='DefaultFormatBundle'), + dict(type='Collect', keys=['img', 'gt_semantic_seg', 'gt_masks', 'gt_labels']) +] +test_pipeline = [ + dict(type='LoadImageFromFile'), + dict( + type='MultiScaleFlipAug', + img_scale=(2560, 640), + # img_ratios=[768./896., 832./896., 1.0, 960./896., 1024./896.], + flip=False, + transforms=[ + dict(type='Resize', keep_ratio=True), + dict(type='ResizeToMultiple', size_divisor=32), + dict(type='RandomFlip'), + dict(type='Normalize', **img_norm_cfg), + dict(type='ImageToTensor', keys=['img']), + dict(type='Collect', keys=['img']), + ]) +] +optimizer = dict( + _delete_=True, type='AdamW', lr=0.0001, betas=(0.9, 0.999), weight_decay=0.05, + constructor='CustomLayerDecayOptimizerConstructor', + paramwise_cfg=dict(num_layers=33, layer_decay_rate=1.0, + depths=[4, 4, 21, 4])) + +lr_config = dict(_delete_=True, policy='poly', + warmup='linear', + warmup_iters=1500, + warmup_ratio=1e-6, + power=1.0, min_lr=0.0, by_epoch=False) +# By default, models are trained on 8 GPUs with 2 images per GPU +data = dict(samples_per_gpu=2, + train=dict(pipeline=train_pipeline), + val=dict(pipeline=test_pipeline), + test=dict(pipeline=test_pipeline)) +runner = dict(type='IterBasedRunner') +optimizer_config = dict(_delete_=True, grad_clip=dict(max_norm=0.1, norm_type=2)) +checkpoint_config = dict(by_epoch=False, interval=5000, max_keep_ckpts=1) +evaluation = dict(interval=5000, metric='mIoU', save_best='mIoU') +# fp16 = dict(loss_scale=dict(init_scale=512)) diff --git a/segmentation/configs/ade20k/mask2former_flash_internimage_l_640_160k_ade20k_ss.py b/segmentation/configs/ade20k/mask2former_flash_internimage_l_640_160k_ade20k_ss.py new file mode 100644 index 0000000..fef9743 --- /dev/null +++ b/segmentation/configs/ade20k/mask2former_flash_internimage_l_640_160k_ade20k_ss.py @@ -0,0 +1,160 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- +_base_ = [ + '../_base_/models/mask2former_beit.py', '../_base_/datasets/ade20k.py', + '../_base_/default_runtime.py', '../_base_/schedules/schedule_160k.py' +] +num_classes = 150 +pretrained = 'https://huggingface.co/OpenGVLab/DCNv4/resolve/main/flash_intern_image_l_22k_384.pth' +model = dict( + backbone=dict( + _delete_=True, + type='FlashInternImage', + core_op='DCNv4', + channels=160, + depths=[5, 5, 22, 5], + groups=[10, 20, 40, 80], + mlp_ratio=4., + drop_path_rate=0.5, + norm_layer='LN', + layer_scale=1.0, + offset_scale=2.0, + post_norm=True, + with_cp=True, + dcn_output_bias=True, + mlp_fc2_bias=True, + dw_kernel_size=3, + out_indices=(0, 1, 2, 3), + init_cfg=dict(type='Pretrained', checkpoint=pretrained)), + decode_head=dict( + in_channels=[160, 320, 640, 1280], + feat_channels=256, + out_channels=256, + num_classes=num_classes, + num_queries=200, + pixel_decoder=dict( + type='MSDeformAttnPixelDecoder', + num_outs=3, + norm_cfg=dict(type='GN', num_groups=32), + act_cfg=dict(type='ReLU'), + encoder=dict( + type='DetrTransformerEncoder', + num_layers=6, + transformerlayers=dict( + type='BaseTransformerLayer', + attn_cfgs=dict( + type='MultiScaleDeformableAttention', + embed_dims=256, + num_heads=8, + num_levels=3, + num_points=4, + im2col_step=64, + dropout=0.0, + batch_first=False, + norm_cfg=None, + init_cfg=None), + ffn_cfgs=dict( + type='FFN', + embed_dims=256, + feedforward_channels=2048, + num_fcs=2, + ffn_drop=0.0, + with_cp=False, # set with_cp=True to save memory + act_cfg=dict(type='ReLU', inplace=True)), + operation_order=('self_attn', 'norm', 'ffn', 'norm')), + init_cfg=None), + positional_encoding=dict( + type='SinePositionalEncoding', num_feats=128, normalize=True), + init_cfg=None), + positional_encoding=dict( + type='SinePositionalEncoding', num_feats=128, normalize=True), + transformer_decoder=dict( + type='DetrTransformerDecoder', + return_intermediate=True, + num_layers=9, + transformerlayers=dict( + type='DetrTransformerDecoderLayer', + attn_cfgs=dict( + type='MultiheadAttention', + embed_dims=256, + num_heads=8, + attn_drop=0.0, + proj_drop=0.0, + dropout_layer=None, + batch_first=False), + ffn_cfgs=dict( + embed_dims=256, + feedforward_channels=2048, + num_fcs=2, + act_cfg=dict(type='ReLU', inplace=True), + ffn_drop=0.0, + dropout_layer=None, + with_cp=False, # set with_cp=True to save memory + add_identity=True), + feedforward_channels=2048, + operation_order=('cross_attn', 'norm', 'self_attn', 'norm', + 'ffn', 'norm')), + init_cfg=None), + loss_cls=dict( + type='CrossEntropyLoss', + use_sigmoid=False, + loss_weight=2.0, + reduction='mean', + class_weight=[1.0] * num_classes + [0.1]) + ), + test_cfg=dict(mode='whole')) +img_norm_cfg = dict( + mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) +crop_size = (640, 640) +train_pipeline = [ + dict(type='LoadImageFromFile'), + dict(type='LoadAnnotations', reduce_zero_label=True), + dict(type='Resize', img_scale=(2560, 640), ratio_range=(0.5, 2.0)), + dict(type='RandomCrop', crop_size=crop_size, cat_max_ratio=0.75), + dict(type='RandomFlip', prob=0.5), + dict(type='PhotoMetricDistortion'), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size=crop_size, pad_val=0, seg_pad_val=255), + dict(type='ToMask'), + dict(type='DefaultFormatBundle'), + dict(type='Collect', keys=['img', 'gt_semantic_seg', 'gt_masks', 'gt_labels']) +] +test_pipeline = [ + dict(type='LoadImageFromFile'), + dict( + type='MultiScaleFlipAug', + img_scale=(2560, 640), + # img_ratios=[768./896., 832./896., 1.0, 960./896., 1024./896.], + flip=False, + transforms=[ + dict(type='Resize', keep_ratio=True), + dict(type='ResizeToMultiple', size_divisor=32), + dict(type='RandomFlip'), + dict(type='Normalize', **img_norm_cfg), + dict(type='ImageToTensor', keys=['img']), + dict(type='Collect', keys=['img']), + ]) +] +optimizer = dict( + _delete_=True, type='AdamW', lr=0.0001, betas=(0.9, 0.999), weight_decay=0.05, + constructor='CustomLayerDecayOptimizerConstructor', + paramwise_cfg=dict(num_layers=37, layer_decay_rate=0.94, + depths=[5, 5, 22, 5], offset_lr_scale=1.0)) +lr_config = dict(_delete_=True, policy='poly', + warmup='linear', + warmup_iters=1500, + warmup_ratio=1e-6, + power=1.0, min_lr=0.0, by_epoch=False) +# By default, models are trained on 8 GPUs with 2 images per GPU +data = dict(samples_per_gpu=2, + train=dict(pipeline=train_pipeline), + val=dict(pipeline=test_pipeline), + test=dict(pipeline=test_pipeline)) +runner = dict(type='IterBasedRunner') +optimizer_config = dict(_delete_=True, grad_clip=dict(max_norm=0.1, norm_type=2)) +checkpoint_config = dict(by_epoch=False, interval=2000, max_keep_ckpts=1) +evaluation = dict(interval=2000, metric='mIoU', save_best='mIoU') +# fp16 = dict(loss_scale=dict(init_scale=512)) diff --git a/segmentation/configs/ade20k/mask2former_flash_internimage_s_640_160k_ade20k_ss.py b/segmentation/configs/ade20k/mask2former_flash_internimage_s_640_160k_ade20k_ss.py new file mode 100644 index 0000000..5c41667 --- /dev/null +++ b/segmentation/configs/ade20k/mask2former_flash_internimage_s_640_160k_ade20k_ss.py @@ -0,0 +1,159 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- +_base_ = [ + '../_base_/models/mask2former_beit.py', '../_base_/datasets/ade20k.py', + '../_base_/default_runtime.py', '../_base_/schedules/schedule_160k.py' +] +num_classes = 150 +pretrained = 'https://huggingface.co/OpenGVLab/DCNv4/resolve/main/flash_intern_image_s_1k_224.pth' +model = dict( + backbone=dict( + _delete_=True, + type='FlashInternImage', + core_op='DCNv4', + channels=80, + depths=[4, 4, 21, 4], + groups=[5, 10, 20, 40], + mlp_ratio=4., + drop_path_rate=0.3, + norm_layer='LN', + layer_scale=1.0, + offset_scale=1.0, + post_norm=True, + with_cp=False, + dw_kernel_size=3, + out_indices=(0, 1, 2, 3), + init_cfg=dict(type='Pretrained', checkpoint=pretrained)), + decode_head=dict( + in_channels=[80, 160, 320, 640], + feat_channels=256, + out_channels=256, + num_classes=num_classes, + num_queries=200, + pixel_decoder=dict( + type='MSDeformAttnPixelDecoder', + num_outs=3, + norm_cfg=dict(type='GN', num_groups=32), + act_cfg=dict(type='ReLU'), + encoder=dict( + type='DetrTransformerEncoder', + num_layers=6, + transformerlayers=dict( + type='BaseTransformerLayer', + attn_cfgs=dict( + type='MultiScaleDeformableAttention', + embed_dims=256, + num_heads=8, + num_levels=3, + num_points=4, + im2col_step=64, + dropout=0.0, + batch_first=False, + norm_cfg=None, + init_cfg=None), + ffn_cfgs=dict( + type='FFN', + embed_dims=256, + feedforward_channels=2048, + num_fcs=2, + ffn_drop=0.0, + with_cp=False, # set with_cp=True to save memory + act_cfg=dict(type='ReLU', inplace=True)), + operation_order=('self_attn', 'norm', 'ffn', 'norm')), + init_cfg=None), + positional_encoding=dict( + type='SinePositionalEncoding', num_feats=128, normalize=True), + init_cfg=None), + positional_encoding=dict( + type='SinePositionalEncoding', num_feats=128, normalize=True), + transformer_decoder=dict( + type='DetrTransformerDecoder', + return_intermediate=True, + num_layers=9, + transformerlayers=dict( + type='DetrTransformerDecoderLayer', + attn_cfgs=dict( + type='MultiheadAttention', + embed_dims=256, + num_heads=8, + attn_drop=0.0, + proj_drop=0.0, + dropout_layer=None, + batch_first=False), + ffn_cfgs=dict( + embed_dims=256, + feedforward_channels=2048, + num_fcs=2, + act_cfg=dict(type='ReLU', inplace=True), + ffn_drop=0.0, + dropout_layer=None, + with_cp=False, # set with_cp=True to save memory + add_identity=True), + feedforward_channels=2048, + operation_order=('cross_attn', 'norm', 'self_attn', 'norm', + 'ffn', 'norm')), + init_cfg=None), + loss_cls=dict( + type='CrossEntropyLoss', + use_sigmoid=False, + loss_weight=2.0, + reduction='mean', + class_weight=[1.0] * num_classes + [0.1]) + ), + test_cfg=dict(mode='whole')) +img_norm_cfg = dict( + mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) +crop_size = (640, 640) +train_pipeline = [ + dict(type='LoadImageFromFile'), + dict(type='LoadAnnotations', reduce_zero_label=True), + dict(type='Resize', img_scale=(2560, 640), ratio_range=(0.5, 2.0)), + dict(type='RandomCrop', crop_size=crop_size, cat_max_ratio=0.75), + dict(type='RandomFlip', prob=0.5), + dict(type='PhotoMetricDistortion'), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size=crop_size, pad_val=0, seg_pad_val=255), + dict(type='ToMask'), + dict(type='DefaultFormatBundle'), + dict(type='Collect', keys=['img', 'gt_semantic_seg', 'gt_masks', 'gt_labels']) +] +test_pipeline = [ + dict(type='LoadImageFromFile'), + dict( + type='MultiScaleFlipAug', + img_scale=(2560, 640), + # img_ratios=[768./896., 832./896., 1.0, 960./896., 1024./896.], + flip=False, + transforms=[ + dict(type='Resize', keep_ratio=True), + dict(type='ResizeToMultiple', size_divisor=32), + dict(type='RandomFlip'), + dict(type='Normalize', **img_norm_cfg), + dict(type='ImageToTensor', keys=['img']), + dict(type='Collect', keys=['img']), + ]) +] +optimizer = dict( + _delete_=True, type='AdamW', lr=0.0001, betas=(0.9, 0.999), weight_decay=0.05, + constructor='CustomLayerDecayOptimizerConstructor', + paramwise_cfg=dict(num_layers=33, layer_decay_rate=1.0, + depths=[4, 4, 21, 4])) + +lr_config = dict(_delete_=True, policy='poly', + warmup='linear', + warmup_iters=1500, + warmup_ratio=1e-6, + power=1.0, min_lr=0.0, by_epoch=False) +# By default, models are trained on 8 GPUs with 2 images per GPU +data = dict(samples_per_gpu=2, + train=dict(pipeline=train_pipeline), + val=dict(pipeline=test_pipeline), + test=dict(pipeline=test_pipeline)) +runner = dict(type='IterBasedRunner') +optimizer_config = dict(_delete_=True, grad_clip=dict(max_norm=0.1, norm_type=2)) +checkpoint_config = dict(by_epoch=False, interval=5000, max_keep_ckpts=1) +evaluation = dict(interval=5000, metric='mIoU', save_best='mIoU') +# fp16 = dict(loss_scale=dict(init_scale=512)) diff --git a/segmentation/configs/ade20k/mask2former_flash_internimage_s_640_160k_ade20k_ss_nsmx.py b/segmentation/configs/ade20k/mask2former_flash_internimage_s_640_160k_ade20k_ss_nsmx.py new file mode 100644 index 0000000..988d949 --- /dev/null +++ b/segmentation/configs/ade20k/mask2former_flash_internimage_s_640_160k_ade20k_ss_nsmx.py @@ -0,0 +1,160 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- +_base_ = [ + '../_base_/models/mask2former_beit.py', '../_base_/datasets/ade20k.py', + '../_base_/default_runtime.py', '../_base_/schedules/schedule_160k.py' +] +num_classes = 150 +pretrained = 'https://huggingface.co/OpenGVLab/DCNv4/resolve/main/flash_intern_image_s_1k_224.pth' +model = dict( + backbone=dict( + _delete_=True, + type='FlashInternImage', + core_op='DCNv4', + channels=80, + depths=[4, 4, 21, 4], + groups=[5, 10, 20, 40], + mlp_ratio=4., + drop_path_rate=0.3, + norm_layer='LN', + layer_scale=1.0, + offset_scale=1.0, + post_norm=True, + with_cp=False, + dw_kernel_size=3, + out_indices=(0, 1, 2, 3), + init_cfg=dict(type='Pretrained', checkpoint=pretrained)), + decode_head=dict( + in_channels=[80, 160, 320, 640], + feat_channels=256, + out_channels=256, + num_classes=num_classes, + num_queries=200, + pixel_decoder=dict( + type='MSDeformAttnPixelDecoder', + num_outs=3, + norm_cfg=dict(type='GN', num_groups=32), + act_cfg=dict(type='ReLU'), + encoder=dict( + type='DetrTransformerEncoder', + num_layers=6, + transformerlayers=dict( + type='BaseTransformerLayer', + attn_cfgs=dict( + type='CustomMultiScaleDeformableAttention', + use_softmax=False, + embed_dims=256, + num_heads=8, + num_levels=3, + num_points=4, + im2col_step=64, + dropout=0.0, + batch_first=False, + norm_cfg=None, + init_cfg=None), + ffn_cfgs=dict( + type='FFN', + embed_dims=256, + feedforward_channels=2048, + num_fcs=2, + ffn_drop=0.0, + with_cp=False, # set with_cp=True to save memory + act_cfg=dict(type='ReLU', inplace=True)), + operation_order=('self_attn', 'norm', 'ffn', 'norm')), + init_cfg=None), + positional_encoding=dict( + type='SinePositionalEncoding', num_feats=128, normalize=True), + init_cfg=None), + positional_encoding=dict( + type='SinePositionalEncoding', num_feats=128, normalize=True), + transformer_decoder=dict( + type='DetrTransformerDecoder', + return_intermediate=True, + num_layers=9, + transformerlayers=dict( + type='DetrTransformerDecoderLayer', + attn_cfgs=dict( + type='MultiheadAttention', + embed_dims=256, + num_heads=8, + attn_drop=0.0, + proj_drop=0.0, + dropout_layer=None, + batch_first=False), + ffn_cfgs=dict( + embed_dims=256, + feedforward_channels=2048, + num_fcs=2, + act_cfg=dict(type='ReLU', inplace=True), + ffn_drop=0.0, + dropout_layer=None, + with_cp=False, # set with_cp=True to save memory + add_identity=True), + feedforward_channels=2048, + operation_order=('cross_attn', 'norm', 'self_attn', 'norm', + 'ffn', 'norm')), + init_cfg=None), + loss_cls=dict( + type='CrossEntropyLoss', + use_sigmoid=False, + loss_weight=2.0, + reduction='mean', + class_weight=[1.0] * num_classes + [0.1]) + ), + test_cfg=dict(mode='whole')) +img_norm_cfg = dict( + mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) +crop_size = (640, 640) +train_pipeline = [ + dict(type='LoadImageFromFile'), + dict(type='LoadAnnotations', reduce_zero_label=True), + dict(type='Resize', img_scale=(2560, 640), ratio_range=(0.5, 2.0)), + dict(type='RandomCrop', crop_size=crop_size, cat_max_ratio=0.75), + dict(type='RandomFlip', prob=0.5), + dict(type='PhotoMetricDistortion'), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size=crop_size, pad_val=0, seg_pad_val=255), + dict(type='ToMask'), + dict(type='DefaultFormatBundle'), + dict(type='Collect', keys=['img', 'gt_semantic_seg', 'gt_masks', 'gt_labels']) +] +test_pipeline = [ + dict(type='LoadImageFromFile'), + dict( + type='MultiScaleFlipAug', + img_scale=(2560, 640), + # img_ratios=[768./896., 832./896., 1.0, 960./896., 1024./896.], + flip=False, + transforms=[ + dict(type='Resize', keep_ratio=True), + dict(type='ResizeToMultiple', size_divisor=32), + dict(type='RandomFlip'), + dict(type='Normalize', **img_norm_cfg), + dict(type='ImageToTensor', keys=['img']), + dict(type='Collect', keys=['img']), + ]) +] +optimizer = dict( + _delete_=True, type='AdamW', lr=0.0001, betas=(0.9, 0.999), weight_decay=0.05, + constructor='CustomLayerDecayOptimizerConstructor', + paramwise_cfg=dict(num_layers=33, layer_decay_rate=1.0, + depths=[4, 4, 21, 4])) + +lr_config = dict(_delete_=True, policy='poly', + warmup='linear', + warmup_iters=1500, + warmup_ratio=1e-6, + power=1.0, min_lr=0.0, by_epoch=False) +# By default, models are trained on 8 GPUs with 2 images per GPU +data = dict(samples_per_gpu=2, + train=dict(pipeline=train_pipeline), + val=dict(pipeline=test_pipeline), + test=dict(pipeline=test_pipeline)) +runner = dict(type='IterBasedRunner') +optimizer_config = dict(_delete_=True, grad_clip=dict(max_norm=0.1, norm_type=2)) +checkpoint_config = dict(by_epoch=False, interval=5000, max_keep_ckpts=1) +evaluation = dict(interval=5000, metric='mIoU', save_best='mIoU') +# fp16 = dict(loss_scale=dict(init_scale=512)) diff --git a/segmentation/configs/ade20k/mask2former_flash_internimage_t_512_160k_ade20k_ss.py b/segmentation/configs/ade20k/mask2former_flash_internimage_t_512_160k_ade20k_ss.py new file mode 100644 index 0000000..3ef531e --- /dev/null +++ b/segmentation/configs/ade20k/mask2former_flash_internimage_t_512_160k_ade20k_ss.py @@ -0,0 +1,157 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2023 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- +_base_ = [ + '../_base_/models/mask2former_beit.py', '../_base_/datasets/ade20k.py', + '../_base_/default_runtime.py', '../_base_/schedules/schedule_160k.py' +] +num_classes = 150 +pretrained = 'https://huggingface.co/OpenGVLab/DCNv4/resolve/main/flash_intern_image_t_1k_224.pth' +model = dict( + backbone=dict( + _delete_=True, + type='FlashInternImage', + core_op='DCNv4', + channels=64, + depths=[4, 4, 18, 4], + groups=[4, 8, 16, 32], + mlp_ratio=4., + drop_path_rate=0.2, + norm_layer='LN', + layer_scale=1.0, + offset_scale=1.0, + post_norm=False, + with_cp=False, + out_indices=(0, 1, 2, 3), + init_cfg=dict(type='Pretrained', checkpoint=pretrained)), + decode_head=dict( + in_channels=[64, 128, 256, 512], + feat_channels=256, + out_channels=256, + num_classes=num_classes, + num_queries=100, + pixel_decoder=dict( + type='MSDeformAttnPixelDecoder', + num_outs=3, + norm_cfg=dict(type='GN', num_groups=32), + act_cfg=dict(type='ReLU'), + encoder=dict( + type='DetrTransformerEncoder', + num_layers=6, + transformerlayers=dict( + type='BaseTransformerLayer', + attn_cfgs=dict( + type='MultiScaleDeformableAttention', + embed_dims=256, + num_heads=8, + num_levels=3, + num_points=4, + im2col_step=64, + dropout=0.0, + batch_first=False, + norm_cfg=None, + init_cfg=None), + ffn_cfgs=dict( + type='FFN', + embed_dims=256, + feedforward_channels=1024, + num_fcs=2, + ffn_drop=0.0, + with_cp=False, # set with_cp=True to save memory + act_cfg=dict(type='ReLU', inplace=True)), + operation_order=('self_attn', 'norm', 'ffn', 'norm')), + init_cfg=None), + positional_encoding=dict( + type='SinePositionalEncoding', num_feats=128, normalize=True), + init_cfg=None), + positional_encoding=dict( + type='SinePositionalEncoding', num_feats=128, normalize=True), + transformer_decoder=dict( + type='DetrTransformerDecoder', + return_intermediate=True, + num_layers=9, + transformerlayers=dict( + type='DetrTransformerDecoderLayer', + attn_cfgs=dict( + type='MultiheadAttention', + embed_dims=256, + num_heads=8, + attn_drop=0.0, + proj_drop=0.0, + dropout_layer=None, + batch_first=False), + ffn_cfgs=dict( + embed_dims=256, + feedforward_channels=2048, + num_fcs=2, + act_cfg=dict(type='ReLU', inplace=True), + ffn_drop=0.0, + dropout_layer=None, + with_cp=False, # set with_cp=True to save memory + add_identity=True), + feedforward_channels=2048, + operation_order=('cross_attn', 'norm', 'self_attn', 'norm', + 'ffn', 'norm')), + init_cfg=None), + loss_cls=dict( + type='CrossEntropyLoss', + use_sigmoid=False, + loss_weight=2.0, + reduction='mean', + class_weight=[1.0] * num_classes + [0.1]) + ), + test_cfg=dict(mode='whole')) +img_norm_cfg = dict( + mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) +crop_size = (512, 512) +train_pipeline = [ + dict(type='LoadImageFromFile'), + dict(type='LoadAnnotations', reduce_zero_label=True), + dict(type='Resize', img_scale=(2048, 512), ratio_range=(0.5, 2.0)), + dict(type='RandomCrop', crop_size=crop_size, cat_max_ratio=0.75), + dict(type='RandomFlip', prob=0.5), + dict(type='PhotoMetricDistortion'), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size=crop_size, pad_val=0, seg_pad_val=255), + dict(type='ToMask'), + dict(type='DefaultFormatBundle'), + dict(type='Collect', keys=['img', 'gt_semantic_seg', 'gt_masks', 'gt_labels']) +] +test_pipeline = [ + dict(type='LoadImageFromFile'), + dict( + type='MultiScaleFlipAug', + img_scale=(2048, 512), + # img_ratios=[768./896., 832./896., 1.0, 960./896., 1024./896.], + flip=False, + transforms=[ + dict(type='Resize', keep_ratio=True), + dict(type='ResizeToMultiple', size_divisor=32), + dict(type='RandomFlip'), + dict(type='Normalize', **img_norm_cfg), + dict(type='ImageToTensor', keys=['img']), + dict(type='Collect', keys=['img']), + ]) +] +optimizer = dict( + _delete_=True, type='AdamW', lr=0.0001, weight_decay=0.05, + constructor='CustomLayerDecayOptimizerConstructor', + paramwise_cfg=dict(num_layers=30, layer_decay_rate=0.9, + depths=[4, 4, 18, 4])) +lr_config = dict(_delete_=True, policy='poly', + warmup='linear', + warmup_iters=1500, + warmup_ratio=1e-6, + power=1.0, min_lr=0.0, by_epoch=False) +# By default, models are trained on 8 GPUs with 2 images per GPU +data = dict(samples_per_gpu=2, + train=dict(pipeline=train_pipeline), + val=dict(pipeline=test_pipeline), + test=dict(pipeline=test_pipeline)) +runner = dict(type='IterBasedRunner') +optimizer_config = dict(_delete_=True, grad_clip=dict(max_norm=0.01, norm_type=2)) +checkpoint_config = dict(by_epoch=False, interval=5000, max_keep_ckpts=1) +evaluation = dict(interval=5000, metric='mIoU', save_best='mIoU') +# fp16 = dict(loss_scale=dict(init_scale=512)) diff --git a/segmentation/configs/ade20k/upernet_flash_internimage_b_512_160k_ade20k.py b/segmentation/configs/ade20k/upernet_flash_internimage_b_512_160k_ade20k.py new file mode 100644 index 0000000..cd57003 --- /dev/null +++ b/segmentation/configs/ade20k/upernet_flash_internimage_b_512_160k_ade20k.py @@ -0,0 +1,68 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- +_base_ = [ + '../_base_/models/upernet_r50.py', '../_base_/datasets/ade20k.py', + '../_base_/default_runtime.py', '../_base_/schedules/schedule_160k.py' +] +pretrained = 'https://huggingface.co/OpenGVLab/DCNv4/resolve/main/flash_intern_image_b_1k_224.pth' +model = dict( + backbone=dict( + _delete_=True, + type='FlashInternImage', + core_op='DCNv4', + channels=112, + depths=[4, 4, 21, 4], + groups=[7, 14, 28, 56], + mlp_ratio=4., + drop_path_rate=0.3, + norm_layer='LN', + layer_scale=1.0, + offset_scale=0.5, + post_norm=True, + with_cp=False, + dw_kernel_size=3, + out_indices=(0, 1, 2, 3), + init_cfg=dict(type='Pretrained', checkpoint=pretrained)), + decode_head=dict(num_classes=150, in_channels=[112, 224, 448, 896]), + auxiliary_head=dict(num_classes=150, in_channels=448), + test_cfg=dict(mode='whole') +) +img_norm_cfg = dict( + mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) +test_pipeline = [ + dict(type='LoadImageFromFile'), + dict( + type='MultiScaleFlipAug', + img_scale=(2048, 512), + # img_ratios=[0.5, 0.75, 1.0, 1.25, 1.5, 1.75], + flip=False, + transforms=[ + dict(type='Resize', keep_ratio=True), + dict(type='ResizeToMultiple', size_divisor=32), + dict(type='RandomFlip'), + dict(type='Normalize', **img_norm_cfg), + dict(type='ImageToTensor', keys=['img']), + dict(type='Collect', keys=['img']), + ]) +] +optimizer = dict( + _delete_=True, type='AdamW', lr=0.00006, betas=(0.9, 0.999), weight_decay=0.05, + constructor='CustomLayerDecayOptimizerConstructor', + paramwise_cfg=dict(num_layers=33, layer_decay_rate=1.0, + depths=[4, 4, 21, 4])) +lr_config = dict(_delete_=True, policy='poly', + warmup='linear', + warmup_iters=1500, + warmup_ratio=1e-6, + power=1.0, min_lr=0.0, by_epoch=False) +# By default, models are trained on 8 GPUs with 2 images per GPU +data=dict(samples_per_gpu=2, + val=dict(pipeline=test_pipeline), + test=dict(pipeline=test_pipeline)) +runner = dict(type='IterBasedRunner') +checkpoint_config = dict(by_epoch=False, interval=1000, max_keep_ckpts=1) +evaluation = dict(interval=16000, metric='mIoU', save_best='mIoU') +# fp16 = dict(loss_scale=dict(init_scale=512)) diff --git a/segmentation/configs/ade20k/upernet_flash_internimage_l_640_160k_ade20k.py b/segmentation/configs/ade20k/upernet_flash_internimage_l_640_160k_ade20k.py new file mode 100644 index 0000000..4163ef5 --- /dev/null +++ b/segmentation/configs/ade20k/upernet_flash_internimage_l_640_160k_ade20k.py @@ -0,0 +1,84 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- +_base_ = [ + '../_base_/models/upernet_r50.py', '../_base_/datasets/ade20k.py', + '../_base_/default_runtime.py', '../_base_/schedules/schedule_160k.py' +] +pretrained = 'https://huggingface.co/OpenGVLab/DCNv4/resolve/main/flash_intern_image_l_22k_384.pth' +model = dict( + backbone=dict( + _delete_=True, + type='FlashInternImage', + core_op='DCNv4', + channels=160, + depths=[5, 5, 22, 5], + groups=[10, 20, 40, 80], + mlp_ratio=4., + drop_path_rate=0.4, + norm_layer='LN', + layer_scale=1.0, + offset_scale=2.0, + post_norm=True, + with_cp=False, + dcn_output_bias=True, + mlp_fc2_bias=True, + dw_kernel_size=3, + out_indices=(0, 1, 2, 3), + init_cfg=dict(type='Pretrained', checkpoint=pretrained)), + decode_head=dict(num_classes=150, in_channels=[160, 320, 640, 1280]), + auxiliary_head=dict(num_classes=150, in_channels=640), + test_cfg=dict(mode='whole')) +img_norm_cfg = dict( + mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) +crop_size = (640, 640) +train_pipeline = [ + dict(type='LoadImageFromFile'), + dict(type='LoadAnnotations', reduce_zero_label=True), + dict(type='Resize', img_scale=(2560, 640), ratio_range=(0.5, 2.0)), + dict(type='RandomCrop', crop_size=crop_size, cat_max_ratio=0.75), + dict(type='RandomFlip', prob=0.5), + dict(type='PhotoMetricDistortion'), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size=crop_size, pad_val=0, seg_pad_val=255), + dict(type='DefaultFormatBundle'), + dict(type='Collect', keys=['img', 'gt_semantic_seg']), +] +test_pipeline = [ + dict(type='LoadImageFromFile'), + dict( + type='MultiScaleFlipAug', + img_scale=(2560, 640), + # img_ratios=[0.75, 1.0, 1.25], + flip=False, + transforms=[ + dict(type='Resize', keep_ratio=True), + dict(type='ResizeToMultiple', size_divisor=32), + dict(type='RandomFlip'), + dict(type='Normalize', **img_norm_cfg), + dict(type='ImageToTensor', keys=['img']), + dict(type='Collect', keys=['img']), + ]) +] +optimizer = dict( + _delete_=True, type='AdamW', lr=0.00002, betas=(0.9, 0.999), weight_decay=0.05, + constructor='CustomLayerDecayOptimizerConstructor', + paramwise_cfg=dict(num_layers=37, layer_decay_rate=0.94, + depths=[5, 5, 22, 5], offset_lr_scale=1.0)) +lr_config = dict(_delete_=True, policy='poly', + warmup='linear', + warmup_iters=1500, + warmup_ratio=1e-6, + power=1.0, min_lr=0.0, by_epoch=False) +# By default, models are trained on 8 GPUs with 2 images per GPU +data = dict(samples_per_gpu=2, + train=dict(pipeline=train_pipeline), + val=dict(pipeline=test_pipeline), + test=dict(pipeline=test_pipeline)) +runner = dict(type='IterBasedRunner') +optimizer_config = dict(_delete_=True, grad_clip=dict(max_norm=0.1, norm_type=2)) +checkpoint_config = dict(by_epoch=False, interval=1000, max_keep_ckpts=1) +evaluation = dict(interval=16000, metric='mIoU', save_best='mIoU') +# fp16 = dict(loss_scale=dict(init_scale=512)) diff --git a/segmentation/configs/ade20k/upernet_flash_internimage_s_512_160k_ade20k.py b/segmentation/configs/ade20k/upernet_flash_internimage_s_512_160k_ade20k.py new file mode 100644 index 0000000..2cc0a4a --- /dev/null +++ b/segmentation/configs/ade20k/upernet_flash_internimage_s_512_160k_ade20k.py @@ -0,0 +1,68 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- +_base_ = [ + '../_base_/models/upernet_r50.py', '../_base_/datasets/ade20k.py', + '../_base_/default_runtime.py', '../_base_/schedules/schedule_160k.py' +] +pretrained = 'https://huggingface.co/OpenGVLab/DCNv4/resolve/main/flash_intern_image_s_1k_224.pth' +model = dict( + backbone=dict( + _delete_=True, + type='FlashInternImage', + core_op='DCNv4', + channels=80, + depths=[4, 4, 21, 4], + groups=[5, 10, 20, 40], + mlp_ratio=4., + drop_path_rate=0.3, + norm_layer='LN', + layer_scale=1.0, + offset_scale=1.0, + post_norm=True, + with_cp=True, + dw_kernel_size=3, + out_indices=(0, 1, 2, 3), + init_cfg=dict(type='Pretrained', checkpoint=pretrained)), + decode_head=dict(num_classes=150, in_channels=[80, 160, 320, 640]), + auxiliary_head=dict(num_classes=150, in_channels=320), + test_cfg=dict(mode='whole') +) +img_norm_cfg = dict( + mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) +test_pipeline = [ + dict(type='LoadImageFromFile'), + dict( + type='MultiScaleFlipAug', + img_scale=(2048, 512), + # img_ratios=[0.5, 0.75, 1.0, 1.25, 1.5, 1.75], + flip=False, + transforms=[ + dict(type='Resize', keep_ratio=True), + dict(type='ResizeToMultiple', size_divisor=32), + dict(type='RandomFlip'), + dict(type='Normalize', **img_norm_cfg), + dict(type='ImageToTensor', keys=['img']), + dict(type='Collect', keys=['img']), + ]) +] +optimizer = dict( + _delete_=True, type='AdamW', lr=0.00006, betas=(0.9, 0.999), weight_decay=0.05, + constructor='CustomLayerDecayOptimizerConstructor', + paramwise_cfg=dict(num_layers=33, layer_decay_rate=1.0, + depths=[4, 4, 21, 4])) +lr_config = dict(_delete_=True, policy='poly', + warmup='linear', + warmup_iters=1500, + warmup_ratio=1e-6, + power=1.0, min_lr=0.0, by_epoch=False) +# By default, models are trained on 8 GPUs with 2 images per GPU +data=dict(samples_per_gpu=2, + val=dict(pipeline=test_pipeline), + test=dict(pipeline=test_pipeline)) +runner = dict(type='IterBasedRunner') +checkpoint_config = dict(by_epoch=False, interval=1000, max_keep_ckpts=1) +evaluation = dict(interval=16000, metric='mIoU', save_best='mIoU') +# fp16 = dict(loss_scale=dict(init_scale=512)) diff --git a/segmentation/configs/ade20k/upernet_flash_internimage_t_512_160k_ade20k.py b/segmentation/configs/ade20k/upernet_flash_internimage_t_512_160k_ade20k.py new file mode 100644 index 0000000..6c61fc3 --- /dev/null +++ b/segmentation/configs/ade20k/upernet_flash_internimage_t_512_160k_ade20k.py @@ -0,0 +1,68 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- +_base_ = [ + '../_base_/models/upernet_r50.py', '../_base_/datasets/ade20k.py', + '../_base_/default_runtime.py', '../_base_/schedules/schedule_160k.py' +] +pretrained = 'https://huggingface.co/OpenGVLab/DCNv4/resolve/main/flash_intern_image_t_1k_224.pth' +model = dict( + backbone=dict( + _delete_=True, + type='FlashInternImage', + core_op='DCNv4', + channels=64, + depths=[4, 4, 18, 4], + groups=[4, 8, 16, 32], + mlp_ratio=4., + drop_path_rate=0.2, + norm_layer='LN', + layer_scale=1.0, + offset_scale=1.0, + post_norm=False, + with_cp=True, + out_indices=(0, 1, 2, 3), + init_cfg=dict(type='Pretrained', checkpoint=pretrained)), + decode_head=dict(num_classes=150, in_channels=[64, 128, 256, 512]), + auxiliary_head=dict(num_classes=150, in_channels=256), + test_cfg=dict(mode='whole') +) +img_norm_cfg = dict( + mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) +test_pipeline = [ + dict(type='LoadImageFromFile'), + dict( + type='MultiScaleFlipAug', + img_scale=(2048, 512), + # img_ratios=[0.5, 0.75, 1.0, 1.25, 1.5, 1.75], + flip=False, + transforms=[ + dict(type='Resize', keep_ratio=True), + dict(type='ResizeToMultiple', size_divisor=32), + dict(type='RandomFlip'), + dict(type='Normalize', **img_norm_cfg), + dict(type='ImageToTensor', keys=['img']), + dict(type='Collect', keys=['img']), + ]) +] +optimizer = dict( + _delete_=True, type='AdamW', lr=0.00006, betas=(0.9, 0.999), weight_decay=0.05, + constructor='CustomLayerDecayOptimizerConstructor', + paramwise_cfg=dict(num_layers=30, layer_decay_rate=1.0, + depths=[4, 4, 18, 4])) +lr_config = dict(_delete_=True, policy='poly', + warmup='linear', + warmup_iters=1500, + warmup_ratio=1e-6, + power=1.0, min_lr=0.0, by_epoch=False) +# By default, models are trained on 8 GPUs with 2 images per GPU +data=dict(samples_per_gpu=2, + # val=dict(pipeline=test_pipeline), + # test=dict(pipeline=test_pipeline) + ) +runner = dict(type='IterBasedRunner') +checkpoint_config = dict(by_epoch=False, interval=1000, max_keep_ckpts=1) +evaluation = dict(interval=16000, metric='mIoU', save_best='mIoU') +# fp16 = dict(loss_scale=dict(init_scale=512)) diff --git a/segmentation/deploy/configs/_base_/backends/tensorrt.py b/segmentation/deploy/configs/_base_/backends/tensorrt.py new file mode 100644 index 0000000..a7f4c56 --- /dev/null +++ b/segmentation/deploy/configs/_base_/backends/tensorrt.py @@ -0,0 +1,2 @@ +backend_config = dict( + type='tensorrt', common_config=dict(fp16_mode=False, max_workspace_size=0)) diff --git a/segmentation/deploy/configs/_base_/onnx_config.py b/segmentation/deploy/configs/_base_/onnx_config.py new file mode 100644 index 0000000..43621b1 --- /dev/null +++ b/segmentation/deploy/configs/_base_/onnx_config.py @@ -0,0 +1,10 @@ +onnx_config = dict( + type='onnx', + export_params=True, + keep_initializers_as_inputs=False, + opset_version=11, + save_file='end2end.onnx', + input_names=['input'], + output_names=['output'], + input_shape=None, + optimize=True) diff --git a/segmentation/deploy/configs/mmseg/segmentation_static.py b/segmentation/deploy/configs/mmseg/segmentation_static.py new file mode 100644 index 0000000..416b781 --- /dev/null +++ b/segmentation/deploy/configs/mmseg/segmentation_static.py @@ -0,0 +1,2 @@ +_base_ = ['../_base_/onnx_config.py'] +codebase_config = dict(type='mmseg', task='Segmentation', with_argmax=True) diff --git a/segmentation/deploy/configs/mmseg/segmentation_tensorrt_static-512x512.py b/segmentation/deploy/configs/mmseg/segmentation_tensorrt_static-512x512.py new file mode 100644 index 0000000..1fa5ef6 --- /dev/null +++ b/segmentation/deploy/configs/mmseg/segmentation_tensorrt_static-512x512.py @@ -0,0 +1,13 @@ +_base_ = ['./segmentation_static.py', '../_base_/backends/tensorrt.py'] + +onnx_config = dict(input_shape=[512, 512]) +backend_config = dict( + common_config=dict(max_workspace_size=1 << 30), + model_inputs=[ + dict( + input_shapes=dict( + input=dict( + min_shape=[1, 3, 512, 512], + opt_shape=[1, 3, 512, 512], + max_shape=[1, 3, 512, 512]))) + ]) diff --git a/segmentation/deploy/demo.png b/segmentation/deploy/demo.png new file mode 100644 index 0000000000000000000000000000000000000000..1e82d7a0773cea14b36f0021fea603de0961b5d8 GIT binary patch literal 307861 zcmWKXcRW>p7{|YNaqT^ljO?vKGOv-Hy*JrABKuw=DTzo(p&v3cvl1>cGD5P~rR+U2 zFZbuKb6)44^E&4|-}8K)&-?RzuCJ?3LCQi30HDy+P&EVqesu~3NZ8duh;BOp02t8M zF}^P@E1{~depgvWT3K3JRaQm!zPyHlB`a&%WI(S zDBM-hQobX9N8LzMTtP}vOYt`9uB5We9i@9L9IToq+M;*GQ98h_cvT)KG2LKM z(^r*Nlh-!Y5tkJgl@dm2DYNl%(z7s$$lSicbwgTJ{+_a|s-B9_Z4o^yeI0XMc{Mpr z6AcbNZYfpSn<9Jyl48PQeB6RJC8UI9@83Hn5NH?~4ha|gg!2ccnzwHAvv6?=N(yQl zs!80tC4Em+T3+&|Ag_Xwl(LrG11p_V0zvuyy=}sYxr?QLn5R#quc3{R+5^Q$!H#O0 za*rc?WaV%9Mf>_Zwzqb-aDL=y<6*anKhm_&A`l4kJ8N8`B91=xhPJwHkDU!Hbhx?M zOda$W_cm6J_skxe286m8IhZwF?UQDF~#+*Pggl#TR4U&h40dwK7H_T(xqGBwu5L03;*M*O~tx|OBM zJ<+GBvHA}mVt@bWof&kBd}OJwTGm>dS(5vvAn8+WVdIZxXKMpX|Hm(K-y6u?3eHS2 zx0lfia#%Q7O{*=Bd-1yTXNQ~W-Dl~KBcFH@{&pP~r&fEIc6WF8^z{7r@uRo5x390S zzrTNAVBqJ^pM!&gLqkKu!^6LR{Tdk=`ThI%=;-L!*x2~^_{7A-SIr&)>g)mzI{6mzP&oR#sP6*Vfk7*Vi{THa0gmx3;#n zx3_n8c6N7n_xASo_xBGD4h|0w|NZ-SbaZrle0*|pa(a4tc6N4tevZfEFD@=FFE6h? zasA|48-M_ysj6fg{67EUxm}9qIDzU38A# zFey!Edj#XR!=J)}Ge4c`0!u?Aw?e;W*&at;R=R%vBdv!$YAL_Ek%H@scxmJE5^IbO!4~6yCV8BU&-yGje=5y*d16Bw#(RT7}1%2O4Fr zFXH_^d)V!zRzocQ-t*Wjt8cPA1gALtpS2|tT8-BH41)t2Cp^z6El#Vet5wurq~EKa zuJ@gnV3G7IexsIWU?G+!X-wu69eQHOa)L1^*WWYVRr8Qve%xri%LVTZWMOc1DIRqs z%NHsO_|_p;Y+R08T~I#?%%CcXr3f(HSXX=8+Au7COH#7-b}hq;@QfTw^d`TX?iLQdzZbAQ^VDqc;lYAQsilpL|0=SXGR@e`taS9w>mGbpe>Y#cm|crfMR%63 ztzt;{-hSuxU8F+rDN*?5erI1gd&k3FB3n_r{P92CNn#3J^(Xtx^UY`i$&5l|v>E?fZL_le@=CD; zUtC^XzpHNZpQwo50}}=^IP|st$6s!5pp%!zA9P1ErtNw@MA{C|jg3joNR`o%#vIv> zTK>(flNU3gjsL83%7-^X2b>o6blk`bH0fN+RO*YSnxhJ9=52Q7+OY3-fHgKl3nO8-g|!caKviW_0*E+2E%M%h=BI)hnP0E-yf;qmSmKo zL_&ryKEYm}o?3)n>mM(-Y0?sV6);Q+$CwH2WTXS^(w8h^_1VtqI3m2WM-k%<&zAcN zfA5F`^MO_3gL!0d`z}Ag4n=_zlwz*>ZDzo0AFW$xVK=PkmQ-H;vsk;A(j?uO>;!62 zYs|S=97&ihGVWoZEsgD#$4)jEHr=)oH8VGH`=Knr&c-N3+WYDa@}zp5s1_xdY?Cq@ zOZ8vl1~r>MK*gYfn%FY0IvH(dER7R8(P@|)&Abu=MR;iVNbQPbsoVc&54 z5rn|}hS;6P{FzWli_^Hd~*1PyE?8M7VWxM%St3T$Y&h5YVjnkm4 z(fa&m(d;;~AoQi|n$smCqe+1n(`GM!(^?CQq5evd{7r-{?W*;_)pIOx@~m zZ5O+P9zr|*Eo>*HgJKmCV`Wt4jB;cP@$6_T02rUu)m8Lu+y^@UEd|ArfUh^QlkDtT z!?sBV&s`&43dBqraj~ld{d2x??y7ywGhx*>?vR_FyCURlGM3mGex1dW+`)qo;`6Z3 zP~H5;#4!WIZaKCUJJ;x5tbAe;y5l*+2?zHvZ9(4204D>m3|}ly;gD|;^dJ|~zpP|* zY+l988l)LxGBPsyO9^6sA})IlVU19;0~PwrpXY~2*!O6Fuq)R!C}T3wZue4Pqt{vQ zn3B~OUM8S#$V(T0`d^n?rm-kZ(U4;zDID;X#-~;lR^K!7G2S&PBL8bxWq#9x{yB^D zGs@>7cp}L&^fi9)z~r|Rcyvs)78PIt<$@96QhhOHA`fZfrZ?5B!Pc4$Wwv&g}+ z2tS6*3<;`F*VnmzFhNoD*-65ypRh}IPO2dIoT);wCGs&rAvoM|%6=zFfM|E=cJ050 z$!-zF9|wFeOf9})r%;XHHhs~s;<_a}`4?SDU~iTa#5{NbGbR5%S*Zys%k9q3%J21+ zh|&~%db_i@kAN2IfBajbmJbbSZE3dgr2o5qP-CiC$5>ZS!Bki4x%5IZ?7mWIqi+Mw z{2mnqm{PhB?6cNtjv*jM{((T6K_Aovuoccl1ABTbDGV3`SXr~o2T>y-=KogyG~G3_ zB3rPqb86|pt#8`TM6!ZEW&f&Qh375}zH0dK zPH!W-v(rUgf_YS3sXHBm$)Pk|)_G!uTXCjG6?#)ccFFJ%1PWkR4tP>#ZHQEuZ-h=b zOFh$(->!0W$cr7kUNmVS@6W}hu!{hoK*l%s+1u%)`67CaTj)lrxL%6cH%=`2a%-{{$;fih8;uC}#`kmzE z9ar5BJ->yu`n1!Y6x1wgR1t=WOYx|sZ?AcZ4wU1($z}pbFB6XE(fEebLxFcs7NJJ1 z_>91dve|i8az%$jvFsVw`THB__LDGL2!8aE-Fy7f7=rJdP@4}ucBWUnB0G17A`%|?J8EjWsKMn*rHA7e@cUW%OGbVyH! zU=J*E2DeFK&L50zPjp!o`rnRqk4?bblRVL`_^wUm0Sdezt|l)V-~N>kX~2tvKZ)`n!6DU@aIAM~p8h5&s?{|Gg1O03noIq08E_Zm2+I`X{_8tTY< zfZqp~rU8km&2rg$rZYha8Ta=7{VXyOrm@ndk|O39qsq{9!46H5v_XKqyG%LI5ROv9 zO#Ytr4N^1X2?($HYAH-B(6Dw+0`zMhQlg-sr?;>7XN6l(9|JWMNBp-vr_8jr%-~Al z^0#m!%3u)Dk5K(Iu_)V>MLrdzKJ*tVh>iGd6up@w3{rBo)#u9fXTXDmGe}96I6d0VfAaP0LbCi?jgy z``B#$>W4KSzE2-XoFm7^GQ~*ar412>OaDW|K*{A`b-5x95CB!TPc*R;|5s~cnE>U!qqygx7dm`G!tl9Kd-X^$@sw?_E(Hr5in z=Lq*_*8kih*liLx!?NN*oy9?E>&$amJo@1!)!p%Pp)<47Q6Bv7sVQS0uBldK^RnB5 zC5W_$2R4D;GKu_m^&82+kQ6x$_Bq{_Kfru8L;rJDx*iM8CVrFN3aBI~{^7))olQUS z{UN(c(^Syft^~xNf?XnUI1^WlNTl-GJDr+YNpg`(`n2^H}HRyaNQ|GMLhaUL+-g&mm z1cf9AG)zK4E)7P}i|so%R;(58PoWG*LFuRrjFdx}Vs`0q=7-(UGj;U`dAv1!fKgX8 z%S_u#%a!+TA=t-ZHS$2BVCU*-h|Hy~dqTEBF=`;bfhw90)GnM6Ud0Ps;T?K>$cPzE+* zENZHe<0U`&X$KlM-gUniO?)uZ2Csx%SzsWYHnvLmo* zh15{hv6Dp=VMuS!4U7moJ*q*Iu)9WfqO>#Ljk)NI*j*H}{dSJ0WQ*`8Anb8J{%kSH z(QI8RoSk=b|3ts9JY&+cSRrt09w~;fFkKwlwx8wrz_1;y{Ftq{fyhIX>H$J`ICj|> z-K*5{RW%uLR6+YgpTHz!td{ug>zA-cZGq~ACVSejw6)S4%V$|ALOn%&-$s+l&xO^| zVTUaeBm~vP#{;0s`7-3PJDh4?DJ21Nc&^qu{vVG1O;fdCEuV&-{+ z;5LLFrOz(@-SI%dH8@CWIca5&`479RVOZ*qsLQ|H2}H{@+`ZBP#4YT|-wcYQFrpGz zyne>xC_Dg|Dht7wSJ`WhxA1JN?eouwHfZ@V9RqbpI?&%m4ls5|C^axhfdSI~H+IAa z>e|N+XFoS3s3<9IC9?)dAoi7FKhs#-+EIHF@XmxZsTAhjQ<;@pBn=YsAGSC+QOxW+ zevRL33n`F}Nk8pBM-4JP()K)Fc77ws$x00Hx=283Z)W?zg$JYM&drH2ET(>2;b`Cf z$EDzt_C~=d0KttB*S*36y}iA;aBQ5KSUb+jWJfsQ1j|60l)+5EPz7J<7Uj#@%RlaB z2my5nfMEbw8jU3u05mWbgVkgFvFu)78ym5Lv|y_2KgVM^X-t~M;EMTsAe5cBwD|^~ z+YuEt_^zW-Qnu-zgXa;QwbkH!OT@wQz(&BMk%($}1ggBI@bYYeka>~wK$k$%iss$E z6tiv+KXh_-j*RT^@XlQ)P$Sx2r+Abys9*|u$2m-Kp$Mi29o~^>!d}O?Sq0YAjyjJB z6ZMg+8^=JcB*qPHg{p1esnARL6Dif*T|_8E@16hHCoHrcUre*%!+Z(j>UWg@2C8u- zu}3?XYKzzY^N?iUVvnSd__zMcy>DzKY)URMS6aW=@?wJOCYecO%E%IvMU&8S{_sbtwaNxZ*fTTvNR(EN2;R(=!@af;!^);fP5Mr z_>c2eJf;m5l-3nbYTrgzVkpjup#``Dhuh8;g$B8xWNrH`2Pko^zOHRnui?|ZWT8im z>##gBoNxd0xu4! z7_Oenynp4K1`wo;4IOJmW4RU{e(q!N%u)39_Fh0=6ejt}6If@E!jX{&mlwR+M9ZOv z>zw`z!+FojZ0?wR%YQsa)fREHFXznx!E+%gVCEKW zn+m5Lu>{}w=CE^)k|eoc6rirlYO4^jN8)l`#C8A>cA_E;9khL>NcQ$ci7~*8U;h*| zL%a8NGv<^1Z`yVoT7fnD-{3j+y^ZmJUUl_{9)cpVrR8EfMo#+2ZxRElFJF@PQH43* zc{Vl%6+fSl3oDD8V*3)>p4E3^z^X574_)+;ya4x6g~#!Shd4|BAUDX!c>pl<5OiY~ zKVSrI^)2yYi_lW4un)QwBin(E8RWTbxLYtF-ubxbBYejBx#abwC@RGm1WJeuqX7j$ zuDNbqmc0v_+{H~Lx(#}79IN%7T7*>|8Uk$8x0heE)Wz)_jHBZCK#x`SYd`6=XH+>o zbYlmlh-mF}4x)x)2|XFlb8XcrUv@(Ay!N@U-gR;D#?C0@xL%mDHMWwKQ@RT zweCbu2G0e85eEu{Z?26uxQY1_yI#9Iw!PhB72A>hH0pjAc|N zBfB0t_^d|`AJzI;c;6v=f`J25wR>j=tpiEk*qjVNAMegvDniu09j5;!xNqfrMaN>F zeiYJsD0DyrP?A{)x$jO8r7O2kx-<=a=<6jc9DY(+RcE@uu}<>lKZOQnWcUgG^v*(i zp&ol{OUS{$jix6%EiI80?;z#Jhf_}X+H!A_FjHOg_;z#Zk|A$@(u4iszA%io07KZC zVnpH@D7`dLaO}p@Ug*cN*IAa?w&Z)Q8l1!zzO0&~%mtEf3w|xl)g4)~-PJc}!B?TL zbT-9i>t`{KgDJf_7i1v9yD7w=40;k@s=h^aSIC~=TNfJ2RL|?iLJfwfXfV$VzzqPV zl_$h^_~03qLuTt9VdYPvNZFZz-PTrotj+RpG?w zi)3l1A9N%u|L94ryOM>|8C6X(7-pR1nD=N#qYVw6auzz7Dwu$M)fa8&2|tx`S_NN% zAw6V>m6Y!M`1~5c@00z28-m?a$K=(e@uKYGHleOv`%B$IGL;baBhc2t=K44d9Zw4& zLmYGLj~CQ6pJB(Q_76QgvC$wG8_#SVq}}>slT@VQCu_fm$kLWv|H@7`Yl?trnE)=?;UN-nH_yuQ}DV8P

ycMN0&o(M&PeZ;L*$^;UUzXMEvzR--{fB%a=QaP-Hq2O zNI(_)-+P>Fg_N#XxR4RZG9O55lr zIe1296UoMt4!B-`M@8E66i8KTYB?CH8C{QL_sN`JC`9~P+?WO017=@LG5fkxh zUTZDr#IeTOTl;xp8-)mCu;0h9+Td--8a38zT;cJ2`^sZFPe^bAmlr9_Qtz3jkxZj zWf;lLOI|KmPxgvX?z|+!M#VV z-loD!8UF;r8wmAw1u>Y``V)4xnQ%qBnr|x^T8Oh8q2#@%tCwk9wI@xvw7=oMg1I^n zAsWvIs_4EP}QGQN4W5eDBvzU)Iv zfMwB~-q#O0b?BED>^h&5-5&EWvUqO?5i_6fywrxX7&6OvF(z~dsrOX);QC5ROSOL| zzQ3S`sK(p_800ikn!QT_{34GhrRiN0Y}9Es3}u5KjciSYd~oJmSb*^Rs_0uJ$4BfsXX-P&a58~2$_X{ zi|N-rkWZZw6NcA*E#~IDiQdb(xEE(DB(#813;M)GedVD7GWMH7LNwx)8g7tHUfLM@ zg@Fw-`&L(}c|z7DhcFSo;uXQ?CM3fRuj!9}4#y(C#l>w`bT4xufzG);!Vg&^0S60y zrb}E?(=n0?aWK6edBf?#0*KC~5{`2Dg?{h_Whc}D`j$D06!|oo zKl3okO@akU9;ZN!iY2NDlgCk+a+I8$vHT%sk^ByeWq)vLu(+@#qWnn8E0tnTHY;oA zp^v6oZil{8K40*tEXpp)@ z2KK#*dZ`5J{R%z#V1?_ObC5{R4)Fy(5HC#v&%}O#rLc0$F!K}d%H!EKzB7E*%gh_# zqZ(&Ke4A?+uD4&J$-XOoZ)mZdJGV1Pk29q>;i?yQE&iH`8R1xsK6RJpgppw^Vj^~@ zIKC6cBO5~#ALat=ZBWQ0B4vL|dicMH59ZyzoF))h;%b#9R$<~#7bWcQHU-6hvJxna zXAlA;H_s#h6ats$4hjoe=QqhkL`20z+2t|<<#_J2HlAN0T%cCKnuvp3Sv{pF6tcI3)R|w$YiHyI{$SymNQU+apVqfCGTFJZF&BjB%Y)mxAprs z9M^C7sG)es;Wi?RSVSsovWB*cL-sExh>r-RyxjRQi_Q+as22cgQ2dGqZr13_zm@G- zBUid&!ieUS;vH;cD_ZLRGb<`D$|IK%wDYzd-v`d{? zf1jFa8jfDf`Jp_io~6f_@1GEm7xadj+d^8C z7K4fskMzyyq(@F}KMICE`Wi2$-*-~bD5)IV#=_@saqBG&TXCLi5c>V~Wh62-u{666KA8p&8Zc?Eta`U2mP-Ny| z+~@GpT~6ENoy`()(uaP6#Z!>%r^WI;+GA45Q?u4yM%*S)+B+oB&S11<^WC@RFwb8L z=|Z(79{HxF@-ycNRW#nLTue zq#2zPL99ewarQ^3aq)vfQZk8Kc-9_>W*}u}l+x$_Ld%xy>D}>hXBufUeRjyexhv?f zr8$XF_wpRd$_VUo{1YSkp)n2%KT^A(slgNQD_o@PGLL#ZBm@o8B37dUFj3ioj>Q<$x z>j+_*$6`<{RhM^FhCCZ90-MZ11d56bXPc(39xVdg;CS~QcXMdv%%*Mab(QR3dD}C- zKc|XQYkD;&doMU@37YTbA8N%O{Jtc(cFxRisvYXUulDNT1oH1 zDK+IY#^qwD`RU2t#>Sni25NJvr+eAo`hmhxhag-_XjrPDL8WZB`n|e^)`8lFQv!2b zLdodp?!(k5?S|j$9?vB)z+44^4T9KyJCP2V5Yxq|UdLTDoGN~*5X;D(FZ{t8e1_bX zbCiW>s8g-O<46WyBiW^-1dR;e8oeD0)oG&$SjpNV4`!|)uQqA-?rhJWf{?vr=fYIv z^nD&({3FoqVUh?H_)gf}$akYJdR!}G)1=Ea>wd3Tf6o27PsNJm83dq{S+IT!yAzQV zLGW77XWj3P_QqVs$1iGDFCSh_n4erO(Cc&cPVtE9AYO1XW=&&q(eR4!|6$T1O;oWTj>THl_l_p5u!%`A9EG4qQK=wOIj@M6rr6@o#uL z6;V6msDNj;%Cmue$;0-IfrUWqM&MTfq^4`i9=or-a(s^T3MI6?tkp*aS})}kWMjL4ZJ3IJz|(NCSl?n4pHJ0Ay0m#_+=H*H1K=eM!d`WZ$S2kCkARK zGV*=r0Rai78yb{YlB9vuiiO)*m86wl>fZk`|9r3I+)e9AKtP;zF{+r*L&VeHrnK6+d$#`F$W+e91vlgOhLq_- z$uCsbZ|eineT1Y;JZI#{#nJVd;kHtPgD}~d7~*Zb_Kw8~js$|Z3iRbT<|ssl%O5Tc zG~FTGnnZ8sm{a1j^l_ra_>cc`3RGGP=^iM05n=wF7hNs{ULgl!P-%EBCL|=_$-%$^ z#c1S*uPxA(oMyJ!I&P$+-3U$0^#s1XOTzi{wboefU>e`^PTh}%66PEON{?iyxiL{K zk;ltfgpMn(=eApv{o}2J&GGSh3%J~i1bn_v^xutcTPP#^(K)*LHPMmSOuE0Y2|u0v z#h{zqOCh!;;6b8$@i>)`FOreiV!;~n*ra4Lfxj`L%tj-4m1qY_!Qnl@=RYhK|* zy0ysxDAxHIwZq(|D2sd3@&1^}5V03VuURqm@105`jL0UAs_xj4hq@M}-i`0=CFo?H zH|Z@G_&I&E3)2B%)2b^h`Ff@&I2qnl-_ZY7=l~RC6c@KDz5duP>eCW;k|Ctl|MW`GBYb@ zJOyy>F{O^VsoY{EROZklUAsbGHf0b)2f>kx6bP@Wi~x+!6X#0ToD@OuX9uT4 zW(EOGNz3)?|CDyi3vMEq8x5a=iLBSx;5PcD)XGRd64@x9@#~4jijR1zZ+Bf19{r_0 zg81=R6n|f2xJiM)yb=TkC2D!p7MMvreuv_TYIr$*zvOpBsrIYOe&cMd z0-SS!>!Sj>mR$H3qBV8#^_Ol(?}8{syo(Gj<|ut1Z?(=g8a?IWfs|`5p&RZe z3G8DK*v^J^S2a94U)<#cl@0F>uOyM8X8#4Z-Q(5cwbcAJOyNs4`$E_w8nDF}T6=mn zP=A_OE8nN>8PP^%m4hM@r+7~FCp+6)F7%z#I>`Zn2CfRjjwiC!o07#APd@E7wkOCn zDz}+vA!^^|S$@zh^&Z#o^ipT|=CI|LW$3)7NJI+kflJaaJ8g6EnK4s+9=n9kWqPOL zc=Y+vsi_@yC8shoTQbcR2He7O+4(TH^3;tLnF|N1F>7fd?Cv-A#moi26bFts{j9kM za@d92|G@s)!Pj##6vL_ME>nMRax!Kdc3!`1*}dd0h|AVs3f` ztZ_*7Ut_jq)&=r6KO3Wl_gNoaKl)N_;!Emyr!}5A zTNws=dUOM%58EvD!Ob?F+Z1p=oWJK$x>PS^6&!n;!}aG^iMUY|!|7uNJ7?|=h#)5s z5$$W^u~mh#D-HLniorZ^#_JhD83{3d_pnskAmYUfqij;rTo1dnt1#jl>u8h`0z&dh z;EmHUuvQMvi!B1})M-S{pAgq$ls{4*5x19mFR1X-pA4P4AN+0=fnx&~a^_p{HaBs< zXY+p(>^v?dw$$$s=%E>q2LGko<`du4NkNfI7UX)#4;f4&Mf`&+W|fp06eicgEJosW zowz+|&Qe$_{T2^SXg*s@Pk>t+Tu>-7IV|+&k@g1kcqZGcaGS3-(z+*3QT}i&rw-; zq2E6^)ATj~eNmO_g8*|4VXu7U+(koI@j3wW0eq{YhM;mM@ApN%Tq*RxqyY>@oE9tP zb|>sA)(s4PFIH!{u)MPWSA6S9sDba3h4@^mRa`YiG;}xrx6_8r$7K1{a%*y;nzm|H zx!Y7_`dj9`y+3W`S)8u0;jkMU1FFlP7jRr*X^7wT>rs`1HDB{jP`9XY>_`mPGtpV8 zS8sj}hJ-GZ0HAQ6e$UgBIN_>8%J`$Qh;+9xoy_ zO+SpHr%+GpU2?jiEkM1mYN!wDH%#&$v}x2OVN4xF1rZXhwFIeeO_CVIKEet0_@PS0 zxNow8hW6(0BtaFk!*r>IX(D`IL(3cmyt6r`8$AE`?@plNe}vZ%l`BUB^sBWSA|X9$ z7`y8kX(*4^UnN;d#>)*rvYdVRKzHZ&Azy!|WO0JihU*Ribrt11*&Fd@57}GPbi5C2 zh^kM0W@}6vjPJ3qUL3LnK+1gif7cEJ> zaIW^SATcL=tfmE6uy&qDiq(cfg5lhN7cwvy#izHZ1#v@E zC+L`a;wnNhqdeid&d5}d&(!10#wszb(ia}+^V^3CPQ)a19(uGeA(kWsEsi8YB!9Rv z-QXtX-6KVQYfOTeh)j3NW&ul{bSq5;+)=@_f8Hb2@YUoDtKNz((^xdCVAEqd2JFm! zm)aM5Zf)ZNBa)|M{trY*U;6j+^J@fPbh==`so+KaKNgda^oX|YeQ5wPOSPj=lmJ!G zBRHRmHv9;hIXb>nzfS--$MTdO@T&onK7-`M`nDDbmsM2gwe|h({Vn%uu5Dv%%=mlF z(N^ag;SF*^E^zx$k>h;kt@!?3oDAY3q`yBz zeQV~}_hwu4o6hv8!9hFU7A=2*-k;a7>nDHkH?0!5?G9)@NONeisK6c}w-W5rffh^<#i7}s0(C7t^ zz%EaUS{dxkF@otF0%l`|I}L8bWA9;+qR_n|eRg%Ub%WH@0FwhsgNk%aK zS};@gvR^`8?yCwXeGHW2Gj@Mok}Ld;ayXm2mclRn=sWG-)*E9cS{a2G^AdPFx+I8! zondDbnCDMd<~I0{CK^YL$#U*$ZC7TG1=j%3uIJS*Fx--{H5=|n0An)vuh9GQ*8Yte z{Hmv7&z{8oQ*PigLpcFKobFQJzm*A|*a+nY2jNXl@1dI4*r`IX72VX7xz+^e&-a%~ zkmOy$Rr>E(HbV5{CEfC1EP=^C&{ zLn@m7#G!D#b?6LZ!4jb zyw=)r$`BR8^5;DB28V+sB3_&ndv_;?L#S=Dfu`tqD)5l7I?0D8GA2;t3H}#n|3rTi zY~Rx|^8x0hyQfvrIWG^Nt~J@8+lndVji5deg)g^!`X>0%5*&#MaUrhIj>HM%Hsr7K zG7x7bwqU#6I@6fCB6?Nr{!~z)&@W2@ymA3P1NJ@Q3gNso0Odo1*NB%7KL2MU-^Ou* zj;N2U3Y1`xceh+AYl+W-SpIc%3uCD5q_bq%=PhX9{d7VnNUO! zGDe)lEe|C3TOfF-CRZyh3;G1W?6+VM~fwu%tG$E64yi!tPy#%2c29 zEIF1zF#yYojfX?@+gH+tC^FK02;~OG>>fQfaHy9s?Q@^b&&ezW)=F3I5%8hr0vZg+ zt93&@wPoAXNKB8d8r4cDpk+XXobL6Gy?Fdo15S#oGk*0^@g^x3Wbth6`pH@A!)rL( z`*dorgoK1PHX_9%D#8*?czN8aOpOH?*Uv6-nB8bJL!hMH2gc{BZvpU1OESM_a~6_gL8IJ;&5)-74YK6E z6R`__zTfzyp|PDD*$%&vEzU8)$fdubYsq}%caRwHD5QgC);y&7G}!?3k7PFKxyS9@ zvXO2`q)2je#0{5@dzRAHa@@EJc55hC3Zf70dZ>9cLd!c|JF<5P&-*p~C;0oT=aY@y zO4Ce_CYjlyp(Ul&tB8J<(hfGq!RCBHr_<2OBcd0F_G@;}e?7QDL_x(iu@!US4nR)M z+cQ~9063`-7{$e!)L8piADWDeo1o#zKRK%(*PqnAiaJw2P@hgZ-uf8Bp2O-2!@qju zxVYD*;Gzg7w-~x(2508Ot2H~BaME&B=qhfl*1w=pNI~vY#JBe0ZfNdY zJY1g^^w2;^$)U!i(cpvH@pR4Hr-_4=1_gdqVLK||2R~3J9(e6%-&hl{G16Tz--<>{ zK3*nw-<>3iCM(=_Y{gp8UfvmLE!0hgC@#(ncT)}L&p!3|$8joL=8WTzi>4HHTA0+< zuH9@C1z@68%gCyf)-QS+D$9G!*>5%$XKD20m@0-*UvkIy?>gni6e;SGifZ27*VBW& zixOhvQ0M5*sN4`gk8Z9INb8=y2ETaKRA%(bbaf?@8tdtY=)n)-;oB)!{Hc=Uy$y1) z>d-wf&@402o_a8cy&2X<5F0nR$WixnUXBQF+1qyTm&A;D|Xm7Ok;ndzBdC+f(6U!{wfD2$Rg zI_!4pF+?ZhL4A+JkXR8AhjUm#cv8x~`aF43X*ECFI^iote6BMP;i89xMWGaDNpbth zS4;*SBh){o7!f#iHp>N_c^OCZQ{RXWsF(A0q-=p)qgwgMhIt6IBPo5$L@|g;9c6zg`~b$`^~&e0%ww zFn_Y}RU@6$uBHDIP0B&6m*WD-7RyA7q=dtm-l8r3;w&!gmh{TbJ3;%Hn0(gar!H_=hJ3%OOqN8gp? z4EUk2IB9N23yL0uP$6+}L@oq;`67O+@p$U9l^J?1IzeJ}=yfFP-hJ@0X1h7Y3)vZr z_P6(EB-=5btS}PDsyMcMtffeH;wqeLdyNw} z0r=jN$Cxt9m!M+v@_fgM6dqJl98zX3Oo#iKyME7r9fC>V2ZIKb8j%Ixstg1hjBj>G zTgLvA*)1)3tPcc`Kp!a9n1DRmRLrD+Kq&RtvnBr)h6IfL#y^0u23Ve`|K289C25sF zfM7O@V^m6Yb=+tt&N6M7|3~|O%~zD&FXg^{9=$RWJI#|k_q%BMO<7)r(=?XA_2@R&*wQ1a|;|>8{Kx>ilO_9Yw2PtD?eul{_A^2PbMPd*HcXtxboRorElk4VX9Tf7Co#NOR4rSq`S@#5^_zgcJ1 zW_EzR| zIV%ckDFEP|ygwb%Y{PxT&M~H^hLPp%CcKBAC+VF+V3(WBJ5Fz5uM{&vsS{4CyyECO zMdE^ahFt3pkN?%~Nh>Q*>B&*Y^B^&AgKuCC*_60309e%s1Mus1^puUo`Ti+SBbvP?&kwb!sPXI{#F34{qk$D#6E-_h}_V($eG39q) z(j9jAi6vgQl2l(WUy((g4v7;Wl0f7Re*3?s{c>Xc{oD-|8e|%=Z-TnvWuKOLlVlmkd-ZaWfV{dH)jsj10kMAyyS3=yeJ!$h?Ji&%lOi?^EcLq&-&IE>)SIp zI;@C`&as7fbn#Ip(A2H$yu8nA!%U)?QZJ!YBB^H|VyO4=-Zu<}#e3Qx{QLPW>N!sZ z-F6#~PC$IhVRf@}OfwB6E5>G1odt}vyrGuH10Nl4kxeS!8T)LG-4_gmjG`0MU zK`S*C25E=K((SkBrGec@B1q~P`OdjyA^P6KRnX^YPXJxI)!n{}qjfT0MtjD9P53qn zY+NeLiWz5cebdJ{!m{5BnirTe+mO*ESykd{`MXmyw(kAH0NHY8{_{sf&BzrWKyr=+ zSYyr&4o6?AL(#m|gAsT}>^gcaYv#}DEPqx^tye2t4*d+Xt((5*wh>ffQ;;180dEhx zzoME=LI+785c$j9*Qm%l&Va`s}?U5;m=*u>NNN~HU0 zbdH<51UE$XVK(h)zDCeZFF*YX_%0}k8j>Iw_6B)vpRC;C)mY?K`RXV;3B=)fJh>tj z@jFdLH@Ch$+f&^)G&Ub2$Wgx= zx$&Ru-pCf7pMZ38>pOObGg-ac-!owWcM<4bhnmJ!a%vEQfZVzBbzI`C761$NE}fif z)2HiU6A?EV_voC?;m4)uHi4tA*sw)!Mh)9LHlPcHdL|S2>X-*FZ<#HdU7Sv zEv79G_!@&GGo}6fFQ}xq@KseRHR0z)rO`ppV_(zQVxvq&uWfXWUbo+bAHa~NIx`RA zsa}?iv4FpYP;e_tjQsDnCynTKTxmrN;QMEc_7oTib8R1^mp%ct`{l3Q+^vaZLD|rR zQFL+}MSDx8`#?sRu4M8J9*nM+Mv(8i6K{WD&sdOb=Gr?hAcA~QK%b!y42q?N>@=B% zUN(iIUfl`$zG8!2_awM?>w}#XKK^|%=VrANb#*sEwRA=N6pepg_j4|O<`=)Xx5vrP z^u{gk8^jQfL}YFHuib}YZ0<=WtT_ycM|VHgs1~){!Q%pwTtH!z4+GNy{&Ot25ipWj z0Z+-@%;WJ;env)jShr@u!>SnI=dIzB@0yW%tNxK zkd0h-OJYp;g!Y|+hA3j}J} z4Guxg+_UHVv6>AUlTIkUp57atz} z2_c7mc-TEYeoKs{Z`J$(eIZSI*)~2GI=m}8)&-(SwEz9qiFr9&3)3W(UMX8_r2YES z+g^0|(Sa;sL_GJVd@(C>KEJPX+gU{VquN?MnZo(umox!DUGXnOQp$$|avoN^n6^D)@Ntg?-zilZuBbG4afZ+a+srBL|xTLgbGI+GD_3b`i6~h`q-;Jw36&Q+h>htrjARCe ziC;%z7+nSK>&YF{;M7knDGa|>83yMQW0!HdUv9A9XyW)whXU+-P$!u3H2oR@WP!m6 zdr;WEaxPH-(K4hQ+wkX>LX0sx&np6$lV)g*VP$4iI$HOeN`E3xSe0-};JET^pi;0k zfTHb!KXAfUaeDGg%pvwqvu& ziMFO1ZVh+#a*&Bo;M=!v19$XSUO~Xit?!4EJ+uatvoy;+$5NEr+8FbcfcsKQzc~m0 zL9UsXf>xWPTAxPIzm!3UhSP84x#mXTE1O!Zf1-QVi6!*(~;OZMtZ(J2>y z?%F>HJfEEp(eUY852@Olnm2OBevY*SMmMt| zHU0mhTl-!)jMn^K@)!RB%%P~+_jEqNeGcU0M9D=x6)Ut5b5Utt6+0W8*b!7i0f!R< z-G3$s2r}b8Ye?q&O87*?eAVUVQx*g;Vcf;bVtsOdUGq%+KQ(OJ(DCno>H{JJoG-y+ zLs&6(=ZJdp6&d3JJx=K2Lay<=%kDukpK8c-Gpr6W;^*3sXW6K)_)~#wBaK#m1%xa^+Xf7h2j1O54ghNJ@W6{dS zb3uj-!Zo3}>zLe>i^R12XC4A;mk+;ucZGqZw0vi3Sr7sT$lHg>JZK<;{@OB51sd2B z`%%YcvD}a3U)rg@dyps)eN^$M;P+%h!~EU5kh)5We;>kFS?u7UB)Ddq4Ys6Y?Hi-j z{@^t!C2j)lWM4CG(WeUm<$qadEdZ(mWJfo@3$m|+|YSz72m0*=GL0|xMIGA`uvE;i2uNRtfv9vpWXdQ`7VkfXVU=IVW8 zy0w}IpSq-<{p^X#lu9v4sURBlNGRyQRx2V-%#1U1HKx)&ag>gIs^p`l0-c>Rzh*RS z-@I!Css4)atXrIDwP<R-2HgP}2PW!!mC$kHvJ`;Ygrgj(%P&u(^};+>`LU zp_#|_(@fFso|~%?IzO0&G*c7|Sjh8~QOW2$W9qhF#vS)B@OE1B{8Bx+0lji^l- zYR~gckH3IY(8(o0ls*Y7|yF@n5N zdvbPmY|^n(U*g{6&p-X=@g<6@cv5cGA}5gl`o2xx)VqJPpnrdV;E}3!Mt|4@Rk^?C z%9Ot6(3y-bo}eyM4@mi*qi-)&$wspR)4*dS=(0mr7)CH-Iu28Ne#w`Sbl=aYxV(E% zmK=IEta&=Df-IL*vL_H4d*9``#)dzVD|w$(!^01|T>CJ1g%Ux#XU0Z|KuvC)e}1Ik zx+O^tTGtYJ%{GSR{mYqcwv%n7$$|PkNykxxZsUC{Pk=sABu>pTPD5qI^rmBX(LG&o zU^JW=b?G!qnEs!{Dx;kWdl1(4Z#@-56;4EJE)RkNIy>QZ($>Re>gPZI;!KgxOCXD| z&8qo z1f>xeG)=&D^KLVZ(!)=1k|YO3#Mwyu4=vko>yB;A*J1 z(o^uj90fiM=D@kbC2OP$nC5)Md=Bt3{PW0mmA0ztr=);?7XvM>ixCItVt8Fxe;NJM z4=M70YKgppugRYM-)+~}LeA_`km8KQW6k68MRW>;JGUOMFl39T%heK96e6N-P%Dr>>3zekoIY`WV1f+{AlQ6SiL~l}-viW;Uq#z&5e?N%hfmHcNMAFW`x=X%E*tLDy z^6ubz`}=>a`{$Q&1BqqjwwB&mQ$Iq8^#pG)bV~S^ zt!259=KBJI>fOq8YgA|?DDyTVsa6*hfj@8;Ujxkx zM^+oB>fTQf#&|2a{}ejWXIBEm!1&~VV$lChb%T?Fw_PcG_5vS2HuCz&og8kyHE&RU zWNxTy$6xRz2xQIsv#lckGVw+)@P`h{GHU?%t~t*6dl3-PIrgT$TB^8gDsIA3sTgKS zg}HACcft?WZKb;k*7>i&Caa|FOtky4dEz|H4*kw%6+y~nV^eC`W@;0!ypi$8Xv-06 zLQJTY}S4QToia``2s$SA5n0y&hB$4in`DU+82lm~dMg-AFK1XZW&$j~9Vk>Y*nfq1&`!a>vfN z_cv$&e=lHMYD*Q*rM67k%wOu+kv~8?CSi%SFQYz`&0nydU+8Ho|CpIE6eO>F{F|xo zP~dQ*%Kzt9UH+RE)y_`U)xSGC<840Bne_Ne|5SLr45XYnkV`Eh*a8`VDycO*XNiHn z|3Egt#8g#zhlKlJdNuehuQ`A9`HpPwHbm}#FqfFbx3ECX3fHN%i0j*%e}&Sn!~n?@ z;e|I5{)ARP5_wa%?#{PwoDb0^QgT_`McN$(oG<8=&iNnA%(K=yI{~nh*|Qz+oL6yc zF`NB4iX0@LaO`837UJi=e|Ig>4R>HFHY?!8h-3x2r`O&R#outP(Nevv@N3Roo7A&7 zDgT>Cj&Q!4K?a-k#e_*61Odd@yzC;yv_xu=B1b&r31Gqw0FW!EPX0<12?#hIz&AkVpu?L_ zo@`(D9Cqoozj|sN7HW>QB*xWo<}KRXD_-eqzran}I&Gkb@*X1hN-#j2`*l=39vl?yc6v8FmU3y~JoQ~o#ukk`nl|1TfMdd5!Q z;BqO(S^sjjLU<4XRkQ@)foOmkiwe|O=^v%Xt;f___#~`=K@8w5|Gyl6=VuYV08(ZM zKz_5GynktWLNfEfNBZexK(WesXEjOaLw_|D2+2sz;^4X2;G84D?CvlO@{x3r=<_x4 z(WqDbmIA~6Xvfn5Hq6SfbYECIY2vNY48LTbheTDCqhpT=^5^y8p~HAsB_dKbilwlr z_M4#`JMNzgEj61yWChj^o;FI<$Um@Td@Bod=up=T_E98vHR6y7_io+^OHAMBdIv?6 z%>4QLL0V0(u&kiq@zk7OW!d_NrddV?r(XRz}mS$5uR=qnNhObU;=|KOq&;0D4 z&~3RXBPlna;k)ZeGF_*@mGtSIpolN)W6IFUKjFifK`=pC5})e}H;I9tB-jPwg#sUdHn6`u1wd*rAT%0@beT^&3(!NP1i3mJ@3L$j^&L9Iy6u3?#15LsNot zr7`ed$N$95y#5W<((FuMMEdtI*MfF=X3tZS+vXb`iBaJXc_5CWFpxF;0p*ac{6Ur3 z4z6Rs!x(z=AYDjQ*l@GO<3Dn1XJY`U%{xjnE$3ORvSnIRQx$Qa>{#g+&Py7i40B z7w2Byt%MJ@A7v=Qq9u`o3sIz<{rw58(=C>I56n~}4{Rpf9&Q8er+c<-FkeMI4E+e+%Y*1A=2N39<whcbf>p*3lE91o`ji;?cd2UHMoyj~{Zlwq+!NMGneC+FSR30s zEOIuwN+jnec4z8$YSO-H^a`x6W@K@!wBezcpr8_qX9VQC_;PEjwezeOCYP~rMP1`u zN;YfVs{LkJ{+$wGX+vUi^S0qTywoMf1Fs>GsvYa6*(Y+pdxwjPF2jYIWZlbnA72ie zzQp6-6kHr_ybO60{#uBX)-?pPQs#Ndt4{SVSwe~do^(3=Lv5<`0X?qb(jTr)nv#Yt zC5O8$b%ovfw?Qj(v0j;+qVDd|)Wl2hL|pc{HzzPxx31!z*>kM`n0qh1L!O0Z!>KV@ z^P61ofB<&^&Vic0!7?#-jNZO$HWluPvj}pU&Snkteym=M;^&G0=)>wk-8k-%dHvzc zPBoO^S*o(-{Wah~T-jLgyW-==zt6=MQ%zlv>}_6mq8RlEx4c+*)w%c{Yvm_c;6>Rw zIB(GiA!>=w++Wh|6w<0P^twJ19d!MM;=L8ygW88YR z{PUaYYWr**2eMXjM7;@VuFAwecN=AY}aX~7-1)q%s>F7}2 zf$rO_>XX!6ri9vmr{VLkXJQJ_Xz&l>mY$=txJ~|w4q@WSxan7$KUt@n`{hmlH&^pf z*6%)kZmOMbZk)ukd-F|`K6|DepU)yCJyBW;#~&hx)_1IZgNz~n4^NjS-8W&}u2k3~Q9EPG zGO#ofh=X()pr2~b?+^Sm`w`a(Imokg_Vv9@i~aqW8P8L<_l&J2l*C;AyiuN4?H5fD zI|7zeQ&N}S!^Xn6OHU^zjna+qdW$RH9MV60AjzT`$9gR96%EAxBIi zh|v77fHTP&Tsy4vWjvi51=ny^(k)ZcW0cO&p5Dff)3yE5;Y}FA8iP1bl!L}-@fT*K zrWzUXzNTUUFa)?4r3kVSO&{d~m*b3s<*er#8mY?fb1y;Yuf*g^qHCYA7>D{&^edtN zJI8TvN|lZDCU;#7btm+=fGq()V$4g$6aFI$p#c8=q?mbPk7QP~TDqhUzWRCd)mdMh z-X5>B+8R}U+FYD&J?C4mmHB5pRV&|zN%1P(oGU3eZjM4JNu;eRlI|y<`*KB-FODN` zmX^P|*=!CgkUMK{wPAb-W62`wE`?6~ZBnl@A18|Wf0)6>XJ@Zk!#+L1#PizUCr6-s zBB!`>Jgb=F&YCb(cQ$UBxH&u69vPWe>~>BnobLJiLxUf_b;9aQ_*H@tf9M>PB~d0!np z?D#1%;@Il}uVUi`r{ep42)Vg+|H9(8OTRN++8KeQ6R#K0UvmT;J<~qhvInR598J5_ z5eWF}X?Eiik5vt&FTSmbKh8Vj5M<5zaL;@=JWEy53F+tmvsxTzBJNy!SzMp>&y5Hz zJ)b93@X>sIOTy*toA)PUVO5XTolL71f)qUC;*sgKKQ;5oJ;Swq1?volpi5?^dYno( z6{HddQ+Lzc3mVF18C@F(e-ioFzEf9r@o+J#er$Uu0zYl_aqHXNMKm|8>K=srKDE(< zSY4s-Z1IN=V92j`s&RwX#GJVV-d_@io&(AFxYi>D8el^KBN-Ho{#9562gb1nReyzw zpwWbXAKe+>jkpTSGAdo=eQ_txSE;Kn7U3ntZI)Y{_|j}-RQ{IT=1;c`1!ZccIQn=@kWvDt+v*|??bZoZV^SFYD`ZoQ z&BLi{^Sd?22YQPmAKLe4_(@AXdmh3HvSns1*o*z2mo;-0R8P-Sy9$k(~^YvFO1BO z(fpp#S|8*nkx+b+lXi6X6`%N9g&!3rkD5psa3C)tau?~vXAaH#Ht1$`fLrM(9L+@r zru<5Y7#!S~;DaKQTs%{7lwi&tJVye|`*%t;)Q@oYwfvEX5PzD^sb|oMgxHfXYxLUF zfn}w-rUqx>856sztsn%Nv>Z#AK+EX*4n1c7z+iBVix8p+MXSJ|AT)cRy%bf-O8mD^ z+&LJ{x1hn}l7G6Vs#NsL2s(7T2-nBc4?*Z2aaNFUP3f$!;p|EH23bFIcYDQVw)M5R z4m{rlTa>)eO6eYooxlUDLJYB)28KpPgX!7@;rkKq?(MQAu!0DgXcp%ps=eEQkEfT& zA^^PCc)=?NG^o)p(OTzbZ9&kDmdV-bg7Tu>sfShYt;f8@*RqOH*C;8!iP9G>INv*V z;sWQzw<05-hY-O$4|ZhDb98Va3V$~o=R=!0zCZj@es!js#rui=cXsyO)O5}9L6r%W zWgY?P_r)H+J10^H4z3}zrRDwp8~AT}mDc?EV-RNp_hi2ROAdz3`VbagM7@GIl$TXd*x*1`D;5} zu7@cE@SpClJUBE}$eWnL2V&~iSAg|$IL>1#l|UMS4uH6r9rZ~%&mFrI6&Du=Fk!lv z@z(pCe+DRZP?Qb4xm!K$U8#NO?0)q^TU-0fZ&RZBk<}I|Jliw8RX-R8nE=6QWu75` zs0h?QY}zQ1RQ+^gsP){74eclnl+P?kAjKAsz*1zK>Voz+PWLDXSosAZUb?B!6r8gB zyK6AGS<3LJ!R2CyIfOCYf8A=mmbepZB+oL?A_K^tAXe~tPv2Uo1{n@r(l~w_O8nyM_I3`nXJ|C+ zGW5lpJc*aDchtCfKm(Yyj9O~Vs`Fc$OByVekN^u~gT&6VSvBKCo=VsFsG6 z-&^4W6(UMvXg%qM>DbTy{QPS7RQ3!!0MkG$eR3kG>OZ}IMF4k7Ov27Ly_%-P{9NU4 zJwJKNV?TLX{r5QX^ZBW$ZOpDrET_#LyTcN_`l9c13VnOu<#zPE*Vc4w z4pW50HXKgwI#@P_FqaUXm~g52mS>$N2W-zHNRlv8=p;re^xv+MEt3d$P7)AjB`8QJ z@?`T=-L3p3WMz?a=elHQNS;Ejs9x!t91d}D<69Y=B#fgySuml@+?7v$`t?8n{q*j$ zs6>6~50F(d&0G8TMmp>})iBM^LTU^1xyY)a2C!|y+rS)w8bw!82IUAiK3GAe3DLLZ z8mdrZ2FrOhF|!G?3LuLMQw)UK9X5A&AKqU}+rIb6EmPt(@xQxC0KIlYX$CP0AOy%Y z1xhU$bR@oxKp~CLZ=kL^3fX6dpDY;tx#RqLzao!4Rq_rDz#w28T1iE|;-Q*OEsi_m z7XJaOO7ENpl0N6Q<^Gy#q_l9b>g?%rL`uB%Ov|e!3$~3NiNR-qzW%@8l8DgYpa6j0 zV4_hD5+GLF%uvo3YY9WbFqxGZ|6oGQK$`?TAa=DqUA16J08lw(UAW{IOZ&Shf`!4- zE|Q)qccxjKj44wF$UvNLBV0%@!2`4cQuA}ZmM$BrkpODWWlGytKE%r@=oCP;zQ4bR z6-Z%zq=w*}O1n!8MErVEq**!nmj>0f`uB`SMend!qEse+D+jX84rQJkaH`XAup>4& z1>r4B-VwIL?a2>unGSDX)Sf70w0ZjlX~b)%5i z;gC{_1mBlZ1>P<}M*x!XSWJNaRaMxB+;O$uP9O7Y;ZBa4ekD)-#6iAk*&ybYg~cYV=Z??c?($}gEAP7AfT%tUNUg}*7{$m zQXGywGIp(ii@z|FO42(2Jw(Qgi3)*S9qmqcJ;oeVfsd`1ox}ketFxE9#RpY0=Yzpa z{PK<3&!rU7-R8>7%%dotf4H_6ra0Y|B_wnT7>B8xWL7VKAA3JrM`163H>%}dd0aKj zyhz{55HGgssHm6m(BH^c+||oPYGnINqV4>cNb2OD!fyv=R)UMZC zk<^Ag{r-PHTcnw|aNA2caTAp*#~u{e^=?wUk9kwx>9Uj4Lt}IlrgeJnhw*Gj@)&g# ztMpx2fcG8H!RYkiC>eBY<;UCd5XG6P!bP;K)F)vy^TO(zEEOaHY??r|z17@*(Y$;* z1^~ME3IxsR1&H=4If9`ds8YAc?7Y+Ru6IBVXqG)tMYw+j(VfX#r)o=3R59Y zwu17GwI1mJef(d?%7O$aS#;&c*S_R3=s*2gQe1im`*KXroUj4^s%7ekAX$a9`X%iD zMi}~>V-%h4T7@bq%}eP^*&vthhEXYVW6ejZ3s-+D!;8yqd#ThFK4w%LfRgi)EO5^ifZczd6PZzzHPR1&BhwOZ&3hUs3gG0mygI-HPX&V1#7|<_ z@D!vrGVs4S+4&`AL_%lIU)I{zX4bPpIM#lx8}5SpgWJ^Uoie0`(KvpHMwR!!S5zPb z0;FVV6K=?%?j(?AHfG>&xQ)a?!oIpssHiuIk^^2|_pd;ZP_ZfC)~qwIHkYw5gp+SN zO>?H~Tj4mG>3sON$uWPW2od7xF6Zr7&sp=y*^A2a1N3)8@-9^ zc9oEOV4rCi!E#Fmg>E7N%LLFsMoI5~7NO&P5yL9Fe`#ZDAesjUq)+}D#05*xuSPAZ z_j8xS6#EAs`t%C0B~XyP6}^626#1{kL6!|Hx~S;Sy)Li(jqNwoO~mI8QqbTDX@M(k zW|^+hDM8`It8hXX>^{yh#QNx5n(?e*ZU46&V%x;j5Vo(y?v%kzmOPYS||FJ9S8$J|mJg$Oqc&&h=P@BFn@ zSFajEZA)h0q4$JIk@TWs6u?G(3F#PV55u9xo#82xQ1k>0NvDCRcoY?}fI?-lc;s1p z0|egRQ`8l6{o!bo&g6VA56IDsZ!o6)rO;QYnWw4Kfb~4rvi2wm1Ss1YkwedMYp8+j zBTiC40+D+MxFBFZZoga16NzbOrMgTyel_Z^LaLE{#rO(WY^hwgw76aVGv9>)6|4yz zPEL~*{keB7$#APMp-$<@sZ7c6Q$0 zF)5*WfuS>$p3|gZr9*MqJ~|F-R{N;843I3Gj&#|% z|NOJXBPJjiod!dy!hF<09K(*_#gF#y`Vg3`n%(ovsLIqwKD+;(EWP#56qxJAs5$fvEtJrg&}ESSV4 zV7Hf@2T9>|ntmJ9ey@tR6!?DpL0bL(P>H_@9z{S}OyhDy%JgI4Zd`aS8v!yshqvya zo|GcF?%_=&HwZx6!vYFIOJs_cv<#B#zeF(uJG^Kmf6>?8?9kg#K@e@y`1aSASNXqc z{^lsh%YE?q&O>Hysqrix8stY#8N2*lUWpK7uQ^qL zXU;f{*|P3$VR9Pczg{K7?uP)t-%d-%dzVwwsmxHf&SdV5#7UWDTGu|To@z7sC!OVu zCWq`WwWD=#mOI1vlZ50uA{AHC4`xrzzq$mIT=hta(ni8}ghjIS2E2%Xwol`0tlC*H z<@49jYJv^l@bI~8HYCxKwPC*&qu4P?-7F4s3>U+8P>(O%{ff+SQZ<#bj6mdrUTmTgtP@Aik8Q-1^i{rcO7fj@9ZZW@$CIeXXzf0|IT^L8?;QvmUAW+V?*m(8hR!P?`}NR3 z^VIOz)$8E5<1K9XZ0lfIU94viYtnESJaqZ+kH0&Wtxqc!|KVUK4T;^J;89Sz+SSlh z+uJL)f7%5Uz|2%m1c^U&6{yoY$z;F3#@BwWtx(BlQ3$#fRZ|eA%y=!XTzBQ+Ud?!^bjVyA z$mmZ6uD#QalJ^^GX%uGC(pThmq|XliYeAnu?BTWAz_?9KPWVz$7r@rXsA{3z=YJ9_>VYD z8}H2(f9x#=rv@iFOf{!|V$1`-DaJXO>w|+h$xvpLKSJ+x(2^ZPt_K93LKvRB_-Kt9 zghIixyP6SDl-2MR0jlY)h_T5Zo~arGUXv&9vfj}^?e$*9%n{89ptlZsypQ$D}II(&(J0ofLO)Z-YgE@<2yNDo$)P zLoA2D=yy(6DFq^L*8UZ-LCB>g_=<|wNRBVp3vD@unMdzrIHmNFrdiJVzW)XIVDHbS z9)AU;SL}Gbq5f;f${knJ+bp+umgD^=t_+%RF1haZO=4P8M+d2S-@2H^x`WMbTps5~ zQ~x<%TniST?Q#$SVaJRRV>S18k!w2tDOXW~=tRAYucOh8i~1E)p_@{p`(tXifu+?y ze>n|ZA(?~#^TI5OLx$@f0NfgzxAF3_^IO*;;-JVUdp&tsCMvYJ2q4sBr`CaI?(dSF|5K$G z&!3;q=C;>JJv4g;{%z8-Naa&tL=|_i>?Kg#fnmuvK*iwec{SD+PLBz-1~7nF{gZb6 z`tYHP96Dr$bQe77np2&Enx=JQamIq74AQT=nD|we55eN5|%%`p~gxl zd))DCgoVWlw+j`?ND~(AsL^R;4!Rst1LiMb`y05Zjj43=ToXbHjp2U} z?A`|G(LX zdcpiTgK96+W{xk;G^%Ij75|8Q>O;Z+|FyZ`)eYu=KskRYO}t`e&K*|OZStm=UTpV6 zhF^0apU`djw#$siL9h**4P_Fwyr$GVIHcEOmTtb=Q7hT~wIWgD2ggZ2xEMX#K>z zUkac(JNWhwFKhK6kqkVVIoHEQ=YMQmclp<89KJEjNM|EL+ZHW)e3R}ytzCa5(Mo0R zoVEi!eEAsi>Ju}poIrQ{zv7kY=|`J9T8P3j#~0zA!X>(7fGdG_^DAnOCXCnfH$4)* zaROkdM}>#8)g(9s)4ZsG>QG){ubSZYj!qs0D11?PuJbiN!f*Obq-{VT=OdmEEL$oQ zk%65gfZyac_qfa725SjbPXayeb^1qBxSWEezR$|cw;)!Dxd+==Rmd!<9FErMlPTG0 zx)k`kaUm=R1}YRkRQL!6`$W)Rc`Wb*~N%esRMep+M?^nd}=@bJ|SwvKC; zSIfa}LJxx?vBIemxzl(FfRE${+cRAFB*l_(b1f#H^fqitcivLr(tr^)7HIDl<4cj! z!6psdP6{@EuiMk1kUck&!xC2raCUM0R!yyLrL#%YjOqQ=8GPlsY%xSbYQ$k_ELV1U z^OukgxO1+VT2;dc${+f1a$nRDzxlMqxnO3LS2P~I0(B)aNXESVTf&R79x)v;<>li5 zNv}t1_HbT`RZc&!!U)j2nv+3+8OK3qQClzKsM`}1eaJ@`Yq4|DC_xf;E!%ouk2s~&meHZIvI1iIAoV$u~{y@OhQ3wz89t?xkg`aJ?iExu=F#{ufb$%kY@d?Tstnd!E#L$bm z(0M`ans=GL0lq@23>+!MMORE=qIfs#c%JO0tIvgeze19_We!}SN*4hu-~V!?+)OvU z7%M-JkR-b1OU0Ksx=7tT!sSly*x8*Az0t0^nF=AJ{oUhpe7rS6FApeA{eKT!7sLeD zo-Dr40sAj%Yu5iG}Fm^Tn;4cy96hOL^Ik z2!q8<0{BRxEsTekn;Q>~au3v}u1m8y<9RbS7On~K<925!7)d-hlMl>|uli8!!1>c4 zlsyldM8*vGPDUbhocEX7$Mb5IT>15~k?TPZXCWvE*Oz0_%}P{*j1~2$``ws$hAstH z2pk>p5eN{>cSGM#WOR?t3RP0==WAsv8odrPgj!+9WYGDWf3}H2ZeVMm{DyH2$!Nnri`Iy?#@gRY7GHqPP|#lFLm6iPxB!C6=_=D5O94hok$J{>KJqi* zZ_3Fp2tni%oX;Z0shPAFAnDH3iI(`anVNk(?!uBN0^S<5ds3$fq3sY;rpyu^92pN; znO)|bdVBcGWq{+9JD$8^`MhlLR@5HEX6$@+87n6x$W3H^8Yc7-euFMBxXdqmMPFDk z4WeOR@Ls7?0(;MlHUlyN>w;Cij~Q(ynb`g7f#ipYixQ1!hR4tHx`MNDf#m+iZ&-NS zW+Dm!&#bGCmpE;tO>NB0qWd7M`c3tn&ot}+0e_oXtI9nU-(Xw$VktR*i_G=VqJeL0wN$W{2|?)GNn^mkQ$AmfFLO;2m;bADnq)Yq`On3o9%txUDy7A=i1qM zzH#55TSM-0VCWS+J$Bto0bqNieDK+g$QmPFqN%jN+H;s$MOo+u>^b=r4Sd{5{gJno zgZQn$jo`PusarX;papYq2Lb8;uumEP*Zv20$(akHNYIL~{;ayChMU>oCP(rM?{KrK zD$`YzhYSp3>zGM`@ojo0^pw!2OCroMFI534P=<8y&UjhE!0@+^fN0#@RD+*j@kz6W zFx~wAP@OVRGLkwCJY9RHN3g+!Afz!#R1$A!DDHtEzK*KIDo}?9N{*GE7zoGw=NO{$ zIZX2$hPrLk^*&wHHk;9mzcK8dd+usXm#DaT?;Q$v7YlwzJ1ms0!|z;31<-mN;6wYu z#!4?bE09{#j&O?z-0YW{%+R2RVD!kfe*2o+sI~SHBpe8Yd;vaE?{tCzO{eSFdhIn3 z@Wa@m43gPGu|Im_h!^`~jQNWI4Y_Es!NBJ#s|SO(`WE`nWeGb`B|DBUtY@*JOuoM$ z?;`ZnO7n1nrUi1WQ;ZvN@MS3RlXJdRa=2wh4txEqCG78uev-=QCuN*e!m=HK&n_#B zneew^8X~stL78M^DK>)cf2EHXqxotg&rsHuM}`erTEf-6bDrPZdL-hrLcLG4$cqcsp9fr#z0WA zGFq5w&2Y8M_jv`e&$->$*_L^)SX}AdNA~1aK#O)W9ljLXcW&l=wK#sL6KF$}1ltoT zJJXoEbm~EhkYmW^2XZQjvgs|H2(M%NGhAvLp|HDX9mkhDPl&`b@6xb3whKqQQxb zpSS3EEPhrHM3JS$mUN85gTraSyrSvcDt_W+zl$#1_VtpKAOCxbw=74gBCi9RMVwqP z-#T@j+S2)*e^FiB85+y1U_fxMD}O(4Ajv6P)w&*?s=D7PY4Og7G(}Q)%0tA-@?cU-~j#J!Xcd z{H=3Ox;b(Ey8T&VG>d7)czv=GKmw`QcD$7+&~F%%`7*bEk@$QAuY|sG@3$47oTZ$x|fc=C73Oi z)k)ubJ}v}cLUz>Z2WL=RbMQ5^?{zCwslK+_*QZ}<`Il7)?SFJ!A>o9SmxTV)ZLb;m ztR-O(RERe*ab*$qZKi<)Lv(!up=LZuM+kTuZc+N|c>|PV(@)c!`K23hS^|JG3k@qs zB7OU-fz_9g5R%yIOxR_7G%fwY#URI~NjRs$+s29g{IR=!Vr*Cim=otW5}&ifgrues z_fh6g2wjr$^?@Y$a&+s@Hh76VZ2HQL1Kd9B=k5Ss|7Km;>M$bTlcF!4t$R9!fT>9b z%?{Owg7HR9`Y1$sQyN6q2hA+&?5B`}L%qpDx?fLDdST;xI?IQE)@NlrMkH?TCCZg2e z_cXwITVsl372kQk>Bf_vcg(bn^b;dleob~(G%Zsy0IOTs74isb0mK8U=)P%LwSLMm zhp^AuBrqwQBm^0P$r57!w60auCXYReHwjY8kB!3R0n7xE{36^v(1&~#Kz-`(i%XQM zg?VE}0~YH)0|Uj_UY=KptC zrzn^rQnC$oecWL`W+)%3I$VslzfL?9G&*ZvYScXqd1J?I$A5`|tYCgjGCfdKx((PZ z!2^q5Ym~8*xift{1&Rs|4mu~cBlHcq5qTyk+v&;)Nif0jp_TBz@9cIr8=gJ<=z6i3&h+rb1nSm>5OZL@v(ck81}bu&VnAHiX5y!Y3}4I1kPw{) zvjKF5Y1vFcN&(D)}n)Mln?02p&fR zZni`OS}wHIjoE6>SW5+6ie(-*UDQcn#X)T(5Vc6syO!;SS$AYTBxzT(q!Uk5$Fn?5{oPmp zG!$g8IPS2G{TiLx8_euZ518PSm|;kq!BaM^>sGOBy~sqHxUcst_4OD3y?r~)#Oz8= z<9L+Uy?0?$%v@jJw?sqHAkY$9R?;@hE{u3Pg1>^ijd*m?HFvT!Sf%3Sq7gGeF z^TqIv_|qG^9r==ymw{u8Mh9rao>qCCC3caEEp}UJnMXw88tx#EHtqxL-CGDM`NO(e zxuCGDl<@$Xq6ZQI0(!7U2S`%?EnD9$RN?ymiPcxTwzw?33v%F(Yj#p`ptgQ)+rRX9 z5(lvM|DF$F%=czc7+C1=N;Tn3v51I|*lN0$jm_$Obz{MVI-qg?R$FUVurqHrw_cLW zNil!9L{b7r-=Ht~sHPrZs_NNe`@~CFRdz*KR#!b`0s})BJGn?SpBY6{;Th0jPw>w&z0g!xW z#$2<#;B%=+KDqCmMD#oxNJ=Vn?7W?9Z0DJnocj|hTR3k^nN^KzX81}9F$0Th~P1WfM_wri!?{1ldcm{bpUf{=SV8+M)5LRrs>yKcxz%=7P z9(Guv40+z!xMBg;1_b7Mj#Geg6BbqKRcp3cay(rL2lx5I(-eB-Z#Y*tkTHO((R}2w zMfAlNSXyErK-^D3+!0i1uv_uhS;>ZCma;+Ek(C9n30sA54#u7#-e`g=#E4LxXW^HMf=x`E3+Co+Dkn56n-8izsLOdfVF+p8^q1$ODyd8Oofu`Z;IX z6B67y2c}rqACB%#N7fqetfdl0}a=&r{dt$d66cqopS%Qlq2g%=Q@$ zB>F{0Az!DtCo|UCgT1y=yD(J`KJ*(q{};VoV=rZ?j1+pWkPt8N2+McgSwei$;>62V9<3iisitt{H_= ztAr8zN*x|L5ORQIW*G}LL9?g*b@4Z$O(zm$lVN_-bi3v5e*SOd`80V<6|k287#e;M zN5{_!Lm@l0=e3`a!NJw*S2BbcJw7J@CwbGM!+-#h-Q6loV5 zz^SB1;L^?T&9=IjH1e}>1c9JF{8^oQ)& z8h}je?ewmrs$;f)1XgUfQ?XTV^E7n^l_`|St`e$`jaQRM8;v(yxnF}Wi$%eUK%Iv7 z>WIwf7c32Nh*0Mf`~T#GA76`8z6Jb}yM)T6->#MVUuya46~rQb%n5PB&-X{P@;nR>UA)TFSLSRWrU6-(j_jE{&^2%hX%`NDu&(eKe~!O;Bs`s& zH^KH8<{yM50L;Cl$2RO5#uP*ZP!O_DnTRYBsrkNW(hD=Ej_-MiB z8!Sqjamk0bBrugw5bkpH9;$1xVq;pG8;-fU@Q|?Yo9UzaF)^@lQuV$`WZu)a*UERj zRSd4ix#LB{fX2!o&B&Bs*eqQ`_UB*OU8q>?F=O0$% zGHrjThlgi_9T=E|_%REu3>zdDmh2?pR_)+f|J=QF8Ltz)>Bf!{8B(zA=SCTn9H8>+ zJw}F6CTy-5ZZ`PAX(;+2qb#?X4$c_z)$Sd8m%!$~l96Vd* zdx7!W)*r_Ah7~}$?N@C=Fb;iDYq5MNb3f-2q6e7Lr`(IUKT;&_avVjQbraLY&Mo64 ze@aI>eW|9prZHv8#l`&40Wd(P&(1O8t9CMDu3A}Oc02>(!3Y%$9}_wFmPwv~3B3xq z7wg6ev9+nt4$fP{WL>b<2O#ZX<~bqgoJ!E=FY4+vntuLWrfMyDF}uyBHWW@WYOUGG z0Vi4bgs1Y;fzM0nO0Ek(%I9+1BCgULf{QfJgJ&&_IJYz*1lj|PlVB!f)d=3Oc&MIz z=E>(WkokuV4oy!-3o%oMF~@3t_$e~qzKHH@pajcSS~^5n6)Yl^&?bWP`#1@`@V&i1 z203vS@OI57DCN*J!}MZIhgC|2(eLSgKJJKqy4`X)r6@8~S2c0>*2t!CCH4F@D71Nr zh@Io^HZLp=OZ=h8y+irsANns3isYg2i5ye2KKlo>Ap1*lZypS!g_Vqr$7-$H4NmTH${8YDK5vioJ{l~w*ast~}fOF5#kc#ysf16_*TD2X; zq5D$PYHpESVo^pGXSbgmQ%6-En6@E>UVXjT;mt*hd4h5;y`KZtXFMnf?@C=BLIR3N z%}gXw>30kK;ZBV#z1uQ`p_XD}33Hc7)PVw!J5F9 zBMsdDu<8u2e1mU53@W@EJt|dvANXE(JRnfB2x2lNK>{p+syj1q@`pwmt^TuN7m;II zd7Q34Uc9({w&5MFA$?}pBpBDDk>^5 zgn(xM`Z<|{E{CL^o4!i8%4d}a4i1iPQ+142(wa)gSB}L+fuoz9i0q_QP*kn|t=8L8 z`e8wnjePWp!--gup1{0ZFI#N^zYixO#d&C_uXh!3sap~AdT;r&tn#s}+|t#uJ;sd! zXDQIXH;?=ZM_<>{pL&^5fj^)JxgGAl#LnXlgiWU`Ei6c{ibt~3B*lC|Es zbEkWHfvUzQl|VM75I%jd<~8|O17MYlphS#z8{0?UdlkwL*2;$dqFvvysn(vd0}LYS zmDIB_#+^Ng^Q;9wJW7Xl6c5`C&@vT4`~)8!eE$C2MSXCyROQFhet=bcb4fEqod0rr zrg6zN_|c}DMCyBvTDJMzF3jt^NVB!;;E#>63fDDKO2eOopKsOrxr*G4q<@ z|D_S=)c<-e69!UFJIsT({Q)IRi7`8KdnMo2tDA!e8w$JI^`IzDWPW|j_KTi>IugRh z8)n0?&U5tO9931$2C&)$fZl+u&YPGT2K7v&XOFdiXz?>$NdaEZ-lv`l7N-m4;0(~6t!rZ!WEr1l4n#Cp(oi9`vzTo0eje*; zQckjMp55t^eHvX?;UsEX)8=9?>aQ#d;yWAyPkS5l4!B9lm460K&FJpcF4SMf%AQb= zgL7{du+IGWxgXa1LeoK;n^I2A5N$>@Z&3EIVuG5Tk?}D2o!0i!$vw?E*~p-u*oa)s zvN{C6?(ue4;r7_0{DW%A+lF!)u!lFiRC0;>lNAqm5@c@>@|_05zqHAK5*KFGpgcOG z;^Ep2`k})&lkLyp7`+muuiT0xjtf1FrHv`?h8Wnh-*ruKt$MF%z6 zZ)lnjM2Uj_rv1ym2ji{vvP5#2U{2UYsSb$yBOUQ0oLH55+r0JtCG#qn)!_OJcc-b4t|LzT??pg!*x~UrOHqDNhzy=WlzWpfi@+ijV21=-Z-Qiiia*jler+b9870ug zX?9l=Y8Y-hoO-!9TRJ<3>A}{GF-M-nO7}@&fp~L8h2Y)O2JR^Nk2@oBB(9$0?5f=` zd9wvs927IM8zw+jkefAA6+QPNy#aNF9Pdf)uuYi^&`GdxzZLr)r&cLc3Lh_lE!$kQ zf~8CQwEG*)%VO|T@%$?dE-uHh8mn?cF~SD>S!WAWbk~Hu``=(Z zY)hS7e)=`W@QegpobFAnCKtrcYWDRj-1_M&h4tp(vDQ>^?P1_ge6xFc{HKUEgG!K{ zDR@(Ke71n~KUJO|xcUu0V#@QH7EcylZn8ZflsUhK3%ErEP`57|6!m+HhX!>xT!=lw z_NiT%ckHHOtvyCqg92Mv_F37>2uW_6`=n^s?LwV*KfX`4)$U~# z0N#7jalydI_!+>aJs^NV14W$R9ix=CuD;IR%lQ{MnHh&Cg9VUn6E0q)*`v`+kKz$K zyHMgm268;w!k3M@)lT6d$SYissMn@|&XC9`goj+y&XHOLyD5REZe0?mkT-1Pr%NG1)tbeN{pi{*eBkev0XecT;#;M2^((zTX+dc|eV_4MAC z(j;1Vr;9FN?h0A>n2ds%@h5b>(@TF!siFfT5=+$YaS@R+ZN*mY50!+9wT&QHsEjsnt_IS zz=OC46Yk}uF2SHU)dDrG??KSz0`)z}VnRxMhfkd@KdXQ z=}vun3a~V8wYQ{|G-y(6?{xa-cXg+|x>KrAvSe=?3$8)NG~=s0{uYZQTJCATUn6+^ zL;Qft2Q4@Samt!hfZHCf_~gZ$^f&*r#g9A|Y%>0~AG1ytnvV{u4i+vyNQa+gqR`(~ zvB3oM5lK~!fik5MAzSS34jXhsn~4rjoUt&`p!|^{5M&E zGBwz1Zb|i>DNXXx)x1?V7xMSYP~a%?Wqo6=DH>OWd+jF0oE?3g0IS(Bd*ohsBug+q z*05sN$rwcx+Wh1s=Elu%y3o30*BBbGn_82-81HbuNxZH;<>S*ZK=QsPRg}^yx>SSs zV`E3TuJ{YhEiv#`1Q&GcJrY5RdJNVkGrLnM&>FXsP2c{f11~gsx1$fp=GiHm*n(O1 zHurBh(>?8FAaje*eD(p)l(>Kb+XQimX5UD~l;|PI5tPqM|6z~@RY;>u$i)@^912rD zCICtNhe^B%2ndPI?8Hc_4I^*}WC281YZDn$>XCyGs@qqCT@ znu7MM*LxB#G+&tcLg(sAz*eK<;ZrW<11n1GA=!Kg#;1U$5w$ItuxUx`GX6E_Q+LYv zol57m@r~sSDYbWT0S`$PrYO!{s9lxa$2T+u#&un0&HfysNqC)X@#9s^YjwhhKV0<%Evbx*!M&fFcLW8`Sq80B}!_T z5G5#w;W*Atf9qHGPuIGX-G*@qH1EOf9 zhW8I!x@6LM43$=HRn(k!zWLU#-&u?B&N<1nW)2PP60VCeIaVrf}l&h0+Du*F}2A&Wo&*IH06Y=W5 zUwj>2Hzd$h6dQ}GL8pLb8u2_d@Sx&$@(`M;zIB`fK*3Lqo3 zA6Z~psa^4ijST@X$TyKpxeYGu)|Z^FVeSVIk$B$bZ9Uvt&4e=0H;OgLAc?#)DYR;B z^reBtnSQ8%;yqV}&)XvtHMfPdK2c{Az6_93EgZ0RbD*;Qmdkd}^kVDNOC`}@IvK-I z?*47PuO4ho4Re;U7lD$r10zS3)8|`r{9vnD=lm3Zuw`a%pIe`oN72tANPEB%-9QZW z7~(1K?>(~#Mj~h!nPMA6h8s2--v}qY?10VR_6lT=fDNV&g%W$Ho-lp$NMmL6dww`K zU)gAPAAL(kXg)2B1_Nat`mXwvhTWt29FqslS7Z&$?+7L>O*<37F&i{?acXqevDzeC zXCAjCs#p*SnBJ$mA5&m|;%?3d1!H(09SUSpQystVY5xolk5Aty2&Dl`VGxNg1DZ@H zlxW`4F+6C1Cflyvy8I#6RT>6(*Ltn2^KZ8Q_c-jiKv~6Cwfs_^@*ED}CkAFn95`8; zvyLVV@|~aolKM$ngolY=tk@Y0WXj(wKZ`Ag(PP@^s5Bba3}s)aMUL$-1daReo$`@k zpsq}$&*P6``y=lq;dv)G*v10CG}lj&QT76oABF7B?f0Ss*6ynFaw`)mNIE%=%3d+S zr>|H<@P0>lOKxQ9t-VY9ilP83y-&!ZUchHcqw#$9 zX%4|r9dunPMXE*CBV597K`^On?vTUagr(*S{!yI6%Mbr<-Nt^a&@)aI&n?p$%V%YC zc{s`h1N1e{&I)J!p$vZeQ}cULkHHi5mvfn4SGuc<_#tYO?wyP&-7D&H>Xh&8H6Y{C zSBMt!7pM0wF+LQF@!`3oA!z+|r8uQ`I~y!6oh3jF4pl!7$9;$%3PTY~GhuSDzLh)F z5S`H4?Zk#{R-3-MYDkukZ1)!Ebu^VBAjKQzvzd)iS3UsWDCUd>n_pebM8s5=+> z(sdy~YKKUZLrRa^QE+8P)-|rBY?TIyftL?)A{pU|?L)&2jmI{&|4M}-rI_K&QCA5< z*y|Y!$!y}~vc-iBx4zVg2kf<>hOzd>-WJ>&O(rabX zAK59nlSq2wY~)kB!n0gGbE01#|55=&)XBX&cRWxmQLF1eU42T^~CaqzEnz zrMY1&R2&RKJQy9hn2c6{p;xSHp;tFlq6OCwn9qv0irW7#tkzESLUOHla<4`3+EnW7 zldn83gnxo*=jNFdCB+}z&Vy%jK2))>kdw{_J4eUIqZ5W^>dwyUC`fbr=)S+E6k;6? z2KZ4zvO<+3%w!E?Ny#$))8C_3{`GZeT6|a1>tDJkoA{Eh4sahia(te-UYOTUS?13@Pf1c+R)560eX|HH9Vo zEcAfcTe0W#m@T_7?;`InduBCN5QcCtfp8+uN#oa1-EW#PVhJxR~7J zWaxmMtNnu{MPrk-HGmT~KbF_A#`*5CXZ`NI?iUD&Y!^(BrpEONVyrDEXv&ixiRY5p`wS1UvUOaxtLAe-I{uBQ3&L7{=ao?iGU^^@5Rc$AG6>`*mqlSl!! zEtz@kk1c@u&;aS+(7Y>v7Dh%-G* zkyqH+J~qy&C!Di>MUbko(731Z*gxHEPc3BE$?SpF40f4rG43PgoAe1^q}# zkVz;BGL;DK5_lT;7$$UxJ35I*Us*i$$Df}K7nEQubDw)Ll*ry@xeHJ>Y?{qms|b0|t~VF^8F#4vjKa$BQ~5ctp=)vUL6^*$CnsdLo-bj!n>ah3GJ{tZ({ za~ze~4HRQUB0H5yO#bSl7=sOOD-V8u-8?cOT-~T<6+UpfjBuZ2c4J+-cZK}VrCEOm z-w-JY2X^q*U2Q<1#-0UJyKun$gyio4bCl>shGD!XvT(w~f$G6bKrwN827mjzHF~D& zn6dhEUk`0HNcU^A8O&!=EMAwi+ED)1a{V?+39<`9tF|x}Ny9!t+Eo{V@JK&4 zBDzduqs+5XC4l#yD{~oZ>xRql5ywHhV_Nl3fWC-8l`0@VK2iiLvZ5%U!S8d6&;T*4 z`{Y(ll>a^7MI+$NY;^W$1A^}w6m+Wdh0$bhO#H|n&76ftz6g%M)5C3`OxD{Jz^Iq> zZJ(G+jm!5r|3lA%k(S7hv1~tsjt&>!k@xPutM5x!?(NRH8r;kE5=rXy2{Zr$Uo9>x zPCMeq1sfLq<5do?SNWU9BySX0?VaA2iic6J|8*9}Y@P%et;y z6-Qb?61DTAstb$$UscFVLCH2Z1kgfSO488-apRn?CJI#cZRCf{M6lhvY=j!yS=*rH z9qN(plszeycZuPxxCuVU|3~zHy0*0^>}l|LXf^#NEDSy&H0@0iJpJke%KI*wf|K$+ zr!Y67{Jv5MjTXX+goJQwd=mbIen6#S#AIr6!?Qg#Q>5wS(!8lA^T*3;>ArJ!xH-(^ z(E*wNzVUQ=A|K7HvnQc%45-wEGe&fizhmpDWUWJF(Ko`Sf8R-*u3#51B-4=#WTu^_z*5Gu1O6sl3fc6`$nwUZ-3NplX5@lk)&Kkns?Sq zmf-JSFR#BxAIy@bxUaonstGgX2?l%OJLg=E!FSt}yhDz7Gz}iQR9^oj6X9L7YHdcXT6POGjW zC`v`h2}s0ixTjnoY~Yk5gBToCqF9WWn>BQ!iz5Jvk$!pY&z4hlNQuPrDU279x6<`d zDz%{zc*6n!s?f9%8w3^Op?=Yrivi`pqu3`!aR7k9TG07F zrt4SH|FLJa)kU0-{wukdx9zv{@PMK|lx30Mz*|o{{u%1&Ek&SL4o+k0L6dYtzdAXL zdfq}1aft$kd$Sh>ll+xL0XSus0qSHGD*@Pr9}Mgn^){uY3;n*^XsE!HZ*%INQ|u(S z+e@Z?j@u-B%U-vuGjWg8=6vbrlL?tSS#28=>jADYK3!bvp>n z9IdM&LzVGZz9NNTIqwsJOqd43t-LLmQDVt@zKV#!>y`EA4~AgzxkYE;gK<%@*OUyH zatfgJ_Ps0u!#`;m*G2i(Nl{0eZ-CIti3^#E5B@1T$FSX&2uLt}3p>+-LO~uKp^DmL zyRL4>o)K4NCF+{tYuw3fUu6Pm?!P?6%lI`}9kosNB+mtTpLKJUzT&Rwjb!ghB~_)Da_1r#Ko&UtQR=ctZ;&K2~IjMN8z3k@)mo-lgu!tyj;fZr5g1Xr9R zb+A{<{~lEXjk8bc5WnGp_j||l|Co@e+o`-If%qSEPugVYiS|ZDeoA=r+c_0~MJs-y z|3ki6{3bfOm_d^4cR<5$`QUW^>1Rm-nqFNi=H_Au3{lW#tw_-6d@;4me14?VoS(@j zl7*)BiLmB@1R#6nxblV@S89r#sjS^z2EMDq1`+(%pqD@j=wnt0i{dn*KI`OVOc9Cx1_NJF{Q>OvHrfl;d$K zUvQPJ2(K&0f5%zsGX*eLG4sph`;E)f4XSfZ-;eh?Ud-We4$pn>H!NpmJ~HwufrC}{ zQpPt=o`iqe^fzu?l{fJAF44Et9UvNtEtz+ya7i%zivN$`B}U?5;mFL4gae23N^M}~ z(=P!xIDRy{?j(ns*VN3AMvh@Xr$I5RH^j3mcvMhZv?j`oF}M3#Qd29H))eapJC(Dz zZ0`5En&1BSZ|B7M*<#uG;(5E79miq?x%~6FvN`8C8$bodhX48+wg60$&FiQhI0AA+ zf-SRf{75VbS$;vwI=uSB!j*;pYB*eyX~(9 zp?kP4FKJpt;~ZaMN0lsz0^}?BO~^Z3OZ;FG3BgQOgbU+$hPsZtUS+(_C_$oZff?p* zEoJtPQn|sA&mUGZi3oQ;v_j!ZSDe+csko!)F93k4sVBL`#esTYqYG6`=Uo5&OLcXn z*#n}1_2`n)-$(C}^A0|a1y5Q248eo3W@b-+Qpy_9~Nu-q53Yl{L%LU&i4OK-3ttYxD#*ApF7r@z8p%No-{F; zRAGGmn7e4w%N~M_LS*_6WK?CbG?NNRh7UHpZ#S`89_M3IisJ#WAEvWeYWb?W> zb&h^K;ryDvne|wp`B9|haBk853Dk|dT6MOICQq+9gwtFT&-U<@kc+_;7Vd3{h%8-- z3(44syUwsq&rDD{yu=R3f$eRyXdD9evReZZJcT;B3i0ayHn~pJVZVV5K`93$(4^ao z6M*a*dH6Wwj`w9@u)7EPa6OB6AxY;QvVp2KpmOl53|X zRW^HEznF^3+mMz$Povoaj~7bLk^ZlRU!H;ofV>9b2$_1P{o z*q7gct1hW_Q=J5Sd}_;a7lj@mzl)K3^=JJtL?S5P5gq+f)J$0a?l%!(QveA}ht%r% z{BY0(<#iG=8fLcc?jp~ReJ!yTH5j;&o2_&vbb5)=@c`*fPYAxgb9VVBJ2%@X-uS_P zFIm>2p_LtP&y`Uz85z6HD)LIy39_>@vCel8eSXwFi7_9_pyf zRA8*o(V|=n6pz0{)Q^RDBQE_Vf__S}6@x>iWB12Rvc2`9xE^>A9|Vk3R0kh7z30HK zkrIzSEG?z4?pl6xw`=r?kL+oHh7TtLVX#HhM`9*4n;FI(f%WMotnY?r7pZa1ww&T$ zE{xb(3HlUit+PqYUOjV#nYI0^VX$84&9-G3CYaVkvjKL>n_@3pCYr!#tS$bO!t4u< z_*bDDl7O5pT<#9D)OW#X|Hc)L3>3%?o0YAR0@(VI(N|lgmYJ{YAjDN;#>y(e&E|Hf9Ldwvo`$zEl z-fIoRth)(TN1VAD(~NAfgh}VC&%ct-3kH|*MFAn|V+xm0DtltU%v%(lM9Kc(Z!;5? z#5UaI-P5sBY#JM(*!fj(lXr!o9kDIkj$$Ja9crJq&`T`N3#bCF5w!k*%fmWD3@ZH~ zjbXn5lVUQXKP?<%rbWS6=0*$#LLKD=d$kjKu5$svSY0nK`tB~~X>3!VC}pM{(-U3s zA9@Z~-fq?&*37*@K3~4qvw(p;pSwjjo*EYxaRGE+kh}A0WQD8c!V-2f?S+TOC(mbo zi@I*Y`-@r0vfS#`-;;jE;i5ZMB5m;mVD`uF0&|RItS*)8i%mYI^6=obvxis2Ovu|L zDffAHo!Wh}@WmUp3giQ)_|`Vu_~>&sYH`{5zhh1Kt21gxYWCl<=^qI~#ZxXCWkp3R zf;btYi?V)Myq7O6Oz4@ctf~|kahKMB-QQ!HdKW+k!z2VKk7`mv!WF&ZgK6fImpnHC z=D91Z^OaTvzZ@ah37OGawJ^!Ry9`?%jwjo)9)FY!CDfkRdYR0qtVDqkAjEmz{!28C zKg3C0C#oKNG<_#gVA9&TIE8TcqW#*Alw0#q*{-(}u>up;kr?EegSW8i6k1T0d3C(4 z%xs3SJ@M163T$YWyALoui6f)DrIUV%I=~DbcN=@>*5$%_P>Qa)NDp!}aAo%b(NH2Z?mApXiXZ z5&&Cx4$;8Y2QKRohAGXStAup|waWis6I0mt1{m0{9A!pu2?gG|&50emh}OvG{zgSX z-0gN|1VvjEoWwzu^O}dMW`9`vr>O^`HVe(|Vp(E;M;uqKVSJ^10-KRU#0*|u8I88% ze9y(7EQe+kJjys7;b%@id!h$Ex40muqKML8MQ{=YQ<(@qK`~ax#mI${!Uh=pH`NsC zgow}y+2m*#5GUZjS4vwUCiX3zwlEkVEf@m28^CRDG)>TZ;SWR~t=*V=Ferq)N%st? z$B63}YA{d%@x?Skpc;j`WAvA!E}oTSZA|K^iOE5*jgcAKl#91yZYRF*ps zzgVjT_!l7V{$e4))wrL{G1ceUv>OkSx)He7M>mW#=QAQ6&r*x~O)vM`XdweW*-8=@ zM&Yx5XGzaaW7%-xBkWay(W=2sQt4S)LE`vjV?l{WTNx$O`5Mua%STT>U2YD$Q!;`i z1}p2aA_h;9FZy)voOAL>hQq3!sdEb$9KNiI#%K{EL3t}vONgGa-Ko7N?N*qyK|bep zFJmzsRbvjPzZ(uYyRG0L{*@{^JmU^LtI)uSwOtZ(^rtl|$c+noa1yLNPF4MLmVLuj zL^R_gU7Jx(kYKfumt-RY46s4%M0!>J#3NN`tc8I(W`MEB&T$)32l}}WsLl(uTmY-Y zxoUy<8Q!#%aY>*3<*sv2{`aS}_h6-(=8@#T12L#>T$e2UT7KjyPs_?9&ZCacxV>## z*si{L%Tq9FY0F*J^ z+VxelluHOfmaQelp+JaPoPW+wSzjGs!dc{^BT<(Gfh-AjxjG&fb0EML9I>Uz0rZMU z!+&$mC3eYK4p%!J*yA!$3Z@<^s+u(do^oj%1kh+7S@Hs{is=9IV4ZJgqXUDcZjD>d?8KM&W47v_w~xE>_v6joM|WoAqIfacZd>| ztD^^cwSNli|BUwyZ>|1#<=JsvxGeQs6IG= z+FvF(w$N<)rg z*YvLp!F|CdDE;bz#M$P(vRe3r!n3C?&Zf%7%6>C3UucEU#g56to&g4O-p(;*Cf4Cu zMXwz@=c_TlmkFnM7{IB(wyP((hx@)s(eyaL!m-NdA<_od0kXJuYN2-M2vYpjC4PSF z3fmSJqW?QbK;1Z#xDM}~%i3D&3DRvtjqg2b`iU!}q`0K^Vop7$`h0{$W%>O~+tQr#pA(LJ;qctTxwQ^_p82k!GA&!l-f@8z~y@SfP!6;jA2~x6! zhghv{#Z%q~4D=K@$@>>uOG`-H;N=8FN0_4~QFMw%MhycAP1&fyAb)GaVde8L)swIM zhh46Y(W;vIRgI>UNg>0u*P`WNe@1F7Lutlk%Rx}z*5PjGmwSbbw?X+e-XGid*enA0 zc?G)Dd^R833%v&OAt0a_U_SlzPwua~;dAp}TYJvyW^D_*>4%JD_Y&=CAy-2DXdb@= z$HR@~8v`Rmte341Pi~wb;KsU7p0x3?sQ0*O(|jJj4La%X*DHv!uMf@hHQ9Tda6k^H8=HN~{xKMZ-J;Xo%G zX;))J3+T8j+0Dv?#>eGWwTdEzR2TVXFI@6*j=CNdnpyxk#&(OjQ@`cW|h$CTYD`^ zszZZ#O#m_cMpimFr4Mm4i?h`hE`00QJ_qk2cCV#c_Z!y|tIh?q2YtERjprC0xeK*yT^T_ zbUdawmDLP}DVTb`x#OZ!Jm--*xH>~Ma9&z4pR+DEvQ; zf9|+5L-wXb_6pffR%P$K$qY$$?odV)8L1GFuf6v^$==CcC&}hy&->l)FSy4&9{2g& z=ly!U-p|+Xj#D@k-R`ESuh(|C6(D`{55{hID(UIbKfr0IYz9m0;R8MpcEymW9drp9 z>RPGn>P^>5>)85_$*#yfcJ|PdKachfCl0m36AS>S7mj>ZDc~7PhZ&{a6LIzc$*;H( z)2rPa6|AvOec3k)-q(gSrE|v)&m zQ6|MJZEW&HhX(?(@oEO(0Gt&KQ->|8oCDroUS5Xlt}=C4!xSB1G6ScuhZmpjaRmd! zLNC+y)qS(L<4Qy#$le_!VwQs2)mAV#+c^SyDWiw5n*_d0>U0Pb6f`k`7c;%o_Xe)s z0jmf(P}LS*iWq$W^cY|4rcVzcxOiB^!olGNIL^XfHUFV&+IY|OignD_3zh{}J22uS9E@<^<9`4uU=kMa$Gj;LGS^&(s z?B+1<26rpMY0~E$furHj&>D!1??v)UBnDmQKOxN*5C3F=^X#UzkEaa#c$4m7>O}=R zxuxKV=K!kE(F)#q2C_O%x4`6-7+FQ7Uh)te(n!BhY?^;Xj5>Kpj_IquJAPqVj6}kY zN7hdIw9wZZftT)n8_V~8uAX>uDGsd>p9Uz)Wp%V;lb~pgO9#DUuTWp%zx|YLb7X`M zvYkS59{4>r_bqy*4jC!-~6)^1mUbKF5@W8 zr$@~erGoV(B?KD47V@>gOsqGy3Zp;u|wSG}7_&xh{b6YLkvW|}X}y+{zYZT8##H6o8Q&;7%7 z)iQwFJ;!L>SLz(N+pV=p*o#g$xO7?h{&KC6B7)K|!EoxV`m!TMRqKQ7GQs7X$L1uc3@pX}qOI2WA7(n?J1K2bZGi!`A=aWPk%`~1I!MVp`FS^y}@@{w2J^J$1Z z6uu?E!#{3;BKnbic1yfUOiw#=Cd%2rfw#pbU5F2#;6t`1?7^Cp@EUe?7W$GQ`18>C zR+fW*Hga%Ht3WrdI@nDs+M z!U^!rS12nPfe6F4X{9sL7m6W2R!-YZ89Fbh?eQyGYF4N|KRHoeRCi8nY!0vf4KZ;g z^1D>8eQq=@&G3-GYpdi55%|HfAHTrBjFKvqA3=`!7u{*kZg^pNE3FUH(A_IH-eSoh zD5GZBUnbjHI<5{xbHvQZ-K|9Y4-e0-$AcB{H3vPP>^^wQ!0i5oc~pq{>QISF^pW=W zvK!De0E4cWMUNo@7>1=bhYU^Y3(!fZ%L4BaLJeAd;7g5j)Oi0s1kddzoK*!4?g*{M zA!F`*OP>d3sglfpVdy`zhlp=Wbk;WjxP$K`t$tXCYqUy)(aoR292ueP&>_iyyMH1=h?(X6Gu zoSo7&J^!|qHeRNrNtj58yuJ5)c`MC(@$&LO_W&96Yf4T*fsM~}wKDpxdZfr%0oa@! zaFU8w8=yl#WLbb-Z%R{%zep@Z&x#5dRsoc=!RglMhOUOj0!7F>SrPF;dMMLlD&RGE z98`i=G%;#vJx}xg#2dTQ%EtQ~N0M?bs}Si>)mjQv;}sHsigUuPUCJ{`0E~I)*L+`- z#Mi<~>X2n(P3hUsWEQ_qCZ{^Xa9{b`8}fKuw%M78D|S5!tA@+sN4^n3F@k$v-?d~s z(E}q?03$(k`swTZ`CiDCwEn%Ob1a7GKlLlJJ}O%xtv9G%KVV}_KIpTnR0wsd4M*Dh z{C0ZT7uVMy(Eeu$4oyvo{qm;1emPIS`C@{I!>=DUK|_opqyl`WL-(@*J#2li-h@H8sheS2V+Q~Kc2h?V;Y z@-3X$lIhVcJz`oECnI^k@0?D0@>r>Y(8H?*PUoDC9!IIHHIdl<$G=J=Bj+T#Tsu7S zwPa^!TRf`6*C88WJ>Z`!xx!yBSN(2 ziS?H$Nw`GIo9LOdvtOdRgWD?tYlddhS;D8fl(Y&XJyZsSjXpbZnp0)1GJ$?rEav*k zrHnom!SbV;Bg_+wY~vl9|ZN*Ni#x7Ym0 z;lN%YulD=$S@s`pRVB%CKc)tk9<55R3UTVso(&wfUWbY9sxW|TZkHqeZm>I&3$36` z82PrH)c|tww#1bwW!4Zfpw`5wRiPWgDM{_xT3(KnZ6f^@8~sutbs*$c*lp2r+?j!r zZ`Mshlxeqm2v1R}we^v+rpGhW<>6*<%HL3JILSl(G*0y`C*^hVy@J#)cfd!{yYugy za{v`!mf=pO0h|!OT@sn3k4YYyvvqf+fmYIy?aK!1z+91*fOAzC5y}+9Rx#BL=S9VC z?v#)`lb~a3iFw|;;(2S-z>)--y)D6y;%x$NtUrr~7kg6+DtZU&Tbr1~z@WZ~gubj3 zZaJTd7W}1YUU7dn$Do;@_<$-;H|qTx%*_N;ZnpXe<3naDjg!$IzujOgwBOb_gHpSl zq0*KWG!V=YE4vH>eUzXLh7iBau$C_A%A`52-$nxv0=>-Q3AbL?>9~lO3wk*) z+f6eG+4@cKP{;nfq^E)j{M~)<`$|b&1TP@ZT(Huj<7z9dG{3t)Hgm&5jfyms!ZL&} zmcQMf`=(2N82CVh>#`>bI^yDY^vyl!=h@}23_Ic0v{^PZDO{>I=DG67fiZH;u<_dH zv65u>T{YO`f)bK_F&)<=gUTFny}U@YzV~X2^h`IZom)~m8@0?04%RApq7~r-a(dRTO**bM4{nI6ZKyvBIQ?mpHCq}HgbQ#a=Ef4Q*47m zH;)e_ipGq^^?u50E8UkR3EnQEXZ5u^Qo+3jw~AHP#h7RQa7e)ea;f!9V+P(pz{_Xx zR|*#-WqVS89oPmQ=ZRX@s@B0u2tapeT9Gr_k$@i}Bi|^WY^k8b7S6vX`8k=tO5G@0)R&H_o?8%l;iPqd2`O zcZNjSLSE^kvEUKGVEjcCJpJ{NwROrrquM~X(h=i=iE9=xZG)=Ht)ou5{fks5mKazR z4>l<8vz&-S5Rqk?b7gL8{E*dZY=z782@AXZ!oa0|o5FqXIatMla@%qz@f@w-q$Mdp(pG zi%Vro&lL&d2GSwIt7Zqo=h21yxOhP{MtWK#60WBz40*{JL27yja@DL3hhDFH@OBIqBpuAA7E#?*Kof9ZQn<68d6 z+rUXuC#xsdgarbo!L~93C;rez#*#rmF`mMAI=%eDZbpO(K|EP*65vQml}|Fntl1T` zih&LuMPB#VBXxzl?@#7y3~eUQ6s_~~QC<4AE-6XXC1u*`jniydw-b|MJ_r!ep?|i9 z91u~it-TbQIiC4am_p|B;yl+F>FL+Nr;%Z7WPSA;1<;?0sO=oDOfOcJH!748Gr-o8 z)yJkV1@2ov8L}*##(|Hvwh;LZlR$ac5yPe5;kJm#qG+hvf)%~`%8;a9Q408fa?Ul$tg)GyTJ3|Bv7rR}Z5Gaa^5LH*QEMM@#KZ`J zv@zFHJaQXH@2?~cFmv8QJoLbfKKq<8Pmi6Q#!Zjnf)jI$BHbo;!b?c zoX%am3pCV&Yk3y_>yn)VGRQv1w9@=D3>p02NKo0h$ijGlBG+P?|D6V9+#j~oM%n94@o&d0j-lWgGF_Qd-Lnx_p9>e!IMvd;5bn5NGM#a z+Ol+*K|OlmTW|Ca&|RU0-mNf*e>VA$9u~OmA3f#h$0$Z=7f{grcEA-!(Nk3`aiI_o z89egBRE*qZ=+Pk9+G=fEE-Lxp>XbUi@N19q?G4k|Hy&1%aR60;Gogj(HT_yja0V=n zLdG`UBSuA3jD+${AUz0x_Z3gcAe;pO5fSZa?^n{{&&Vg2v`Me>pFH`wf;x3qzGf3! zA^M;sCDkT6Yxu(8apg{v8eJ;gM8tWi>C3SkyTtzja5SL7oKu+|L>7H{_il2M5Fcgs zB+q{7Unl6AJn)dur}=?Nd0za}pXcY|+gp4TKYGO|3GLQGBM8kF1Pt+S)<{LXM+=y{ z%<)ZDkzBT(0Pz{AvPt?>b_!|{Nf$OSlJS3Zp z6HjLCt4YntF-ZR&b)?8;mPEu;EZzB(@v@U+*fn94$c}NxwE1G@sY%rP_pmHH3WM0^ zVL=yXejlxvA0*|l_i0>a9pC2c786-7%p>o|DvO-kSQTL80-U7iS08lYhPQ#&A|V61 zb-MDwoABH3nXB{p?RWKzKj3##=!>_763nj>CeXO^<;|bhMYt{t-lZg;QFh793UPt- z1Y}FZI6}#(=ip#XqI|BgKf9XaI=TRtb=s&HDu4}YTmLKl$Sh82fQ~TZZj|m9Y29oU zt(1f%XN!=(fjNVBvDRk^G6CVbxJo$kO9FY=$K5NLbTJv$8r&2fntFY;n$E#7M08nB zvAeWK3F|i&hXN_$)3CG_YsXGI(FVvHc;oD2V|u69xL1T=n=`5zU%RUhiGTkCC_{b~ zLvL00^g>bVVMTbiG~6o@_!H&+5+^^s^jRdh;bR@MV6O#aKKCl#`u3{Ro;LR2&Bdd8 zqgQuFgwKS0ed+15-fQANjp*J`T@eT*$p#Z6YPR|)Y&C3rCny~&pGcnb2B z43Ki_-tkgATihs(EUu+?d+GE9vp8Y&X=$R^9_i;0G}9XN&>|Fmx`M>;o01}mVEWp2 z>^@IIHz#?CaZ=6<6UNB?z$&A8L9ZqG45@sX@RL88vI6*C|6)~N-B%9lkYZ!`Y>fSFnEIBucDn5OHS(gMUokPhwe z%S%&L2$noXJAQ ze#~>XMj|hDp!c7xdr+E5ztirUH$VN8V4S3gNJpslc@*WTb!Rp~1Xujmuz-_|gS87? zZYQON*A5V%a|Z&nlSyt1f`>m*QX7ZCnXiw7KY0QK!YNhECI{37f{$AD`|BE=cafD` z)5)T_loisf&+0x3ERY;j6j!z6H`9YRVGtMLWCZ62mrYsuTn@$u!{ zjFxg@CF>v0{+^Kn?A@;3`J^SXjVq3g|Canhs6@c&R_NcdWpag%um}%t+@;(lGVJo| zB&%JHSn&1VEH>6t4kGs$DB=L0zC4SkU;(boUL8F}$sJ=taF`zdAROSaOXQbq^f1AY z7~sRWnOuB}s@lDFH1GB;1vx0jmGi4fXQhawsL9}2NW2#pA8tb`VlE544q|URC{-@l ze9lap&;ZyC1zX*`l%jy+@}8bh6$Sk@gM#v`w-CF(BUVQ9)vpyh{G>w#EK~tX-~lBC z74T;Vo~62^U`A#yQ-^(?fC0hs>5Nx$lb}4@GPa*i`l9@xvb6@Q`z(SEV2*T!$feT$ z8dx_8({L-8>+9c=oq(;?=ETWfIaEYzOMQ4>^oH&8X9vQYQp$@%P^6)&suE(1}*Igz^s!I zJJ1uh15Im94lKv+_4$C85^V}A3W|&hUF}$4*IVt^*p5#(yC1yRWmn)n zi0s#cFh-DXFhgy@p6llY(QW}C(oIY*%LXUEBlMO(zr0tI3w+M_TA&;SM&m#Fxn9p} zRiv9mJ!$%M3IP=ju2LFYrGNWh&B@DK7%w{Pg3l%sNwk`4>W((+`~2&$F}%Fb5-xm>1Zs3K{pGwBY-2~>tUQqCvhu{;Rg$p_~ zn>HFL7$U!5)ao%|yDIfXkIa}~{1)LyD&MCU3=`jG(<(zcwt7hI>u)+KO@hF9PZiFM z+u(jjxEK&ShW@yiq+Bp~!=G-pX`K)qnyWBI4D4QB;TghorIE=e<{(}z?u~KLY+jfr z(U&Za;ztlW)DS*5?MbReHzO0|JVLqatmz~O4D><8&c4d;Q%bC)0u$(6C%Say+}5l! zAx$--)_C+bk1qB1W^hVglklo%K#t2s2f>=5=(D$POHW@i09#m}A9(s`#@|D?iU>0L zMiP&Ljlr9KfI0CRKPm=mSgsV$IX`{M3La8rdke!*ah2b&-vVUG@L}LhM7QO?lUt*B^Z+?8U{%Z!{W?=US#n=1OVzY#pRO^at|^H0fXIQ`?dV}ZCY!xR10}_W{Q$mvsJh8aD8~!=h>NffR099 z@2}y@G0v`6D9<{T?yaPW(Z)~R*jtY|w0dR(butU%*Gt2fM1OX1@8Ph`&JZibe#}a| zjaQl~PjOb8G;QN5KFi;y%m}c5B`MyRgVU} zPW+n14wg#&i`m?pb7Kc7qZ2fk>}rZ9X7c0kvIF<4pUE&4d>rYrjTnk~w$vJ;3grZE zmPfNLX>C5?X=ybD{9Y5%oAMlQc(`>Zo0HR*g4i1`Y` zV${M3rW$`K?p?GK0BRjZd<@y-4}a`F*+gC*T{c0?ks*C_$g{u$ANvD8vM~fiP*SC=8nNDhCuvMf zsF~AT%#=W5@-|;y{069T8N4y!68FmBH|DoSW$XCwf2}VW_&9!eA*dzLvZ%_@A{^f_ z)RlYUtubBzeF0aI`&2*hQR`?nTkCdF8JojFp)J?x`hxYt~XVOsVKC9+D_@wk2r1qmIwb%KA{!l8)d#^Y6&rOu;OSQx3a`k@l?`7*2N z0?wWpY)4WyhAZ*G(0x%UIG=0Qthg1bMB#1iN6!_5N0^>(*0^v3H$%JoNS9$g8447c z8w{-hy^cJp&WgtW5oSF=co2jHJ0A`dYkOQQUHk`l&hO zuk!R+YC~P&M}VqFQbW=EoZ?k9jMw6XVBtGYt*Fc6GAo5}B*CL3*T6O(c(6Vy}tzP0rX*;@!@u&YgjA&`tf&qkua z0YdeKJ#I1rT0mHdn1zr4Ssx>g@}>N%>*-~lClH+vs|rYI(+tV?beOGI<1QhOFKveo zLBj*3M3`f4Wa@;0RTkF}UD5M8{ZmI((s$RZKS&vI-%BkWi#;fFbSoK%+rRpTBbw6} zZ$f#9mezG{e1N483jiR@>b9g@1Rlw1>bdAmdPphW>JE%hT!;F*_^p8WNg*&T6u8-*kqnBKxPIhf`j8;i=O za0J;~P94^zf1Gk=>_m|sygs|K$2>fBbFUhY z9-=6R@d(J_#0I;IvFq#cd;uz49NM7o4J?>%UxN*rU1?tJ5&Ay^VwR%h0u**S}~Jt4(bj5Sf&x`CKUE zX=Pjg`yw`7 z!sbuV1ZvN^G-m*?8d49A=zBsx$W$o<0`7AG;Pm1?1Q_K8UN`U1+Z#I{3*8q(*6ev{ zK@0jC5cqP|nO!m2R|6DWbEL1zUWwvEhWC{c?m@1|I{$tsc>aQo7SWp@!NCqeAe8_o z>k|mN2~_IU)X5j}+d)vmhz|=MPYBKg31aYd4Aimdnz~tw>6Q!S{lgZIyNC^MUjOpU zaFDUx){rG!3n%-|R1;4k1Ifw?EB8kuiQeb3VgE@I zLOSdej?L|-DXk+GhRbE;+He0~RCD!nb#)EQizG&QCxQs!MBAOu{!o1A;E%;c#7NqF z+4LeD(9!@OWt7Y!j5oyAN*@0#E7flMtr4{{I67^^mDEA^gobDfZ_#u}$V&XRUEmT! z&2M&l8Ub$cek$^QQ8h{>BXa$iq6xEG-+q)wOTFj$iYXJUSA~LUvQ0ncAz~H+JI3M{ zf!^O*ON@ke+%djUH>sqDqcWKw|-6`nG}L|&-Ygc%181J4sdajrc04txccH+thN9ojrS|ef$85CeuJ^x&>k1 zG57kN<2pN#%hW`vB1$1wE8f@pyE(sOf6}kzMqUslT#Xmsa(VR@GEM9IG$c3itE+aT zjjn*Fvq_}3v?mNNnlLbDX{1kca?0HV01-w-%eV5ZVPW!oeWgbgGHn^vHz`tE>(`LI znJ9O?pK`ydwDTif1sU6eZxQbafxg=G&O-P1ZTXwPE*Ezj*wv`eDT6}-V5O4-u$Q*B zl+jnwl5~-MrUxwRP%2PhwQ5NSDA!c*RUc*fccCv;fgNpZ@#nkntc(9--QC|1Tvj9? ztycq(l&sCSjwzAd8bs-7Z@bNPPU0R5@IPKM?XOg-jr;rpGlqQF{VG18b08iMQJ1q4 z;0(TKZr(1sgPP=tY10?;7SG!ri}cV}^-P_4vglxL3D zrLk|G?*CBa#RJ^O4+_nY^5&CKsi-bh$d6z#MYI<@o&b!((5;}*8hY}3x$+A;zSjFQ z0osJ*EwDq(_xiy}0q4!wrd&(QM}@Boy?d0~vI#u_s;cFk@`Rn<*Uln4!Tzyy#>f}X zM@*(>ILl4@A;7GZ2zR=_&wsOeoPAAI%4>eyql1uM0e=EUJ2OxJY z?vGsW9=vYSy%*25`6GVjkDr|+=8iBbyUkje&-1N%xR!1HRK4MwW2sxYuHu$A@G_Hy zzL8z6Wu*zLjo-E&=V$S+P_#HFKyjWkTA^D0qxn8>E~x|YO_92gC8ePG(cc)$!FlnF zlrbCa)l-P^QIU;hbGq~Jq;I=rCvF{TWks1FL>@V#TXqr&#p_Ay%{~7?I|>C`-UsYw@sAv z4fNKv5yP#&7v1WX&8gPDF!Mz|$uELksP_24E1m)Iz^mN(F(p{H(mR#!?+8$@Ok7{v z7@9^0J=ZQ`c3m?{5A-1hfFY_Rh6ZJhI`(h9F&Dmd6YLDGH!QV(rj5K}h)k7pEXQPI z-APxz2m~n)N#h18bun}zM;F+-y@`X|5(R1k)Y{bfUj)J7$&pU93SLlKQtIFX`pyzf zDe}{V#Cn|`=I55*Ud>P?A8ur`P)oDRgn5s+Dz=y1P?y(=~0TB2XwTc(d9mMS6+?b9hC|3`9UKHJd z!}o#hXpwV{=y4Fpnt5m{W_RUJ3(?AX^zZlDln4^n!{FT>R>?*6>w4#Rd79xPd@y3| z)Z&_)OE&?s+I%6da=mu1E;LJ1i0Mw=g_U(!u&Yu)Kr9T9L9f@_{WoT7J>;Rnc{yNy z1IhQY3ODyY>npWcScTXt%`~r2+=e^vF8^!qVoAAkMnZIGLl!`cF4~BsMh=zVB*L*wDMq{=xVCmzm*I5!w47Y$m$i_(z8`bPS~_y1^2x{JCe&Ar%7ff6+M$s$CKgTHv5lmFq-`G_8 zL`Y=kBO<_@9(v^dw)AAx08;6sKn8FEQe-GsP8IaZBmGGmnI<4cEj`caivEOpkB}bZ zN6G7RLG894c8gl1$2<54^O`l8d=@d#$bJ8f)+TK{``_|i3f;9j$=Y4+6H$2f*!otl znUA_V+d`6uEXyYCrwOjIrl~5DHt@SnK26c`=5pv%4--Y(BDw;w+@uiQ76;Zt?^>$`=&ahWdytY=PX;&mXCgAud5j~9{i^x$_RI@Qt!6e*7d|) zt)7e=I75qrRz;9)J`^4YV|b$yBj=Go!?{X@-3fCEI=K4<`8dJl+tu*PUdpxhM2p1^ zj-Ni61Jb86L{q&lNLC*`Cd*hEem>ltFeKi99+4W3;r~{^f?o;U!R2e}L+7=j-S19hR}Ll&$bjUg+!Bg};iX zu)6UD>v!)lA*zRh6jfOjf?R;Lb&lCfPM%lvDW-q|xcrYx7q$nK_NWEozq)+n6!w<> zUN1N(2qA8G3xW1XsLKq*t7nE!C`w;V?H!%;;4902e|Flob0n}rNkb8I+FnPCoqgR0 zVdPMgePx6c97CxV4Ywpx_R~nayfrbJ=*|}^$<;P;|H2_|kNiDY$@Mb2!u%t<3H6;R zG!&Nzdp#_=^{Ax8r-@^Yz@}YiGomk=;uXH5_9hHO~ROA_LfE9#VC8-r_(@wBRR_+xn~Y_}( zLF+^=U)B1IM}_aiM=jp`3hNK34SM=kGNkVtJsqlz0#9PO89Cvl0e(rk{`sC1b&n%s z?2a{P779oQF5awDqyaB2pG&c+c>IqVLE5=6mJgLYp% zJI;65-vmJXoFsWaw|AhLW0u(=6*0F^YJ?~`{Z+VbIZtI00)$=+4VwM=eJ3A$fR2p8 zc#)f4TRo#}BZgZ$!s zY2SQ(*0yx8(UGxcaB%t)c2%;jlxE-Un!kO9d|Mr|c)ov*e*}6T@hI zIXV_$Jz<@m^GFkJzus@YNbV$w?ks*I7)~ULvpZ5IhM-5GjmEd?-6H&wjlB)mY3rnl zAK7eH@RLrMXxigN&W8PjZxC%O#$z6q1ZQG^fSNIx_UeK}fiBqcX?}Koe*7lS^Y(|Q zl;e=~laLDfUVDk(H=Yo^^&iY5dBS$LihCj%AB*@Gu15@^`U}C3X;tl@udcG!DUD}@ z`lerk2i}6mjC6<^JApRE`?y3?Q2*rTPS|+ZnyaqS#Lw~Mc-6G;x28tdCpRUgc1DH> zF^`c%@P~Zp!J|S#mA;2Xy4s;A+3omIeS_$9)~o2tJ|)FrwrhKTMa0Jd2)gav7Yd$a zVULRK8rAMP%k88mK(AA7qI)0&y+$^?&9kczhB& zI^QXSl*R|Z^pG6hI0i>cxuoPS8UW_aez*`n6EyQIc8$OX+oCV;EVnV@pY(O6lwaDU zbd9W8ZUH-9BS1^P({~fxg@Btb;tXNGNwsD>N0-r`y zD$q|{aQD+HKnMOZnTnWjPkq>QqB~GiQ*zujt63+rKL56Q&8tNwt$*Qw_2CHVMWP4O zd1CRd!U)mTNTqeqr5WTE9F7}#!uo5&Fze)OtYW{Ry_@5bBXT2O2C?|zAFLq}!#zBv z3*qMSOr)kL+*)BNM~3<$%eeO6;`>;wI(05K^71Bx^6xp+{`@B|9V`*dOpE&dpJ9#Z zoVMTl&f?yB3n3Z2*q3+rmH$P(jra{|Zh$pN>K(Z61yKC7yC|-!SJ(o&lk4F4&3UWL zh+EpYkGoq>G;mZaS!($lM@oFaC}Mm}v0>)vVBhfEm(V}}KC*xwj*_sZ6B}=7JWIyn zhzlq#m=OUjj-nqzWut0XqW|zPG)6>CpuoFy<=v5Jiz-$(kT}$SEtbXq8{d00_RxuM zJ3&?@dB51I9Lc}X6NT$Vw->(tGMW6qUdK(1qVFK`C}Ew>Y`wa7BaHP1xeJb5hj{f# zrTCDJ?4$yL%d<+%QUy^wN~~dt=T`qVzsYhj2LS7-1R*H`u7{m zMDg@b_W#IxeJeOyw{2Lp%Syaknj}xKV*;Z%k7SQzCkSWSlnTDq4Lxi}ujKQQ7YKEqyJzaJO}>xi1>EopJ|+m_y)5(o)75?i z-KZx6quEoN7mm+HOniyxP;l$ZdpAg#9678v-8DeLz$P(J_X=iewECLz_(;}5g=rK< z{1Y)2buZI8M=-X9tqpcDag@?hZ8)Yz3x=j2Wr=Rk{ZRf6XUaIK$$${Wt1}?V8?}Go z>Q8&1$Wby;=?Y4_vG`at;f#J`gITJvV-y)3+7@}XnpUXt{Erv}{WGosKk`@Qr@~Mj zD-{$X6057`&!4~57Pqim+6lh=NBw#8wMW{Bb;G~EP$h$etkI#}b(tRRxj*xeIx4LE z)a#M|s2Y70>KYPGh@~wRdN@WH47_l!Yu1Uzdl-C=lUI~VNyOz}k%hW)(qT2(PSYAr zJ8D-514Cc{tCXcq&t^RN?fLGt)unNPb50&>#+I-8*gIvH^aqLFsUDb+Md2zID$sPh zSn_^zc6KAJ#z_S!%8GW65DnSmQAVi$_c{JXB))t1N#nd%!t7W0_}5;vHzm{Q_){kr z6lvw{UibPOMj$%PbgJfVo6#t{s-H!>R@eGD3pqO1uDHVypoS^f)Bvy4GlwU7q#>S> zLf4i6hGD+3paK(BU>aJCQy-}RFJ$dmw-`Y0mj2+xXoP-kU*cf&RnP6??WXR;JkU~n zm;d?YnH8rPQgr?G*RIJ`^E?CV&(DiAznbBXB$L^3LgY|9QbN+K(*_{GC%T{YM-t61D6LV9r=Y|MfLB*- zuq>szyIjhM_XSkesHe%V-_N<0YW0co#6x)1ZbMNgZUkIY4qOefP|#w#V3zv^&zj(# zN`8#U46hU}nIr(7HMtgsCvXPVjYGEqimq)9Jou zO`(T(`{nj*tw6a*=0M53LK^h}Q)^@>RpO09$4eCX20Cb-sWRyLv^|l3yU*`LByU0~ z8kG=xlx+Edn8UU?2}O0v_tz$aH}K7qem7hScCd$| zRSSVt5A>u+Y%MQ;t55IY%YwRF+L58OTfl*lfU{Pq5~c_G{0fv!n`y3MqANi9zaTXaFE&ZKo zTS*1-qyF8U9K1<#rMEp{VJrBK8?bEu9WBl8>LLbpeyBKdt)2&hVDpg(svr?y_)77< zHK>C(El|xbyr-utw8pj{a>pJ26+*5>C;vhB4*z%FMlKpB#_iZW?F_o=n zc2tQ^FegKC$-Esf*U^Oes3mFX&cV4)+u4PKFSm;tzs|E#wE3J zl8~nXuM$Irhd*1=^KeV2eN|jplT%dW=;v(WR>J(|lwN%~4Zh8xii`hHJcDHr)vpSN%}<;Q-*~n73J&DziIOGlY6U=z#=PDkq|JFM|m^QWJ62_ z|HyrO2muNaYQwN;32#N4^iaQ>cmf7MjZ{p(EXw!szZn+5bWmW-pK5j!YVh@r60U1Y zz>GQY`1{+0C7ZC9vBk5?aRKl4))h3H_8tvUG2QWhN4}vwwAS}=IX&py_Tv7)ns!$0 zvzk+u`!Neky%=x{oP1&WcdbOV%1nvU{HU!ZLw~<46z9PRDw4dP#wvc?GNh}QuKv+w zZpJZVP)KjVl&585J9}qblNBEHKJx3p!tsffqOpyknu-4g2~jcd`j-i;1n~?LeMn$o z(Tw5oW5;oludhnaIw{d^xaqCGKE(=~UXorovHu<;UD(!%q{)NNwzvDXq4n5bA=-x& z5X92F1QBZE=u{#GLWp1u3CF-7n0^yA2soL%xHxyfvp1Zud*df)ScB3Y5x}3ydnIH( zl(c9GC?{?Ar;nA`JF;DE-*9HlI5es}|HUWPtGk*40>#1fJ{MEV%ipKX0)s=Bsi-O% zeYP6@X%uJajKEfwFekOD$&qt+)#en;%`a}h|JJtiXCp~@NxJbr*?&bR>d9hu7~)AHvt&?wmEiZ*-P{{RMkp{9dDg*I;SuaJcOX^ z|0uJrdLtOJwNh*o{nQi4#m~y1`4LH7;J9;w`ls4r>jH&M*j;IOin57_KoON;0l+VH z4SBg_YA=nF5Q_3`Do%I(B;|9Xlg?urGarQ^NH{xCaFAdF@x&P%M2;ISP!hW;Ti0v`9;d~7j6gJ*$B@s#FT1@{?B z+3#034EClxRwqsSbGUI?!l3k=D?${Gm6a6PW?ixyp`2Qw2oR5!vv78XrPvU+l-HA|%Ad z@){fUG_~KxHGpA+>V#D>{a?~j32Xk8jROE~1BCYXFLXDgODA)-w{J`8?A1FSXX)gD zN}d>BDh6J7hWL4U2_u9rLldBEfVzExn z0naBeyt`Me6b?%qU%tTWgEl9>^9++P=@OvkNU#);hrD(rR#rxLsM&T-P~VFb&CBti zD4|b`oqP`fYPqG($*(5AsF8@=69N%!g8HU5lnOuxpKbVv0VX1WHR`ie1w` z{$1&=TgM>YKfMDGQ?RwR!QZLDeAaT4keD~W`h=Dy>xO^0x(P%sZN zc>4Q(08Uo&#p!^Ek|*5Z-jsJEF<_@GPjr5;usI}-h0;GSalMR3*_S|-znpd@aM+@6 z-eM4T{C&AHXwh6lWon!gP6@j*If5+FW&-uPPuR0&b|X##bLw*_yx z=$`!B_B~i1)bMP^4#s=CekXRp71G=lNH4aC?ils9_r;wb74Bmth}W*CPk#wf5qP$a zpwU|0(t1#cFfpnSK~5-)Pu|vb(WzZT($EyKPgG^MX_m3Ec0N!BXi|QFg}e!=tIK$= zk3f(1-wp|fp8nZg(Dh0_LQ7L?(Hs(*+{Hn^;AV?WJ;ZPZ|J+`zs)}!P?tk$V!_4Qp zP>KvI1rtx=_^GB!#) z7#rOZ`|iX`7n>F3&OWPUgd(`^45T{(2mp^%m5uV7BxKbRe@#D=BB?- z3t@B*RC~&5VeKrq7kRgshh3h9Ky-|!jE*1!Qbk?TJhz;e!;n_joK6K~oD36ZbzY(G zH{6Ly0f7y_kqycHNB+4BTtB9SNQ3N6;vP?Yts|YgBPV8cqmF`}-QdA;$~n=lK}OU;-t>TNa4 z(;zR`yp0D$vT}KUVX|a&ml}@1-v1z#JBN%Dyw3bihXUc@8*O0?zgHX^==?U%EA`P# z0K*k-FOBOAv;7}M=N$<3|Htw7oxNpm64@&=!YM+M?3rD*$Sms+@-+)lc2@Qd2`Aav zd!M~W_PqOje*fNo?tDJ)*ZcK+J|EAw4q--+rziSx9p}f=A=f8E=2fEoa7+J5lR4@b zQZn+p=?a;zC90r^FFVfp3N50k*7 z)hy>bN{{pK4f@m>&(=mY;!=B(@sx)ORqr6T1;wS|%3 z#!?t_7CP6^(Wg~<653#h#mre5%u>we&CT%%@b~<#g0X5kOUJo^!Bt=+{! zy$fkGc1%piq<4CgUeQuKad;AYqzo9C!UPQ+aPYE5#=P+7_W0{$4X36Y5~d}HMQR%x z8?n&HWt9NBA<5seH6G9YQ3q@*a>ikC3Cf<>{=5*9x#7Ud(TP0?Ewi_-tor_o*a?R^ z3j}Uh(xSd%=_tThPY(qG+1}T_H*MwjP?HAy2k>ozr&_L$j^f0!PTM<^=Q%-rwR|mr}s?HhbdAD2c@FiBT{zmV=8d!y?CF|zhb0^a51tiwSoxN?tx5zZ0tqNko4 zJG`ohy_^xLT{WK5v+^X0RfkH`0zb>N5Iz-HViGPZDxpc1>YjJ1sse7NiKzxo&))RD zyjK(19>z21)?|MrF%gxAgTDd&Z8x~!Rh{XsDAQ4xfyA||)PCGh3yU+R(>c4XOv>A4 z>~*sF!s>>C(iS|Gl-Nb3vWJ#+&zn$#Th4u$JlbGkC_nLY=au_$5Z4yTd+Re9Jj(qO zm1@YJY0g4D$!_tr5j$5M>U%ve%a8oG2(L}n2POC$?ScqL%Y=m>&(WRNp*RMNQlbmQ)zx zjr=DxJ6+RL^YQ^GD<>>YxFv`U#u=}pWR$msRdkKRvf%YomNva$dWJT?QF@>07x~e5 zY7P+bz?2xV=GVFG9JgO>rQzq|K*UC^1xvnofV%?BSFsmxSfM`ZQAn($ID@#J#Wt%Iz4@% zeeY?8u-_|rvOYd>846lI64hFd?4(I@;Hb!N_z5x21WkSTnfoU7Iuau<5{S%@DpZG9 zE3sg4C3!RfO;T#*qxeyn?@UZ8B^(}&PHP~AO#{BKo=}9X^*61#`p1UU*jFL}efS-ys(inKA z6n`z|PtDD9d=7{n8)j5cFqas}Q73MH`9bWitw6T}A@E{%@s+n#8Bt}YkkN2_(K|83 zBIu!gdxN6yXhMJm0Qc@swt3wMhh9w7zK)~y@hR@+5Pfo+eu7lxPMc$6r4&Uqpvd== z{=JMbX^s45%OU`4LzCeeEDHjucgsz|_Dc6;CBnCZrPurnDViIDh7@J1o86&A!1vN# z5@i@^JQrf$Q;^05e{sN6(d6>?bA3s|Af>XORSjN=TxaFWre)OonWN`sX9=5E9{;|4 zDKxE{q~!3~(h5qH#o7%cEZL*)r<1~ z@%p>jn0a4chal_MlmIyvng!l1pH-0a(!Gfle*arz=`d&LxeVR(f5$R)r;BM6j<~jm z^e?|dgz6QZbdCO34Uyj*o`a|Kd2?hL!u&n4AvRek29Sye<_0|(6i z`Y%nryiAod5NdmwM5F>n3owAEe0_1oIdSr+3 zH}fU98{mjRIK#h2gZ55)j3LX)8Rlnjn4E7au*gq+#0UU`thMDPEZ!iwHT zxJ$EO;4mC%onU4DzVv{M>)mbUiL!ve&FPs5!?J(4YOE`T&DWkJJD`=vQpbjw_q$J=0uVQsVucjJkPZ4*gyxiNCFc=S3e?jFlM>H9%}dNyb8 z7{8!s10)nFgb`wdSKt2QT!Md>{kz6rEoW%*E<}6uYiVN7FNdnhChy|#@$H`f_@Bf) z&Y8mqxjNj*L)zZG^BZZs9*q9j@-#MJnJAmaG;k;7(k3Fni@9;k){VDO<#?@&aE!{7 z)@ONGG-uFi$e~?m$li9uhJJi)&+;~<+Lc?Eg|X4EePujKa#eJ%*?*-8ksLQSO(Vhf z`uD0R_gV-*$~M`+i({Tf`nIz*DTD$5EWS@Xc2!We*TJfxe_Qj5fO|&rHPMk z(S9*K8+=>w4O$E87Hd&x*FU$&WZEFmz)=c$gm`U*!M@|iNTU7?FeKNLTMi_ zSH)>CvYEWB(hapgeq^~v4Q<6kL!l`%bcJ77Bm-G&$th4=ZwWU;Y7OF9LYJ^A zmEbf^3>h!Z^e-ZrDIzvMzb#m3!GNS5NH7%P*T@RP?m754s9R}g_iTDRL(=xx7wZtc zwqKZYz=2qQ{Y0##8>1U?%mBs|${i&BgADTja7-Y-vkxsC-+OA~`c^}W84fCk*>X3j zY4__=H#itP*UK2?vLo7Kq+BK$B~5p%c(w< z|6vo+^hOyy54`ed)Qvp7vngdmxCn-KfWwb`9pr|jU)?19(~)0>g#)99AY}*>v|%(g zi|gyDf#-Hff_rZikRlOF?=ENT*jsK`NP_n&x@!^_EznW>x;ORyy21;E5i@mh4S3hj z*>&>TaM1W-{+Pe(%l<3xdcOoR;q%d>@hM`Q*b5F}eB$fXgtKIwMCLYTffp-dxg$NI zva*@Jz&*40d^1pcDDpo-UgY=us+K7|H1*d3i+ zPDrmwU0(M`uCD(1?YaMe2=4lZ7`e0aC)gsw^>o9oGzhX(>V4Paw~LvZ>tl^@R%EY) zz-M1Cy!vtK0|S^8u-CKx&>sGpuAk(k2Ve}wVmwiWvh~eQcs%tkk76h)GU}ZOr1eveJfqLAuAGm!`qWloOTYe(% z`+lN$thDK-DLW6P1Fq91+hfRnJg<0dgR!!(cM_rO6oxfBbI_e$J-(COo%iEgdYJEP zf2#RARkZ~ z*h^{yLeOY9onlz_#Gtp595tvy5PI*+Pvg-LvS{|q)i~@^E;F?-+>5hb&o0GRIqw{jvCB2J7Qs=~eu-E^U-&@{dMS*rHIozZ ziX+%BQ)9wt)U5evITRHYfZr?Lit|Bk5v%ZXurkx@+jm~3;NG#}A*(&n^;H8HsFMAa z)N9a~lt;MdvCdr`Iz}gTwxG2WtIStqUv*~evC-$NG4HsL3GAyMxXDIKy2~!aSh1r(i zS%fLu>?GNHO)c|{^NPF7xZCXJ^^$=n^Z4WNJAeV8o)(p3=_YD}w#OHrtLmQ#zPei6 zkhX!swI_v%Ofti)W%oT#($O;rAW*>)!7e>{(2@kN7X@TG5b{+N5AspJv&2ZU#B?yI~e{RH3;3T^L>YWQ|qYE*>ObF=aHjD*u!T0;VXJ@x@%*D?*S=a6 zUFlax6E>z7DnryoRqk0#Z+gto29Rm@uS4Jf=1-(?T(L6k&KcU7lYSn)|W zi`GHAt;&*4@t|wvOys5=95KPI%-o6B>J1GfyJ&!VjPGVZsPKGwR?8{5`2}(2Cj+&p zzpO&FMg+({=&?;6U}W$zVhw9J$%TGTN@3Tx|EwcfVRSKqR-{NW`0+p~fdRY1)t+Te zozHYcs{dm#gtXhq{A|ueC|NX2lnY-JrH@0^i@&`uW)st*!}piZON4xmvl|nAlwptE zegIyf#|~J083@5K43){AvN(=&L=IeuzPw@u&AgPD**8%<#9#e7CVz6!0T&gj+hOAh zALr79=w83J)1-;~v(zEASolL%)ure1)9hu=ZVd?sj~Hkm>SJ@soK)mQQ?tPrWPxAs zm?1h3Nxre2YIXv%4zG7ta+8ireNKVenpg^$RPrqsiOoX+WY}|==T%op`DNM90v-MQ zD(pLiK>jnV{xLF$@g3nDE%a5E{5DrAt}rY!VEpAQ}0g>7h!ayfOPkT%h#D z#3fCCm^iBJNWeLF)^L|GX_7t>23QFG%h!4W)0DaOCj@GpQhDyXU`ZGcvM65wPtrY% zm-y|;?jui`iG1v3Uw#4}YK=nRmAEK)37}efMEqhN4KLmwlCD0?Oxwb~S;J9W`P<|x zqinL`brG{J!L~Mrg)kDksat_&!}}~2N0~*Z8mYWuhXQ8NA|q(=GG@N3^?9B=`tv&q z%YU>;-q!@R7KRI0U@=EGwf34mjE{ys*{-;K{1a6`e|qHLVihG`LFrC*MhVoE9*hW; z=|stUwW?!PZD@~4-GWQpN8zPBzwo!Y6(?GEAdic>K9#8bD%6JRID=Qd}%+WTe-z zw0Axn6Y#`aGjBR1(`>KYY@z7IU2DFBn_?PO+Rn2~?`KRV8bfR;$YM{jRAphdr#&&B z7naEbq1d?Vx8KOVwo^6Yr>j-=ZtCtL4f97r=>hEh(>R~t&LMD2nTivNXUp!7SGF{MXL zkqZMD(xgcc7o=SjRaZwR5=FVE131zkdUkw!O3YIFwY93~{DLHO-?h$c-ZJ_l$Z?9p zP56I4EDU^4+F1#(A%AdwvEI^P)_*?i__sDB55dan%v+K3-)%fV+E5(fSE$U{SN!7b z@#?w-_MF>J)3TBNoGn#j^IkJ4%YlzAa)F!bNQ-PwJ$X4lAaFeE{&IiKd9Ptt-wboI zh((9A#jDJ9>ktR{fkDt6mz`g^=-a5Ki=L!~AU}y8H8m?N{f8z_sdKJa|NUc685Rde z9s31{UKd^Ore>h38aqlVjObJEKVB3W-@XGJQyCg2T&3q$Trz2J7HxFKHGX1a%yky+ zY+UZ4v0xZ2;sSmGR#rV32t%@!Y(mSKhHEvYSAS7NayGeX8*+17FdRpz|c!5jl_iV9}TcX^EbM{@BJ+>74|Y@ zBrm$G*~D06!;yL{bIWmIDub7@PX9H1I=@pf#dzXEOQ)Q?&i@IAMu;(`D1n|jOfFOCvqA2~ z&)dd3*#FdsT~=i6_=?&+$}4ruW%i;^a*lhb>8Sa!$WC zGcBDij#@(h>8G0lt++`q6*D@^_o;-goHK=cp9tvw!k_jp?C%;Gb5)PcK?!SVv_+7^ z=N$=dM>vIf%k_ndhQwolph0n_679J4+M5LsVx3q5pIONuIGx|MoNi2=W!_xgtTMk| z2;rW8>vuI<6lpL}01=1a+T@!lX0r92Umi@#RbC3bE?maXDh@NlploPYyCkFfp;~9!L?1LC=A@;@&bRPQF#_@&OkJ# z=0urf_JEtgQ?s;a==;>8o{$HQ0?^$>g!04p-8q&0+dxX~WWY0uq(U-g7ljLMrxE)@ zTN;ml6&@9he*fqmcqDn=%-?y@A_zL}*A3XCPEdyuz~~k#9s7&^fQX*i*1_`M^xPwx ze})jmONQ>K&E)|(o01lK>{gZo-_{@MB#qnk4%-kU3p=tyR?{N@WH00L>`-=vr7=CBvad8|EUwdu2n-0iN} zC7!NVr9qmP+mt-DvC;6fhexR0ON<@H*V6bSR9d4%Uz6hbW#TBr`w=B0=7#Jm77OLsw{wI7~z@R$CY9US{-}mRzga2#fK_;hr ziwyzx8{R( zB-3MqvoY6$O-%EL-$jq)$>BFE!WK~I5r5FLKs-VOU z42JH0qj1^TJ28-BoV_xfOHAds#$eze07t)66HZCw)D{{WbP)6rcgg=f?i@!(nd=M>6HMU)xS97GH@hbcNqUqbbMb3D3*=4tj=UrjL75M~*5qTQ8OINxhDLeU$&GauzLT;UOe zu&#jJQYF#pZqhXiI5({|GE22E`nlo8phia7w@bh&hzi8IMqe(cx9Jv1O*7WKJ%0p8 zKIRG@8p_pOd0F>eb~N^`$v-yCldoE+b#re*R`XErOps+t26$~~f$+7xJ04tPq^XIa z+zp>CVwnS6?w4BJ%)A-vSuACYzI^cqFHTz~__7U2X&xOnagdKb zff4q^R2Q3nRbEgOSnm%qj%H0?{bqorWLmIW+JNhML^I)sIJ|4qfP<&u_zWox`p-R! znw33u^ltau($}-Zt@u{;Hl(rY&y=yTG4;W{+x<@(02i5MHB@}~))0O$P%(}zQ?&f| zyUsMlM#6ynRQb+OSeL!l>KkK+VuANzp*LLk@tzmIv!xu9^Ld3olW=A|8;Nv6Ha@4W zAvyhgjd{+sG@Fo0`a+43*Vb2|Uc(Gna7^LdEh)4aL1ux!?od~=%hS)3Te9w@T)aR# zki?Zv%(Jy1&)&GAQcOegdpiU0FPqDXG2nM01YcMn;~)6`O>`()y_~40m8;PipOsvP z=S3T*tDww<;@7>ZOLhwl!LM=;#juMN;@&z;GUic{X4=9KxN3(<(KHzV{>#E(-eegb)pfIW5hd-j=YMlE@p>xf zL}+WfPFewTIbeahF?FMcK4+9A>b$xciG|<4G``YGKj*8qW6%AIA-}$O@Dk%QEDU2N zssD|STkZz?W3%|^nL%zT_MSHBK_0fAF5{TH=z(VR+mn&2Gxoa=daFA=lZy$7JpIX; zb1T`YKL1xvL8qhe%k-dUAPaA%ka}u~*vnz{+gDvc<8zrHbXyRx%yG~iD3$(uAK&^3 z{Geh;%o%&;h7fpG^YdHVg*QQ-ZwnszF|D*eZ?`xHKu4+-iWVF_h%#0 z#}?X^LPhiQ!nFEzwfe~hKj&=IG@%^)Re}~Jc=H!$+WBYWJ{5Y-o&ziO;|WrdGO8_D z)6-nlY2Z1oAGx22>dmBl#j|moGt!G|j7$CtVdyD?i2Uhg;%dgf5@Q8LH=ON7pzfNf z=@AnKV@lKa40$Zi+}*W54lt_y*w%){XCs-YZ8ZO6;(!oO-Kt!21-cB}*Ct$!k+NsF zq+tPp;g^;nL(!}7@ zg_0a3)6l|iwXkEON(7L=Xw#SFh#wm`Add2SHO}t7z9izW=D#iuuQJQA z$Bb!1mM@cA&aZb_MKJ;`Sn`g`lhxkL7XPEGbY^zD{+*wRMFXp!ybAAJ2;D}l+7#Td z?wt%MMl637<)n1=C|oH%h`sbtDEOt(rIpT44q4% z7gtA;r<;9;O?j2pFd;ZV!WmpQd8Jj;Ph)k2j|aSW64IeuNqf@LbpkajTb)Yhi~K|$ z6d(9;&i*L)!iW&}@&4wXfdK8Lus|e1$W+_@Biri6Zr8!4@;CS$vTCG&g_N;Whyc9j z%y?kqFw(!m!i+fLYxC9Eo8t43TL@fKM@Rd) zebS`;-`2f(Gn9e;_S#~pr)Q?t)$;8D))Qr5yAgP5{H7O{6=ZpxE{F)nMUNoAtSiQW zZ+Na=BfFDmQ<1$%7FO1+=Z3}nNhc@pysBJf6OoZ|!^WvXlv^bjjE7W9;D8su30OP` zhb13=Np3qT#goSg4X%_v`;Pj;e#P5ghEEb|_q>{tIB7@EtWqkC7A(w)O_ zw6${PE&lq==C;O)tgay?k7uf9)wj7HME`U%J^1(yYmTm3Ri7F-aegkD=%K9 zWTQdSfaiN}_S~UXHcDqfI6xIQM#KKRH+WTI^m#-|la&{QwQCTBJ_#N}b%`l|ujHe7 zXW}YqDgaR>WinCNc@d`eP^gy+0c@^|L$pkdLPR zDwL51;UWdl+%V*kigTjWTl*K*m7y{4yA`xTyoyzK?=#$W;iwSJ? zuj{&~mY@qsqxDP+4CW?TM6u=Y)X9wZeC?!nA4)W*%DooXz#p!JDOmZ0(gSJVndaYY z|7Ht=7zYB--K_NrE!TTqf+lXEag63`iP)r|M%P?dXso;3zACoC3_kp=TfDas)fe>6 zXy~3Ie@^nXP)&(;7+_#!b9InZ^*{ZZmUxkXZvJ>U?e+FsT0+C8sl8YOl7j1oJg`CW zeT)!DlYqD!DnFQLeZi?GN#RfLmmaY>qeG?M44!#KA3R1v?|su=C3)}25^H>qcJkqE z&;chQYOL63`ZU&{dNeY!{SQjAeLrIZ98#301o-s%ev$Bc>6q-gxWT>Fo$@Nu`ggQu z!6(gwgNHla-yU%1(G<0w8+K8O^p9u>0P9QfKRd=wPW;`Kjc6RY2($3K)%2CU1`AACuEexb&XxcBss63cp<2TskywNXJn?HE?Pc*Qi9{V z6)rN37pXO0drsOrphjuUf}W882+SMni?UYh%0{{HZ*E(i*8esuf~+|G+&rBc1L*_& zN_Oy9hQc$aO27L-hV!g<-COpgb7NR9b`sC zzuAlK|Ht+k^}pm~42bhUC-S4O1$B$Z(A*_&LbSnH2wf$dgnfZsbQmQikix5{DRROALdWj{lC5sNdzvRxl*nmOYbU1{Wsi>x<(?9P zh3}nra%o(I0jh#2DhAM^;WQB0Gk`6kvZ8QD4BDhvu8tIK9#Covz;9-jq1~Z z$p!Hk7lpG)5(}P3O&w5;+_A>A^3$eD=3kgFQ-6Ie+4_My$jOVpl@n;!2RGhJaLTL9 zeH`(-#2f)AyRR5hb{yM*Ydr>JQ<553poFf#bY;PJ&nm7#&Xv&DwZ>YHr_B^W;j0wJhwX+Ox4} zT2oTm0UxW+Y-Pe1u7kEHK@Q)AMq>Fr%<#n6rWXXxtrtOlj^3MnB3$aK5};oCtW_9) zD2@`3a)Am|0^m}AO@o6&S`0qHafFg#R=Cr@dP?l8a04Ag*Xd?I>ebS9!2r`2mZviu zBzrUl2A^)wdZKCqrLs?~L^oZO@MM_YdCTz%(=_#Z<1Ufpbh*SHK)vQRw>nm`JyAgV3s+(nE)v%+6=Kav;k~=v+T#eb zxWg;CM!4u)ax()plwTa4C9QZ50=#&ww#_W$BD0E zUskAB3=Al)mt1*BygfbLmQ%(CVeXS42lnHoD^CY5aW{9sw$f~US`P=?9mJaMK_!&_ zyIM%YFam_W`-PkRo+V~6?SjKH&#*xMjfzc_z&)mRj;sb+8sH9N@yD&uDWIQB^?dB* zQ23E=u4YD@Af$R3kUk^_bGFPWZFeMy8Efb44QJ$|%}u3>4T|OSWX?4Vp)Qqv#Ft)r zVluy6Z^=D^nv@zmZM!bTwMt=n$)xl3y@7W0Rge2^!$@xvI1u-_b^CVWG_{#iSq3{qx-dC97JM-`MpP z1SpKCo`%9FQx-O$WIFl;D?!Tf=)^9Tc+nc*2m;;!9^aGJNW8SW(SSigze;{+Rxkq) zul&%jtcKo}NNTSYNR2^m;}0_Gh#|32r1h!qY)oKAECRs?lRb4Mr1ef}c!awd8I|#` zD;zxqYf9&mbnk}h@N+XOwW1`I2I0t4VVRIshL5@xQ?32a1XryXwtthmTC+TnP1uFx zGGhv}iL4>9=u}?FTvGdMPk;;wBZp>PVK(Wmm73WjcCQ05w?wI-G?N-zsUQQl+$x6Z*XwK+mDe_cm0G8k|rpg1|f3wQ6) z{Jkp{cUj$|e{(j`=Q=R=I*6fPgs-IUzkV#SFam%BPP)pk4^ZGs|FRbv!%^Y)6$knG zQ|JjnXqZY^SXc}k7m462E?0d#_>{UZ*5Z0bl_Cgp(Ki6GEibDF=6@-iZ=MXqtpzG` z{=$UV5()nk@|`tO^k0nI?A^3UPrYX4~AfYN$I{RMyMQ1t|KTz;H%n--=mXvwQ4Yd>zquGg-q1-PtjkjTu zx_bQA;jyB${x%b3CF5a6VwZ0u$XXoIQkNLvFt#`zmg+D~h-W^aX%XhpqIig&x`{g5 zBNH_67JM2*L5hxS>3?^CJgP9 zX#?~|3|q#GZ05G>!M6dE>`-}GTQHEJ>dF_lJ;z{MiDbgz&7vE7FRq^OO5LAYVmKjd za3`QAK)&7&bZWr^ylyaH+bHn}-*wRfCkv62U+enl&W0Xz%*)>b+vnWTc+7mmr{G zg~fj;s=bJSSj#rC$1js6&X>2_*#_ns74=|X)_4d?E43T|QOF^5Z~9Scg_}`+o&#y{ z^Ls1-)(In3=@~mEAt6g|ej0F558Aqy3E3xTL>q{3pcbcs4vz*D6|OefMHa5lclI^P zGfL;NP-W%2n@euCeQC8Jiv9Bi^tQVWtzyYsQ2*u4&dL1{w8!b#Knus(05NjFm?HQW zqu9aeZu;MA^h91L2^LNiNfRz?4(#n!|U=q5J#9p?p7H&FEF zX7S{4qoj`*#veVe{~(1x;cLf51`;{%pm)Q5Yl7YFUI{%W>TDCEmjbZk28~C6RZcO zPsgvH#y@N`-OCYDJhirb)N%g%_u_3vq7H+6!mR_&FhnkC7HjhYmVR?U6}H#xg!Ppq zBN!jQoBTJpyL)V?O|y6|>SbGQC>rWPn5ja~A0#mNHSVEPK%yV}0|0 z1*WR~##c3z$OFfV%LPoZ~DI6Iw0eH!oklUd;SBL zh9!gakveWc_30f`7XS^&1G^IBjrc@hL-@w!!Ab9be)x<4sdP7c%(&D5qI3Cuc*C8N z#|1>uB6FJcZ4VJY=m>K$ZuWQBm@?874-9XCo#e)RLwZ>#4R-ZNHPBd_b0!Sd0*-6ajKL@)jWSN@(>o2_r^rdHwDGB*CHoKth7o zd`F5&BlXEc=A+DKXJw(w?B+}+ETjuIou?r;XXdM%flUu!W#%U$HB1nCc_QVgxklsm zzK*aDX|fdA{7+0!gVBiAmj}JbFNxxAH}5XJw?1Q+@Hg=o4s2)+WDOUNGKDhB3D;;) zzOwirP`LlhDf2*c?kiJcbA1yn=gd3nO7)i4o>rb7+^u2205{zdo1Q*Nf(%2TqeD)c z0bJvS0hO8ee3w4AFRJO45!NA)h z7dMz}Yurb5Zh+KYFkrlsAWrxAD-W-LaICXrK6oy}qUosl-wQI$xD7Jz*8WW<{kX(z zGCF+l`~mNuh?>u?vV6ls|IIMq41Nse_^z`mvu z#7V%=<~OKOyDihA#chw^;ZTKE7Y^dpA#NeA$``jCV?%mk@HYPQFIDh-Zew)(Kh!Cs z!=9h!+b4;zdwi6D73S4`>C?R5e6=~%HZ)dQskVP(r-(T|+TOQlzSt3BkbjL`*;v24 zy07^RxJCFJ8dG-?u#gR6KH?K1dj*;2hcM0^2gJ+@^=V73F})5~Eo}vKU3S-(y$kH} zF8{?wsM={bH3hqs6e$Zsm3OJQrZ?Iq^F|6Zc<>re&ze`o%F!tRK^EK4R6M^U`o)B5 zFLk_VF?&(P1MUZVdiSdxP_gdxcD+uXSyZ=-YnhwmqAgOliZ?XW$z!rs#X}yHa}%S6 zxJHs{Wdd^McnfwqT-{(B|0O)p?M+Mtmb4?trlx6>Tx;Yq`W{(K%tgo2 z!I<-LwJ=Jrh#WmhDk{aokP*0kzFy83u`9sxUkfwKv6a?RY0@sYA`I;Dpc+sSzep;& z_>i)mFg5Nh&7s&K-@kPW%{Q?;@vf9>oWYFAV=RPn>{>q`mQ#Y}n}3K?qm9psHRwv$ zSRvAJZ^8479v7~bxM!>%0$5)<5{xL9>EbuU^b20@5Z|K`*;AWuud=!hIYz@LyI#eI z-wJE}ni3tuUAN6=H6T@LeB;jDi{jM2aUH^t(iroMmtUsY-;J;KvSg zQ0!Gu*)bL>21pW$OLsTkX2K}KhKIvyX(5rkgIWC+FvUaifro&A02x~p$!(W6{qe)! zq~u~Y`Tw%v=i@eXth5btWs{+C8NqP3@Nbo3&w+OW4nUubdY`HPc2 zt(KsG3*v?BQ0dJo`Elp{-1mww)OyLh-7dTF!3DjA+w1GJJIY%!1A`dJU`Ks zLt5qvo%Kr-&;h(U5OrIxolRMbDjXlL63#P}ajGM+laGEjKifB0#urBb;u|k51Mker$xDcN zZeNmqzR(J7(4u%+9ijO6<~cQ*F0RO!Pd`(m#Db@@I>>!=J+A%g8ZRtV7i)ebo)Cyi z@8jF0h$B#=j6Y+}cV+#3yP<-z{RxuH4c5_l@haaw-st7^ z^|V#23Uc>VFdgB5rN!=wxH*yovUMCcuuI2P0t`>cokb4WD zODq|F-19B)=A|mNv)5$J8{5{5wk)V@m$c;uzFG!Epn?N<*QiVx-~GypaOBkZPt#0` zBMQm|_ctg6sRmc&cDWd)MkH7D=oiP0e;_&Nv$nGNaes!Eq_N}kx2t{x*N)RYWTF?L z138eOQ%+T5K)q)|;0vR?b>|Cn;!7&x-{a#Ukh9*Af5*!PgFA6CH1m@!n~g0dqB5*A zCP9Zg==IEf#f!}mEsLAetE9g=zeYEMz6{*BuVLF)Nf}X_H*$hxYR;4}R8-J>S=<OV7o25lS1KQrB+&4o;=Kt+!VIr?Z-m1`kW-@p_6dr z6=T!Ss>2V}!;4)!m6&fC5Wma5ZsSRGqS@`*=`e$6nMpFefr{tno_a2Z^pfeseSIep z1#NCj3EA^aJMRYK^8LzB%RJVxs^ZNV!^SdKrRdZ3r711-Mnm$&4{WH6dbcEdx2*-j zc*c!K;_{yfR>*?V%4xs;R)Kouw1a-NyvAvS4dMCPv8c&X`#Rs{Ws-LwVBw*GJZVQI zQ@hIamX7|;#I)emiZ(7Ko@kF29(Lxxtl&vSI3gByaO&NVAKQYO?T;61-A(<1tCj~8 zyY+wqL{(f7Iq(<%arxlEh5uce5ABAXJ46e=El<~co#Tk7d3yyP!GChmGSnBGmEjmC zkTT>{x8u0FTB7^_pt;D3x1HG6U8it8g=+35gqGvTG=(Bd6P>c%5ZGj!#~ zJ$T0&Wm7D_D!(l6N1u6pdAa^?Y)l_Jt%z-1Uwhk<8PypSOAh!BuO75*T};S`T6)Hx zVfOtq?qLPenmmRuy@l_PHAe#k*m~KA$S!)Lykn}eiQv0V7Zq3UZC=*%P3IRs`5qG& zetG?IXsp5h_ls<%H)Mt3rjCD0@fvtizCl6n*0QX|J2~=1XX?vUO-+l@kF(n}E7;Z0 z$q2h8Cs_A6FLf;J5WN!jGS6I$1lUuVq`o;Ag^s!Jk;?q)7LE;6;Gm_4(ML2XU#eyXaR9AoC0JARD&Ddu@+7Vd!UeH- zSbQ#!ivRu&?M}hS-A)dUZrtguVS4pj71qcRcp3qUg1i?X9?)cvzPEmq@fyBJU)jBZ z=pw>$Kn1TgMh|4?>IjIDQeN3BhSrjzkky@ZS#pC0z?g;qO2nseyv&+*`J)9H*SC2; z1IRPLR;rUT!3f%Cy2r-4yYpHd$vOy`5HO?4!-0mEdA7u9jhV?liDOjm1~5BY^9}^4 zc)nXY(euVb6CllSQQzaU1*3NVG~>_8BR&QU`*jO&d?e@m0y9?S6{Q zfp2-ne)$NpYMINNUH7Nc;29|@rsHy1*8fccxv@e0!wnyTUQxi3X;)a@drJN5r?&{i z9_5tMi+sy5g%#I2dUjQQVd#H-EbOC3!X2kqZjjTx;-USes+0ksb!taze$Yv$$A`cV zzHaTF+)ZFk7HZUx`^ntzFAB%nB&_{qRPQ+-p+Xhk)eM@8ow=juke6dRX4Zq12&hi>k^0{3`x4IxYx);wD-9COYp<{LJW9HN#vd&@-^I~xgog;R)Opdxs%;& zSj^ggFl_Rdk^Fv}c6@tmSVknL;X-lKS&ikv6Nz16()A6*8wSB}VZF&L2x~<=CY2Z3 z0(6`jsSFYf%lq&f}E1kU?VW5SJsy5PU zVtLad@S=Toktr(t79ht3mH?KOSjmK>hnqO*sz>JTc-rXNXy1|(HM>jQW$z#_G`70B z+Qb8OeOzqat(~51G8;0}H!+s1RIIF(yckIsYbn88Pfb|Z<}O`~E1DN1TraF=vfn{W z9~;AhO7u8f4hzK<=N_a{;a|6)0z%}zV-&A0hjB5vCSRJzW@EpnpPXEa=_kmYB+3GJ)bnu)fFz~Y#itB%NGx-zgY;n*>5rN^x^}# z0l-5TkN(HUQqt#q{(dZ^;a1joaWooyg`M7DQyHd^$wQonnB7B-`^Ph zVi)c&U&09zFIiH|hnIO&2i)h#r4HX2ifd~lS54JovTOXwPiOtFq2WaI8Q!W5QoLl4 zf*L%f$aTbYZ;7-7aX_;DH+6PdUxLG=^n(f;ZAmzU$T1^-GD*OUYkLO2M#0gc z;s?Wp9|@)eK@mtbFCU`R%E}f#azx|cNQ?glJMuQMC8bU8c8F2xbnYDnHcqf0UaDBx z=9_V9VrT9<0dYT`dEvPaXX3TpoQEbkNl1)^DH?l{$CGeT*jq9*^o?@2{nz?}X(WUf zLj&zZp5#8Wsx&F3c}28^E^dD!8^Fqsj9tm%XG+hreN&7V`vw8fIcfs z7-&`&DCz(wI4+!&=w5Udlrp6tyJwPB62aVn_{7sqySCb7acaK2dG<^q<9nas#9EZ2 z3T>$nMcv~&{dUkdezTh1`}!^60oi>%-Ttn@e|aeX-+PBa_2(O~sNlM5JKt3DOOH~O z71l{MPG_@g%$1^|@svPUPfw4qPq*j&sP3WnQ+-A_rjozSY)tbi?o%>tmlf(o{THDp zJa*uw`u1%@t!9c|Z6bXXN%7Iouia^0bv`Sh*voAT>{Sc)s@&(qjw`(8=7 zv(zDs>Wl4xFzg1}KIG&CmTL8ii}qCsVjTD7iz0O!yc?7GnFac3kpeIO=vZF8$&lF; zPu^AKkLeipZ`}%Q2*gVKIIEyMpJ?*iW`A*GZ|PrS^VYyv>mf$@JZFZ&AW~N`-<=oL zN6sxXMOW7Zmxu9cNgjk2JS9=9g6jSBjFdQjq)KtGCY%CVC6Sk8IZyK@qs6D*?|+ zC_oKEKPLY+6aJ%}xh32V!HU16Zide1+l7lx5hC-xA{L74M`$z6Oi4Q7L-j%bqv$N- zn(o&y{@+How15nhlI~I(1!;LG0ck{fgfwilNE@_(Aky7EkdW@K0Rqz9u$`TkydL}P zcVF?n&KQfIh`uUoKjfQH&x8H3eo8^#`U)RlnR}#07pC%+@Gt20cao#cESXb|mn}Uq zSHp;16+qN52pPr5$drClR0AnXPUzs*fZLHxYYPq!)fk>K@R@O7^YBuTQem~o;szVz zXFd;UA#cuYb<`eiS24EXM=Hs95$BeTc)jSNU8D{$HaNfzR$v#Wh!E)kq(_=wqJkhK zUIQS;w*df=6H)^(b#{+OE~7Tyh-HzPD9>>g}^Lk{`EH3xt<@%Nlucbwk;Kq{gE0I z(hmkCijusUw+*9=PIR=kNqomd^jQ+@nC9ER%8!S05Q6YGMs8^mDpP3EvyWf& zaZvl$rThi>?5pfOErba%F|jo|?+@ygrd{&jv+1ArjO57_|$PWOkJRLNjOdtBIDixh@y`t^he z*GnKQ04olC^TBRl23@b>gAsxVEX|N0(%q&27=0khbW$sM#^SRe6I+*1bcMqSfGojQBkMBccQ-z(T7J<@=S_tJ-g@eS!FN%`>U&} z9eLP@Ce}Jv2XmfPZGV{4o-NgI4llA6{Y##zua8@c%l*yKry=iGZi|tFPqm&Mia`EJ8yFcoJ^Bvz;T>?8f z!kgFC`brl0XiycpI=P$->22P-fv~JDNR0)NRJ?_Y1XPa<3IPckAi9g+a&47Di`$iN zbAFzYei!C8gf5<18(Yduk!4t7_LQEV-?NUEZ@R!T-*w!4qwX2{`ifGR!2exwrDy!KW|)H@^yg# z!1-2+Vpg|}VftGmR{!^3A$OHB2TliP(*CRp`n(;3U!GO>j!}b58@%Py)I3n);^Awc zDGG+#KL$R&Bo&r`!b9(e9e8G_q?OcQ_+ zQydMw<7)A}A}$8*zSS%#nF76Uv)9)l0Shn13Rg1dlsO6q+;r2lvmV{w&r7j+_+ZXSKFya8_L`ZOU zCKuvq4&X!1A`amP%7H&F^3&)_>Vzc{FcBgQ_s^+QK*jK{giw68$j~?CatD+ZN`BNF zp-2ee^L#r-Bfd1nu`}?C`f;1QdI>qU-5DbJ-{R7aI~dmh`UKCwFqdlJErXXl#nHga z>L20;|3KubMLI6TUa7PFAfK`?QyWjDqi2jwb#NbNV;4gjKUCP|w3TUD=1?Q^_>n)f zf`8QX-1F)S@9h4dZwv;IR|6Py7AD>lvz7b^GJML+gaiJon_om6U3Fn%jJL!={6(NR zN|_H22f@q}(ddi(caVL((&~5#0L+8k5hgj@fIQVNkOD}#nvz+29+x1TTw)BD0NWn7 zHHh!nW#LEYZzd7osEwZ{GmypjcjD;2WRXgAu2n154XD=dM;Nl`7Zb1EMasRXt;L2Z z@=y;ZLm0TYi;7BVzUvixs5#=(abSl+`|dH0-3*L*fO4tZuVkDtZ1vhDkzJfTs zuM$h=!O?qf^OLZ|F}aR@_BDkhmBx?W{xCcp;NoQ3fCA(*F)uA?wB(XYnj$f}AnNzY z0M%dFV^ukl?@rdQA)WBRU~ae%ibP|sF!#sv*k>2|G8{K9z54}r=z)wkRaeJ-N6+ld z5q1yeHE}MA2B|r))Jz-nm`fZ70IC!~2x1DiU}B4SoQ#~3B>5U-3_q6V&VrRZVOPZD z2;S5s=SuGhgUId<5TZIDFJ@J&ouoHpBihZG<;ic~A2!M#g$+OZ-+2CKZEYPCgpQ)} zNzeL^i<^(TXprAeZp3G;^{)S9bZ1^WOE%zw5kTcn?c8YT$65iYWA>~4_xXLy)$CW+w@|Hx(v^lZOC{|Iw>F5qIn zw7o1B$BN*jrbF2ieyn0+!3Rm1wFa?7cCW4}u&XJz2i3DQ34qPXWB<{JcJZvSISZ$H zc{^m(Xpa99i_{uFOVE1uU1nr2b>;NFIq35K4RLMY#i_|Lwe@pUlX=!<;Is=zL~+!V z`vXAmu%1k!R*dEs2(U&cnD!U(!4^r(9w$=<6!-`y#gUq=fkv@u%2F1I@+^LQq^|tJ zpcD`@*rmZih!6b7mR<3V?&8ZYlB?T9q?`D#TG&xiH|(~5sSrOG1tut#2Z~JoSx@e; zm}qiq62U+Ro5;2;FXD#Z_-M_=PFSsMko(2RTfEz>jKA^dS6a7EK=16_xc=c60HV!X zO!wbUv4G=q(;o8DcB70sp1}3q3oEH)&B>0a{dH!@3_`~Z<{!9ou)KG|fR7McMpqZZ z_K1&1?dZ$I4azz2;fOC^%)TE8%@L~=1p~m?3K-bW0ubip=!kT}#|@rQUBDkz7Jl*q z&UbRNFq|8$N8lKwe@4Ipe*`Y7T;f=fG(A8BUfK|3XK8&{9ZySpA*<%R(nsoQUfmkt z>*7Q(BX#7?A^P1IS%3_+j>l$gKaMtVjH(-i`&AMHGT=SRJ8G(M7*Ql-#?_DGxx?ph zolwr1Lv6&dIw9yTx>qZb0g9)KeIgOr%~q`0|0gB(ytMnnYN9IJUfT6B{2q?U!rWqR zpcn{nBtr1nGi^_mRvHC2;^l9+F?ei-BQMnHQzii*66HScvgj)S2)hw(6@Q88$jE5? z$E-uN9`#N00B|y@K?wkx+NKB<71!mkPXg|F8yjAu4-;oe9;LGau)>O-F|hWc0Qp-m z3#g0;0>F|LIjq?m#5H;mR2EbVU9h>YRwj!P+G;FKpv`9NsPY7Go}ZJP|MNj#e6aa$a(S zCgA1p6vhENx=&FxnrsXY?HPq4|K3KUR1ft7HH%N#BJ23PnMaRoP3 z2$4m&p!!5kY`1F|ZZ19N-RD961&~;2{LA(Ru}g}NZe#%IAxOets)_n-Kp@sy6<=R5@9#2ac|_}Ujt{56*z;sdvA%Rn9k zrIsu4D(sS5jmBr(vu%?1!#~g7Oq6LY52v^`7woekLh^6U5$PFPM7ZzF8grIEWPty8Y z@#{5sKq!(1RSuE5yjl^2LKXt)gDER0FcJ+J_Ku&Fn6XnuA7`t9_&`V zE((lu)riX29r1>=;Jezrdxs4n{TGh5devn-JFZx$BHE&}b7{zp1IVGT_U6VqbLZm; z5ZB`;EgS|bbn4@lIP!zftd9$1B+)pTjTd%e-W@A8ke`~0np_dQ|2XFB~qnaYWoW62hd(ON1Wmt&I4d z$9`m`ySaT7`4Rq%>L%8`-2+!SwhD}c)b`FV53DogihKG1c=BYk1OxN-wv)L1+LX=I zEN;P9D!`lUf$E_Bb-Ta+c6yuA06dWAWx&pO{@Su$o3uW=Duhg442w2vIUDXD`sR%q z_Vk;Qa0O$JPp0ez=VL1diU%m$8!3RmR?%6QFcUtTgQQ0YL|mf@JTiF7%k#fbN?U-e z2?va_!3WT2q;>2ODTMPYhk}>2%!2r^{EE8oSVYCr!ontP zXiM}m#*%lf+fzzK0SEDbPxl3oJ@K6zal&B8P*JUK`Sx{h7;AfXH&8>5W7-oN$ViX# zoRtUxCF1aj48W%{l@n}#Ot0~8a&D^$9HR^=@tHKmo^R3-QN;4UJFp+>s*5u^lw9`e z{puz3xnE5rkTPFj8-%6-hTs09Lg;#YVf}IR1TPgPsaO=GstcGGLoDzlUlIfz5$a|Y zn;+ZErUa0HM%j}4_fJ;45HaAhL)^u-$*AZ^~^4#B!UyfXaQvGiCP9qG%7(Lb4cm8|(Umiyi9>8X-Y)#ya zR>DE5594z2@*Z~XZgVz{=U(ebr=C5!dAk=Q@sN1LBA@d5yBnAvCVoCV{usO_X!Ho_ zOxj_^m9I7(U|ADb)3}6VI1)aVnYoplE0cTQufaj}K!+x;>XglVAX%b}ojHKa45_`+ zcC!x8U_7|b+UobYqRpyCz@H4wa4}2T9p43GT=NCsm+8Cl*qcflOC0;B<+A!E_C&EL zdKRN<;sSTH_Vz7Ka%#(dd&TvpMW(hTkw;)e0IGSJ@9b>gNNA04FU|Nc@X}ea{D6fS z_RP4J2J4goUECjS|4o5BDMpBbGFPN${J5j&WQ-bMk3}s9Y$Sh-iADg9g#9)%WoS^;z#0GmcdDTa*;h$sOA( zDc>`2(%ao_F9L-hd~_mSGh1x;XMoH)#N5c*-Enfhzb!7FeqpboKPFN^Pz1I1ZQRQl zo{;$SsEw;G3MIaiyUs5B#mUvzF27~K4y5&t0MvkfD8T>%%a^ZN!R3LgH;<+J)|Xy( zS?nj|<@)IWwpyU~jP^*<4DTR*0#&Tx1eTwoCqTt$iT1-ed>$ZMI}a(_h9;(gHrtd2 z051*OP?W!KlPWBAzW2HA>Fhk5>D|o#)z+7aW}J|N6%5Pp?`m%QNRLp$kCnNdXymSL?60U%A|j2^%!9@19pGvI|q4a_Fm*lf|vO3D3U# z((^*RPkOlP==$^iiR431D(&8hzgCq5#uWa4Z6$@eKf7Td@HggnQ0^M_r zIgTuE=hGuC1WcURpL;;kPDuCRgfQ{lPdt znjWWS-~qtFxJwFUgRIghw3=Po3S$NH#lxwJdBuzs`Fi?TXUVYCQWZJGLjq5X@vdyV3}qWXhanDOm+1qvSZBhN^*z`*9DW% zP@7IzK>r9kMbO-GY`!@IN8!K()haG|Yzzf2FL>)kVgS?we5``ElKNNf&Cu5AbXZUR z#8YEB$M<(q4WH8A-^WTeS3YnMVUST+wTj~cWC>J@0=2kb(UQl$Wb+K4_{+U0u+iPk zfD$nX?}U&6HqT%h+2bmt#aMn84MSTI0RDFpEzBLV<1xFxcEtpKoR2url!eW(`wW%=iP z_17sZj5y#pL@rCNDxXngzSjA7cy6vD`u+Jof|1xS`XY>B(8Y&e`~3X*&n%vz-tWHm zxARZ_MGR1MY;)VkzFJIEEpaKKmAl?6E{9+3YRk1c*xUwPW?_O3Qe0Cm?CCKxRKHb8 zJK&CifcjK@O7!-y_ymK>Zma#uk0ugHLo`l4qjoJt)0NWeR((yDu3_V^ zE9sx&6o7$FOWObVWYc@tj-nQ5e9m@Nf2=T6-a9wQ-&;41@X}qVL}Tu6tE=<#`YQ5_i zDKZ_d!?ZTe~{UC|~V8IF_;d!7B?N$iR4003P%Zqb0dr z-1j)3D6l+Ga83&R7C;0yFTH`Hz`24Tl|9*g_*e;%(aOWGQwZSd~NZ z-KPS#pV=?+>R2RmwDNHqa?C!Fwdo}&A~2U*={!8M*5oQMTc{~QJrr)mU3?KU^WD-J z4=!XvN-zl-w#)Mwx?{Uz+M7nzU>SYpAg3^88rtfLo|#$Mj_ru?qoj_XqgN*%07BJlO} zRGNpTGiy4c-?=qQx=_(!sd5#P=27RQeEcho=RW(0^XmHT6N#bAbc3JRK%J=$_r#Xc)6%k_W1g{|JK zqgB86TFcizt6+|Z5VD4+bN*FvxE){I=M93vsY<%K># z-cS%)W~k#HAphWgC;L9)aPx8ZF#Wdz%R(~ebsgf3Fo9E&+~M&b&N7TsD7f025a>Fd z`f|#Qad7jKG&^T#!rv<{vwv9fXP)t-KnCA9jY5A7UFLVBKW|teFX?e8irT|hPKze1~Tvg z&{!AFFs`|=uH|`FE-UhAMm~d@pZt#$#IF;=j33BEz{JHwBGWJa`eXAj%hBfCh=6&> z>fK-#rhQIKBXj+o$Ct^&OZ9cxPglJu^FLb`jeP{#Uv}ljX%)!)R2hMIx4v=g;nXE& zI~VXOFHpFjASpaOZ*t3_x9(5{Tv(Nu0fNau9)Ya(yK>A;dH&QlSwOy)p*R#Tfj-vd z!qU)fl|>O>IPoW|-oSLLR*;m1H4(s#04Py@zfMZr8!qRT9SV{@bG!z%=&%4pEyPEg z?WjoppGn$ram&+D?~vxaLLoClSu83In1u1H?3~`biD-D z!HRF1msXi)-b%Vms`vJCY_1(o6i1KQ-8RzIyNp1Sbn`+rH*{)MNcfRQvlTLI->PU| z-M)|S$m@(NBW4BUIo3{Fe!mU%djdvr)HBkzdI_hqI(72OQ{MgjNJot%@ln7EWo9+( zl-Yy*)(M8HKn$yp<#)j^KOU#jzhK1%!6e_)#5{%MzcBh2{WkkM_l!=+&)chya`kA} z&rYD!z$`}G09-!I!BX>Lhjs*{9lp{3t=UlgLK2t7jGau*XSO!L%E+YrqOPCn1@Adz zS0H(8j}j6S>G2c)R*U@g0imnoU?nRsE%7e@U1KndBsp+%jsnl3#t~x7fF$|fWF?T+`Uk&;Ar>39 zkNtBq60-pES^hHpF|n$FMabcU#}^}w_%>cykjrQ+SE*T9Hw-Du-(A|`dr>09eYNLz z_#9-m{5!1<)Q~T+9~HL3d|5gB;G=>+0kVp``)qEy->V{ z>x~-(sPL5qUw&K&snIKZc;!g1ysj%*;-x&rQvgLBK}|tW4&Cqp!}8vz>z^bp?(B&G z)CZNvLLjYT@?ypXg={q+XSnxJnIXA;r1|KxQilXz}$h3OFfH z26R|muo0H5UoI{V!UONWsM^nChi}3L6!>eEmtDfW2c5ItdR(O7LCLS!ai4xET$+5m z=PY!7k;eZ?3P5gL_`Gpm8^#)M?S30sEAZR>dN@FQ|2bu+(eHatbk7Py$dgnA{-#i5&V*E-D_*MugheHh1P5Tv5oUd5*weODq$=-ao%v9R6#{-|VmVY^>)} zmRM$-)>Y=$jV8xJh}d9_=ik!q)m6*EH6p$mwY;8HyCX&VGzFj{*#~H$ioG}w27gpF zHfDo$2$9JNbyt;W$51J(nH|C^zwRm1v1oKVRmLRbg^61h(<%WElfD`$SFn8y$LQfk zWM5v^xxLYnSty6 z$M}fpAJ6B5N>l0$pDZb98k8^%Cl+VC9pXHRX246|ZG^3CA4fIwGm>65EWdQ-8gg~T z7J#IIGHUF8;HBe)8}!bTnt$@Bsgk+3aOVI>S0HB5KfdV31E#U?Gy|#7-dZtJ*}6I! z=}nNlAIP9F(IS!wjPofPlAfL(F(D1?;hjmI#m{8<XLgSUvKa}XdU z*Iq;_%%c5-_xfZ;0dW~L0Fct-7}TZ>T|w;Sr`2m2Vs+?jc>mh4T?6&=)?Jf0uK}SS zZ|23bAC7($efM(o%4e~w#CgIkikpqUWM0^#O{iyxlBOeqAfxLd1g~$SSPu2 zy>VV{Hzh?#t zr}p;tmPTkP+l)LI0hSL5{Mwpk%X~9hRc}1}4n;6eoGm@>P-CjvBQHPI67bWOhU5hvm}$Y+Ndh;kBq1*8i!v+|a%w}%oYQ^!%_#x4!Aa+RN) zf3$1!8uUjV&(?~=3+d`e++o6Jaj{Aey(Md;rJ?u(8%wnkg2v(ev1y&8!d zZZ2#GFWn%ZcZ?(r!czVl_qPGaLw-Aq5nANgv)YWI-ltf2$I{$Fvq_Q*Kdrg}yj9w! zZp+d3t6SrBIPTZ`>n?Q8TPB=cIrG=2#l#1qErJ4j(tk?JwTBPH0ECvj7efHTlc~VK z7IamTKDklUGj=nETPMKJK{?$ODi(AyJI?@?4{lYzdP4xXczi&x5swmpnJIAryhC9w zQ?mSUMgmj~mmiI!2Xq14msx^GxNM(&P1zQC%5>9kn&qz$fuC>%|EBk%mN(S=y zP5_1P=~JcxtHF<>|9~%ee0XlGXLdw`q|xDHWymS&P|v?^>yVYygFN&Ev%L&7x}MA$ zVOHh>R<`9(k&%OkF09|{00lnT5z+h)+QB0RKaLf$*gmmqCX|)tgXPW+OVPiRta_?JHH?42@4uVwO0gud#{#G%IWRTZzlNcE8JQ$H?LX;|$6< zXtqOb3VBlm&wsmf%5UlZiofw(GBe|+>;3t`TxG)Ofybi;f=8xkCqDhIkn|U&0tJ{4 zGLH_QQ~kFZyP}k|Mjxw=-)NhSnGA>n5E>m(8E#x`L4<hi1O)@Jg@qiLW_Xo6%h$jlYE>^w&5rm5+3>ld&hzpM52`hy`%bfMi<9+ykj?JpK z1WB+!_L;MI)k!vW^b6Yl#3RPodl+%zS;etkvV3(xPu|+wFswnq#m&#v%Do0IIl{Jp zA5rXrV5(f)fRA7_0l)}Gfk24bQB{bn|EZJY2U09K>IE19ehFmBtba!Y5>7qm-%GIZ zn0|nRd^r~0Jw>JiOnjeM`0xFF!%uJ~s?;ivo{D|nzEn5HSN`nU>AO|>&@ZO{*4{E% z{-)jg8YjHwS?aZZY1nqX;}0O^+HNnz><)dx+R>{t4ZCp6E%e&@PH#nG58d8(Dwc6C z&Ev0qxs3;#JwV+p9?_(0r1Oo>R70k=)*6d#RgxdxEn}> zcxjHh_rsrGeEm6{|BiFyU2u!szz5tmZ3nm%9u~vX+8S~EOlJ1s`mw0M{KUDneaNh= znFXd17Yp#}LQ-Q6vg|-gH;YB8(>XwwO#%>vkGnBa zWcM2c{tO&^0bQV@;`(#~yWu>^2KpCO<@EsZ)LmcaXig-*8%Pxg0KvxT2|95>w|_JQ z1@(gj8;sd?T=<(}8*)TndUe5=;Pbn-X{((`E(P^ml z!BPipw$N{Yq)kTgz3rl;eF~lzqMcf=QHELGXnlHTSLt;^p~1R6Yn>>CUZZKIs`A;S zU&1360J)!ir%Z7u6B2T{{DdE=VY`0N^BT7rK!I)4$)DXuW6c$7hJ*%y9gCR(Oiy!{ z{$g>>iIrMZ{(gvA|?}1LeB`3emwOwtwPH&=5CKhJO(QzWbuY=p$Z?6vJ0xvJ7 zv{&xe<$Q9{u_Kt~mkpSeVMffMmUHH_GwqeoCHNIM(+pc$UWz@JOV*w1W;K`||8aKR zq7P?Yd=CJNNU>O2&?t5PxAOfdduG4PkaL(Qjhb z_rPKmo?ZTslcrt`Yx{~hS4k4=KdmGiQ*k7^(HPVBrVQo!+4Y{45Kf8P0`AEg;5X2B zSbk@Cb~G;4W?#A?xd1;S`0x8{VfxX!fU(JpZ%p78|7WS)Cyau=bJ|1)S4fu1C!BzV zzk|c4U0jyD*3&m&U=nyGj}Tz~WgmX0Jy&)HxC~-?&`~rh!61-rWmf8;%r*1i4v9X7rWV8~A5DPUf z{G&+eA*vP+T5hL5pw`OcL7E4rU&5YCjQsk9HUK54cR_F8l{UUwVEdMqr#prq_z>Ll zJUb>j7<2{Oc9Xk`nIr+;3x{4irVpl(eM<-A!>1@g2?u6)j*j26-@_)K(%#^QhT3^g zvtwI1Ov>0&82x)_%NdWQ*Wbb^b48$_awIARyK+;QeffvOlSSTPN3F8Tc$}kkwGwyt zx0Y{W10Y``A-=#k3b5jYG08aEiA&M@isb~c>W<4?bY&yAjSK(5Ig?!SUSsE-jU z(ccZQ7O9q5D*McBHSssx1*1I4{~WrlS6@2B*x#k9p0pf zVB*`mI8YWUs0(mQHM1*6PY)c-ET~p?MI#Y z$pq$W8K7SHJ6ur-hqwsLA5b>IvfLIVLp1Lr1VLg{CRPP(aezB6v7tw|H-MOtlsLNa zeQ1t%KtZwdL|h1p_Fr&V#wqbf1{S{1(DpSTOX6=e?pANq!|Rp~qW<8O29SW5t}_)E z91Mthmr^UsQ>BQ@%d@TN1*_)1=6)6+&6$H^de@*=1i}DlQ4s&UbjZ#}G7q>BF^QKe zAK+VsF%noGNDutf0qKnq>aLHJ?ubOR{@R-O5tag2U|&!67~qZe zhwn(1&jA;_sfB4u+fywZSOCH7+x?TrS$F~09BZTapV8wduOL7MF;WF|amn+D5A{_| zdEC-b4vcazK%~d#zE-d!0+8yMS8>39jcQV|tyyWW=DJXq_16k@h|NAPhlA6{%>BFVZ?>d4_WQ>0>zP(oIj!#oO z9q_)Om3Ee3e#!^_QVRg|o4{z^`y^#Bv!o7Q=r=VnJR+8A_X7*dC6^cwTW?%%)6UX< zrIym{1oh#kIjg=n!MA>br663EucK_F??1i8R~b|MatiI1dwvVUQ7y^Tq9#9?TMP|* z6BVfcNIa{;@RtZ|=nQ|?zxwBF`-4L)kl=yB_@TtJ=E^ilw>Td_c%o|l%+=H|KQnRHkIRGb5=QxhH?3~M!h z1PD?mj-}rt{4&C+jO9frKULq+t{_w$ph|QW&XM(3Daa`k9uHu4UfRB?HU=%%2ex28 zDr$FUz+w7J^x&M74q51Kc;Rfn)Z)zgmHY9OFtbsnHswe1o9m3j4CkJ6-^BZ zzTM5Qh-f9WK2D#7vj?KP!oJgyM=GbFY!OOl6g|pD4ia{WykECh10ar;T zNHBI!W(u4HsS!RP_ypf$tisWwC4NaFb3<72j-!w8u4qd^>?2KtWu0-k;cLL{VN6Gx zynO9X9fA*L(cK=^4p-lZ%TWEPNh6WYPail=NcX z=-J7^SiGx&IWGk0QS3UpocPv1sL{3mx>#Sdc1s+ip=}I>IhXwt;dP9@bZWjXSBm zN)tir`L;cO)=0{aH1I8Dl7k?ZaR0mIjPJ5+nseZri;S$bcm6Uy$zJ_KiEOheN6jz* zD{>#^wFz=zi_^d4R+HoGpJ;20h=G)1EdP4p1#4D(m}kadvS?=`o;6 z#5>|hN^Y?m>t*R{eNToDxNyDSm_rMgQcxTO-Gu^1k16HDsT3M?Uo1zwQZ?TYCTQvf z;q<1bM<4NkNKE&{3t+LTNnJgZ+BOVA1x)Tc%aw}9`MA!Y5df1li^i}5+ z+me%`gQ5^Z2vq&j_jNw_Y3|jjZ9a;)**b(-uXFQcXW#Ja06g$g1d6pMTS{dw)%cM7z)5_t27!b z9{Fywf{^wk6y>+or{8LQ+}bnDbiC^7>+86irN3F7;dT88Nl~B!K!TKBD!J4fo~Z-7 z)zGCNQ?s`FiF!$fzXKB!cfSqqSkdW%NazUkrrq(1;3_j-+quo{;`9QuDu)K4yCRRa z&JAb!=X%9$d2udm5}%58in{ba)gA@|NBs_pS%GI$q^0nn+u>N;ru#3p?4(2%B{scg zETxlP!T9pLNv5ya*HgJ^T;3U~9&A(E%UCY{0~__uTlI&DxzY{bTniKaeW9XmdpOSxv%SNkVdeWR9pD=s(?ESorVMaZy8X@~rvA5s0jjOI~I>T7MOl zzj{$hHcPjm`L2?@n2oJ9U960e9z>B$kCl}`$XD-I86>Dvx&Q||exv{aF#IyNh`o)H z0Dy`A3gz@#DCC!XWFFIryy@?w?WCptgq4l%JuZjDGU< zy84Vv5vB|knvgnP;q^X&?$Ebv-|Bwc|912l`}O*U>uBPfOtQ7oQ-i$e`wzB9rTw5v z5P-^i=md3dmcioY$b#PYZJ*^tnS+qAdA21L%;lN2nP#nSEmv3AzDgkZmV~YR$cURi29Re^FiWbN z|I{$>=-rF7N|j{JeAETE;o1~EUU5^EIDnkQv%AZNpp?_rK?$dQg2Y@cqaHteTiz@# z9td!4lemHORuf6CB$e8B+%UBQ%kayBy62dB*EaUkj-VNWzxN;zHbCF|QnuXkFdz;S zzZi)*nEBBj-~OBv)Nr-ZTi$k$ZfTo%yc_OD0sR|d?ceM?tm$>#tLGbWb``+(48sWWCCpT;IBtmTL z$Hw~V61BW|clY%H5F`hZU)(HS*nv3(-)+4MUSr=*()l#BM{6txw%~C+kSRNUfT%6~ z(hoMo8Px$eBV#v^Y{c>mK{@LDixg#Rze3O)qHcJQQ6eUS5IiMsZ*o=e2BP4VXg0O( zKN?BK-=Wt=z1wqYU*nF?^e&WN?WiB>cz4>n8Z}Do|u| z07Rm}VC*acinV2%)Tf~`uR&P#5GLG|jW5=!MK`@&;i46u-qrBq#UihghQ=K&&;+YB z);4The7ExH8A@LzB`;6#<+MDDRobsa+=gBn$cmCeLPlx_XVru(B^kLL_NwpCHv|EF zY}~-i=a>e-by@H3H`Xl?Ee9$XS=PxaB#%@^KbP@ox4|z%%UzwmUI#m$7JBjH$#=pR zYu~=9&b!!}XiU_i#HqP<;_z{BSs8Ksvy3F*0;C4|c|SM2^t)S&#=q_ke=?K>zu8@p zyK7+myPASInzC1oe|F*vN1a*7$?OsmYLTJ`b+g z9~gx`WO4(w`ONVkkf+%QOCRWSh#nvhlMMDnq?k4pPTh!D9#ZVUp;|ZKu!S@c#V>-3 zi~}he@~;kj zi>~%qcmr?Ze?7R;F4xE-CyAuInOfW79=nZ<47!3CL>;4*SR`}Bt=?m>M_cbLQgYAI zHt%p8#m)@yNdDLSfXL||&hi7qmNpfZxd~zr0(kf5z~5gZ1+%9~0M(vu*++3AL{{{~ zF5XI!}0lR`)7@899JY=8uXO|6%^yN>}umjrqzSDh9I;Sn{6 zvwo6r6G@Ig>tyXHKmA5{|)|EK`s!w#ro19kzH65N2!xI<)QDE}tIZzo>r zPq%E;_9d2!WW#5V9EfwrF%tZUWQcsrCw-7#%frVdX|X3xSeF!6dCCxu40hEj` z0nR37?_q{~Hnyv9;?z$DaM_D70iSE40SjVk#CvJOrVq|(@t8aiAXJ__BF*hF*1Ek2E2`W8MG=w-+z% zu1&a`=FMoEZ;P+f%#He~`Tfzwc`r1J`7gB1`EH}V(x1QP((woUqAXt98gmz!dx_bJ zY!9lwHF6T^!|}9}mH4vM@LvXV>hOqwwq_XwxPZ-dyBPSdM`NzLW3grC^ZsSKtiY2w zzARm4XXYFgP?!{$pn97&cEc`AL>@8W0gaVQ(uiDBj8-cD zGgAHp;OIChM{H&-8JSC}4h>ynalwv8&3e>B>bGsXqH7D`ijDb z$%s2U-dMV$M&g6%OPsfySDZ9M-=rMz2HB*MV(2JIT*Sf0`GJB*k|`SSX|2Z2us79{ z2bT+ZX=JzrGAdsMD#?3oeoh<6`OxCFh<+UUO$|bf(IZO{vnIu0z(Pf#t2Cl{gF}uS zwcP5#*jcvnWOvUFH3)u~a0~{1VGFQ4jOrc$#dBp~Fqb$%2;1qng3z<|9D@NFYCL)U5Ti4mPKoA8SP z%hyU?#dbz7Zatez9e**%4R7MdMfU!L{Fwi;V2)J*i)Wx&45v^EAm0GpJdrKicBv#h zeR=LE+ZykMJ}!=j4+INn=VAyDhq9@1K7M19k8>Y({OO#WTwXRSV&m5Qz1?0fBh?rH z-?gc)Gx-{X?xr&j7=2ou>C|I=Zd1~KaZ!^?jOrb%^Zc3jU7b78T<>{iaBusL%iZUj zv%pZ{2(L@>_-i_<@u3%*#%e?>{O@Sovo`z&m^_6^_9w}ANO=(y7yUT3?1=V z?$QN_#&%~c8U8Jfjp(%bNK?;~hX*7TFw-FMqFVP_;kM+&ce|ZMADnO59ubQ)o%Gxn zH|v(1_2#xCd@s9yzjy}b_!hEa))|y2lo@_{oxAdnaQv&F#;~q#&@lA40JzZs(c?2d z3J%;FUXZo30s$4RZG*i4EED#yy`mkV0I(o5A6|yrOFWGI(d(w$)SD+HS%K9}gX3EP zTwgj6aCy0Ly^sM&(%~%1wLePnIKtX00}A?D0li_B@vi!NjDIyejxbme8sUjkL=VaP z#|)KTHhqpPvER0M9nAv6)!*SVJm^w+VR*)s!1P-E8S@npXZ_cfLnmon%DupXqP;cg zpbF7H8diXfrFK{G1?HsSD$t*>dXczOa>j@LcvER3EeDPLztTn0@ENAC6Ny)l-@<{L)v@??&vT| z1G{Mj03wg|fRW6~oIQo@`Lpq`|Adv)g1-QW&Z^1%R&Q5tXK#4B>9BEreYw`jH%xcs zK2NpX%-I#KhCSK@iS(X~_S-5jF> znw*EzC*);sueUDC_l)@NL?d9e&WOd{i^5{DoBHl7^&_xTE35u7l#_I}m%RV}&U z>HjD?%cv;3HVU5!x(>C~i^d;PFczH<*oNOtt-bR0AB>ZMtt%vior#A~R`F2Q|*=pq-JcJ0i zTcG%e;D0fgXTv(Ca=f@GeMk*jO(7@16mwDVxEvZ3s<|?%c<~ae8z0aD)%)(I(p@lM zwWeS9rQsw)6xQw&WLWqWg7o}UL6z);gK6)IkO#-hKWT4T$3lzoQ_jd5v$w zV$;4!3+B2mHe#{FJDGAyrQ#Kx2Zwy)Ap?$4M8D?wy&B#{Z1&8s0wmE=K(v^DwDy=V zauMP}C)@*&fF{SFc-f3oOxZmC6({QbAOMe80R14gxkWJ^D=CUz&6v@jDgN)n_@V^) zX66M+v)R&VobnInFSsab%TsVEV!PQa=k_cc8djETA->WtBT+lkw$K0KjY6!405Ho} zlWL($Z9hd90CSl=_AV^np0LXCpjKo3t>w2~m){06q1H76-aZ)pTa(k(aA07}(&!&}8$)eVa`LjV3hQG8&98hGLv zXSw{lHGoL;{9JKGSTs80$K^$`KQ=MsN88PE_`$xvzBxk~(;B`3^FqCct%%X*eDuqP zw?1#*x-PYu%@^$@=|TJhI+-6Z-1BUxF?Z3-c5!54stwyc3)BU4v-Zpm@D4i=P-FR` zQ)*i?|Mb*??M*->7ep{cArHy@6lD`7wrT)~Z=YAeA*3fS*cSghzU682vt~a0zL`%r zql*19F8hG|wsDfTI`I0r$P2UlSu38koAhUJfW71?7G%!TEjh&aC!__JcP>sRgtn{ zI=WU<*;g1^;I|e#0h0Jeup=Ln?(4Ynv%W`JPZYZ|I|KH-W^aV61P2wo@^^*n>k2M(4+|F)mSf3ePPI zX0%xXZ|(8uvaDY$l3IliZqv8}Hng@1nZES?U3+ZFUK51P%q;X~EzSIm)PNWm z9CCGDpqu{&FUC>%%={?yLD^7g*N?hXN*OzT~;vT0yi-8Yg zTNy0~lTOPnNn4i6)SvncS+@JP-Rxg6FWuf8uh48myT+yZ%e5-F;#2XPV0E zsxzp)|7(wlY0A;8sp-YV6RV)3?!_%#!6+xltD1Mpg7@Zc3c#+ZOG23AR)}XcY)M=#^3xCo~`u!8|r8)1Ni1KsKKlZ#0kOS4{IIi-)B z#Z(f+4Pj|xA@mL&cWe~6i(2+>r~v2JKC=!vt6%qX|m7AYQm%P zEZFdo1eAxbLBNJIIHvs?DZ-09eIZMLRFOXKub$kx{k>A?TnfF25i{l7TakBw#sA5)@!oFQo&rZ8~7`&_rz znakas2Z8QNXTDMKyXx5Oy{%0J($7o_rY0uf!1`gtPAeI3pRoFD?RRe;m*M$*oovx@ zidgC@r(kTK4*OSkcp=MELiA}-wzf)ksrmszg%2qVN*QoTq4%s^Jx4z*1#`Bn?keDf z5L|Q~C=lhh2`U8JuQAXha{j8N>6p=fTnWydP7COdGj{nKVpoGF_JvqMgAGQGhMsCK z@FM(wh(=&+KVjZl#OpU0@x*h8Ir5Z^C~ot8Z(1;Oa_~I4?4yW?MW2jds0knS2Y(!v zuV;0VSYYTb@je@@zUQPkfxGF*gJK9qV>H*5p3eTW>_VWF#&e0*+i%-cWXX++H$Xi1 zG0lOlqvF(RZOD979OfasR9u>~x&gJfmn9z#R;STa&9`T6L6)lTiEoeV4%#GBZ?`5f zt@!cEu-osB@Y~h-C;9m@PfmKR-0lwP#5#8B(Y)XK=7tM(*+`G`4ts$rioWi3kN;pwvH&R+SAr}dR6o*)p}Qgu>jcqfmjItG zHgPYOp2<<^7P7vkw8wI9{p4JwUvg?DZ=&(2Q%~h!`})jVtJBY8xx`e7MBrWbmyJK( z{lJ2d8(kQO^^dVNDqC@|HJ12U6yc+HXzC)8?M8{0P``zzkz|2OB7gBSObRKZTw0gP z>B51>Y@i|FGyfaiMG##ONe>03A9#>^r+bVG+YL&N!5B8d7$9(yQ>J*t?1cO13H)k^ z%Y(R~ELl+STy0&b7DfstR56z`$}h?cA4!MQfEjE1z_MaGZVphU)?>tkV{@Y>#Qv@C zhx>}(d3LsID<%hQIMLw*or+Vlu;9VIXk>2E?$IFR-FK#heFGd5yK{u2Ql*?31-bLi zVu-n!Hi&DKa6!R?LVBx6f36lY@~-M4sTEE^T}6fe%A4_rJgdwF*RhbZnzq4h2@^0g zenI|ksgiuAOmGmdpXq4SJINJK{0DDgVto+l6eo!KIhYna4iKn;Nm!j6b8o#!PnwlY zIg<8%-Y*%geBhg8C@sqQvgE|cZK*l(qAHhq`orH(bKjp}B*D02W39&JNRlI4)-96m z+Ml6Y4Qd!S5$X|voimPUtgc?@!4hZa)<5nI(b7HPw8Gh!BFn1iv!O+clL+c*j;ijK zQxoT3zl;M;r#!5DjBIuI9{qyh5Ikr-UnXHbPHCE!yIvKed(~Oc{yFIUws;u-RfF_r z!S~ufO-BzfUpxw$j>>4g+u+8?g?4;?f5Q(Xr^HN^?g=fQ7R&jc<7Bqa2i>}ey}lai zxlHfxKi!Ixou8i{kfEx{J$@duI69ivxoD25+=6uD1s_mg#S7%{^-n$fFSU(fEmbws z^T$`ND3>GNG`t;=B#nHr`7egkkOqlBFE>}5@%#`+S1qv_HLv?-k5n&O{51Cx z4rMn4bdUtru)=se?E8kmzlMYwzPow_#lHG0l|p#)=2g@x!3LYO?!eBDrxOugc~~f7 z6UJb}M(Pi$-aKae&VOkDq*W8ycnKEuR!?ppIW^HiF}PTL!lx5_>vU)A6^MHu5xhP4 zoTgn73rW!)0Y!hg{c1NlTnM?D%>p+mEU1d-LxsQ!BE1P?{b)D7~P4*v5S`_BENq|N@TGb zyxYSV=M!%lTA$co)JLV&rvz-I24hdw>j@L$BiUXPF`QqzM*|)(6;(yuj)VI(3M@5+ zHz}+hKUQ#JD!)*j5c*}J{9(LtDKC%^nniHvsn3M~Yq$WPAb0Flj6McdTXxmhICear z9Uk`fY+A2b6yZTOsM*{B`pp=)(;%@cdNqQae{YR;YM&yc}5T;*Y~mIXYiTC;p|tF5;#AF;C+x zdh~U@gN!`nWtz`lD1QD2-@ni2aawJK)6%T4?_tnuq1^CLgo%4ZB*i0AMDZkw#FJ-s6 z^ObRtHdCByCmDBbs_RRseBAmq$ZrSK8|nS%p-77d^Jv~EU($!gO9 z3htr;NoNiYWGml!)X?ESjdqe9cN~$!KIKWOQU4ev zEaafsy0ZfvM5zBY5QrAYNCcYS!eGn4Yn8IIhu{EaKOH6b@va{N8+^^4B|5yXv6c_bc1OeDyfV^i(R{%OppzCo1WJ*wPz>c`f!Hw#d>G1(QU#P#Az zGU7|9e&`4X6KWlND8Vu7`wzRhAflqLjme5X*Rt97=VO{Tzy@S6(rTCSf6*kDL4Oz z>7;{nkatvA6cj|H;+BvV9B%%t_XKIi44Q`PV)a7E@0DGu0zr3K?>-YNEk~=A7a37P zQJHBhY@PBP)YkGU?=qO&P~$>D>=}9{9^tQTs%XDpeZ-|mBDuWr-o5UT`lM87xFx+| z*1EbphNS(l?eyFoJxh)o{|QtwFH~g_!vhoG&9sp(MlRweA?C7?LoYtgmfE&x9Y3Z6 zq>9v;O5#!p{J##G-<Sx0~mCX9+sHxZKj-r$-`Bd6`37%-^NHToioyJ^UvaZ zXPE3`Q#6H{lc00X{pYs}>$1afO8oN%y1lQMB{0cc&$WnC7@tO&LnHszlE+SsYbQ z|KgSM*NOc<)6eQNP?Hm3uCHn{{m2fzJ( zg1sQ0_{==hf#SiXm_mAJb!@SRC^rMWMqSbuPXP26=T+VzzxHH|>^K0h^y^y{^Px z6zpAOIUZUQBGX7kf+jvY8bZkdrX6DdeLC5uW3Q|4*XNyc%2JV@X7)J1N6w#r-=M@^ zprrTTsUGgNm;RS@B>EN+P055YpIbuWxP107ZC(@<3;dS}=6esAAq}DLqAabJ z#^=%&wp@Y8E13~2`;Rr;^928q!*_A8+=)P@zRF@SP0YVorD{TaOf1B2)PbP^4`o>C zqlEcp0t|;c+u!oEmGv;-7{?!39IDIX<%`Q4 zoAwfqZCRBC{OkPP7I3<)n01L^AFkZmtZoOg+ipAqZmviwtD5d<`Q?4$GE?RC_~>u< z+^#E$a*pF?eWffs{FhB&qanYx+nkQJHj{t66W@NB_$-3;eQNdG$9fT0OJ7~p#gsYJ ztZ%C8E~>o1%f(o3_5}&PwUf`AR^JusfZNk24Nl+Jn|!`_$l4A5P;(M|r_4@&pLa_UOy`z1IrR9)v-i-+}X@ICUW)h#O*plH1E3Qx4KWl zNiwW58<-Z_Wkb1Y0LiuHTJDnq!#PtpwTh@#vdgE~n8BQvJ*k(yYja$?!gLfBjygX0 zL%kQTiawV+;b6KDoA{@UgVY>S3tiNIIV+vGv(Mi7>dhsMI6jw(;w-=~n#c)k?}woK zcF-D_;nK+ThS2%@eV%b13N%2sm&j4W*8ma=ziUveXaWtr!^X_<%cHk&__48>fnlt% zqqxOqQjE^A)cDjLWcD)0$FmYk2h2SML&EiBO?%oiN9|Tojh5P+*F~$tGo;uHX@@7<0*OYU z6gMslQxdK$fk1#mF{Tt}RZvp#tK((XGrS;-ufQwBtP?xE>ZxNCMXRnMBOEK!s{L2CE`Raw_x^0%>c6WAVH0J!Jpdkh9kD8WWR;R*l=^r)R`ZcgCo#t`Bdnez)D=)b0h( z-)-+4>@O`2Y&bQ=wYDb8mBi24>iInueFz&1;`qiOd|{*TJJi^!W#52z$L>)9m#1nW zgakr3$z|LVOxj;R^;LWx8TcpkUC{DS&#}LQ=Yop2kjW;2#ZcgM2T8=Rqs}p zZRLy6nxB|M=%3B=y+2%gN)42J!4S;a1EvV#?O;dGfio+5O5?pzx=PH8eOGKJk;(0} z`sMDilh5WB>DfAN&!nH7>ZVZzFSq&LUiU|ZXUO`*b9LbP>Q|EilN(=a$Io1~M*N@t z{re~3pu|QhWqVvvQ4vTAa`njo>7MqW+Ye@RL!9E1)CaY=Lacqy3;sY#rz~6jP6Hb} zD>60hb0dSv@StG8y@9bon;3Lq*>`o*RkdatR>kS*k=Lc>NcFj>{0xlt=VY0Ak9<{3 z^GRvPx(rN6|(? zY{7K|hDsD^qkeANWXyq|DNObljgReNNt`Rq^y#TvdkvnXu9%=$&**<~;o<#=ZkZQf1$!Pr6cnU!J{sG!urNgx>`h&m2P^kVxd7W&+fr9;ybBVIT z2y5YwJRIax-CRw|ueUIZeF+7nqWTEC^5M?sqFt{y3$_a~qr4mJ6V<1;PsNko#os&+ zejB_NNXK`z`q~-mgYSKFl{{y%{-`L~@1zh?r!5C&HL6$tmB47!1T?MqG$a8B7Pj-X zSSbX2>Q{f()?ti)d#jMLgJt@ANsxFoB7XK?zTNK)MM%P1b4v6!UW4~CiD2eTgWsJa z&*w;A53bBmB`ne;-9^}Bkz2EA&HaGVIt-m5cXu%^C-gba5|bbx+g4lm!GOc*r1-`Z zqi892m**%l{N&)`&*l0s#hL4wM%<4j){^OPsWCej(*$v+2-W0v;vBN_*m~ol?iO=3 zty<%S!9~->-4}+`G*LXnCzdcyf8L3srTL(Qej?fsPbt_A~724 z*Yw8>VxjjAU;4_(OaJ%vZo~Ac#_t{1fX)B#b;#1d#!`gUEjP(c+Ed`yR)J zsXmaFl>t1~!>9D-=hWb!dh*=;IGjSh$v~M)Pt~)8cc-HZLbvGNr~k% zhnzpp`(nwcwpPkz_KN)$9aTO%xYPr4embk}Uk_)lV#xASqy(Lz(eGclWZCSs=lu?o zM^N~F-Ro=+aaF?0rBaXksi5iM4&0y8)LIhBy}j5?`Hw2Dx#T=NKX+A_|0IqxKvsku zSNpn0Gro}T%YmmXYQI0O{`H^5Qj5xnTV5U{ZBjj;8la));#(=|HELJt?!Hkh%?tG3 z`{9Q1g*g-JIcmMR`uCgP-~GRg0vEy5e2rUm(}x|W%Sx5fJC!^^dTQ zgD!H;*hvxNw@=STYrMAi7a2)`85$aq+l8^Uy=>Q_e`c@T_wFd%q(_#5Ab(u=--qx) z_yX0h3bPJepihD8` z9}UI053^W!L9po~MLuD|Kx%NL>smn7IsPXO`LyC0&3%Z*3 zp`V&tv6E_RDm=R4yA@S^W^;Gg5vKJ)(yebgnm5z3)VTe2=k!uoI=_9fS0K-$!lj2H z&M7X=Y-h_$hE^iKGBO!5h*GF4e>TTvJ1aPv9Q4ZD(aq5*(20BjxvQo%e({uw67k~T z=dob@UhZvu=>-%D(jW-Uk!mbT1Y$o90z{T{Ik#GwhBy+a%7yah(lP#i&e)~KXJ@jP zn^kf9{?$jbF-Ad331#-VrNc*mAKcK|0?b2vc`I5Ezti|(uP%V1hT#c`^M6Ujan2HF zkRtH$X}cHb)Vfl+@ zM~q5Oqg%kccY%xtDMA7&mw%9*iPJ?warG6TNR51q%Q9)aJ+^5$R~FYDPGZTeM+^?w zKmDT~T&bjwi)0L*QF!(*eD0Cpi0`qohazeuWI^GRUGg?egao}44Vn#@Y&x4SI4Mj^g$RMKx0>R&_fP3W&Ej{mwB|mEf0lL_&Y3AoVy#%8vA;Dt}@R zU4<+xM|Cvo$-Tem;-RLG@vHxbj*Xw1I?7DoL8|ZlCm+!?{pr8`ZntkRL_)LI4oz#` zfDM^_4^d`?xoGYLPh#mUXQII%zi>%#4k3+&(1+Un7$I$P2M#?v!m6^80$8+X)^LP_ zXj+2Sm4^V^;sN>fl($CT5M_jP@1(ezGj6jjp&SHs^@u5%}% zRGbtDbQ2%|X-jrksZc{Vk9T|#AWn5`rg3~*0rky2++pOjRp4%jK(Aqc2T9p-RS%+qp!1+p zS6!U*eOfKE;^3~1MQBPSQMdyIz0)`paQ(B(|M|$wpov?OH36W#rT&ikFj=O@AiCai;<=B!cSq=>=91{5Ukx26Z+kpSg2iYsTE5RU8HfJac& zl*cF%plv}N1PEq|I;}6LTOQJfv;(hY^&J>&oH#;EQ7Fs8LaBR3;CqA|;)qPP2|uRQ{bMA9@NnKD0iqAR7& z`gmY#bH{TeajqiD_ueYineReYC7c$Hp!XfA6JL&cHnb$oVU1plhC@bDOxPZ_q!dBe zL*!F!&>WU};hlu{;XNa;a%C-X?8Xq5@5Y9XGD)78%He%T11vXgW$qB2ci@0fVu9kN ze}bmz821hx*-C0_pN2a`0cNqacBqh&1JNLOz%&R0S$Y~Qg@_!#r^b8Q?+A1yf0l=1 z=>iVA8dKiHV~gYT;)Hx7NEbcL(w^Mb*9XI;d;7Pz_-KoTTM>y~2oe{GyQiL;rd}ar zbgE~llbFS0RnvNx>ASJx7Ib;GM`C3@UwA>0b`vMD7Q)SJb`xo|41C zB^tc{r5zGMKi(LyO3sQOEafV!)3a0GDIGSL+~@%XR?@^I%(%~@(t6}WSsL zxcYYh1C6A>qd$Tp8kE;hUc21#0nrbuk_U!ZpuC`v0D_4d{LfS#`1)d@jDBv0c#&%3 zxC-VU5rT$?pd>Ns`7sSIrl%`6;Qd31yk!9P0Fja`XTy>f@A}Of=f~cke%%pId4x7Z z6hC-kCb|i}vJm>P`b&9u@8NtD0)>b+#EPr#%v5+g_V(P1MIzWBR3U|r55-5*mCf+? zlVXgbvMz)HMuzTT#l&+iz2lZ40gymfQGhU)b0Ug?t0Xp?Fs{CQpA0{Acc6_2O$ zsxmzUrGWio{c6C6H-T3_l7+35)=wPVVN5y=S6z1SAbcTm;J+YXb~k=p;n!?F*0 zsm#{b!`2hN_;4M>?Zu`?mz;LSApZgNP+QcrG$ufu78QhZk;1=sIxD%vOsa=JDuzn)TwJZ1wSGd>QrGB>Kh3OHpi*kG}@f zBo-m&C*%#Dt34ZU>!|jZ&e0*!u-Gd+!gIOnzt6UoJ2x;3^BK0P&3mbaM>)7E*O=?! zMvE?nVfSv>{xTD+!4xdwIS4DN;s2C20Z2HRImmeOA@R@PV7yL)$?83nejaR^=Q1Tw+^EjNNL~;e01Stt8HkmVjf#}x@?oe zWe;>D0r5MEKpScdS0F}@mSqv~!Uo8nqY{|Xyig-F_hK))oCqMpv?CK?Yz{}_0N}Ay z`Sg$6koJ9s{lv^Kzlv0|^6x4OK$sb$ZJ0&|;0!tN_=)F5Y63Mlu-AHGl6c9V9C5Ia zjo1uSlyLE?StTfZCNHA-mG1&=#&BLVUxwAy<_W!o_@Df`Hf-RZizq7aqi$c9st#W< zVgeHKc&I1!Tg2S$!dgAs6;gR4!i3<9^W)CMv4yt>oVECEnp9M;?g5n7{U@m^paA3K zuZmcHuLlt-0I|oR&ntz*89!wIm2cLk+oE&3ts=tK%rMb;Y-Zc!{>c3pJ+t=A9Uv}o-tGRz;rzg-84O(c#~3Ht3PT(7jO^6!!eGL z>O%WI6-o*iCDM4_>qWyFDMOX{AIw+VkLfn;pFUVuOksh)RKy+2z>9L>N{fG?BHCZ= zJtOz#&Ds9a)WRA^J*Y9&n#Th5SgG5l-Vf^HO@G&Z=*r8BUwlCFOdt~%jIcd-c+1K~ z`{)d7k&62@Pu2%q#!$eq*PY&?GH> zzz>H*8H<2#sKu=PI#b?suFoswP1I!!i|=zkg1t3IEW{!HqetE974^2}3JvHhX%g^d zhYeUqzyP0dt=Mc-9Wx&i2%6ftIR*EAnR>=LqWa`pav0n0PW#p^E-1k>_b65!0C>Q9 z3VV@^WHr=Gf@l0$xh@j82OmA;l`*ED!TyEKf5G`FxLH@lq)_yu4!>+$)xUmjiT&$0 zms59-fwU5M|MqP$I<0{wkI9}Ag4LWP#OOXwmV_7Xp(C(`2QQ8r&UwM^f`s}u`W$S$ zliO{zt_!p@I7*Xzy_J@!Wlo`fxeBnPu~@uNipZ^5q!U6Cn*&(g-LKV zsImZb0UqRVf2t`TK9e%vAU&-MoX6i?sCje!Vgc`Xf3hUd5N@{j*8KVT`_@j(EpN{u zhF1Gdy<4X!64mr{FsbU6*1Q7a9PC71YhFLX_EDFz2q$ofgYzi(!FN_jdyx*_Mqu3# z%T}KM>k({)NMcn)Xv=4EoO_iOw1+2XH z@iJLoxauu|{#YEaCX`C=>TO*9>(Mma3#9oru;2aNQkz2_5Mk`!Rhi<)L6?gar!2tb zzRIdL_)id-f&JEGm=Z)nnl&6DI-gn;W5|MW!Cpv}C=0FYm@atED-;V43&#$*9?IpU zvzPL4a35JV`n~pG<$j?(gxHo8mlC@Kw~~fZuZ7|l@h&L@u~_Y-8LU?{@8>yBOj?U#vg^~aW~V#kyn1!e>T^0jHRLE^F)VD zb?_9Sw4-Nw?jjC~Sh{KyV4>52RZi|C8wx;kU*f94}cQvPtB>xXGY<^igLdQ@_Vv$#bzV zA&K(`jr%z{j!=y{l9tS0aP1X+NJ5v}woK{p1~p0%@mFHjLB+ zrX=J%n@KO(2Ol*$cf0!^wAkG@@aL*iZynqZNtA^I(I~d^MlLw$TS4$jp5H{73KYj; z>@xBT%&ij#Z7KvxxG&f?&I@>@-bYn=_r<#nBA*tC&DTWbhORe=eFf#I^Vqje@zi$9Y9>J7Mf+Ie4kL#L1S%j7bsP^c#W~0&(&nChSYX`W=pHf7w%@J<;OP`_fEnDd_|Cfa{~?Vz(=lLF=Y$}k zRUgq%XUoUb2$flE>_%ltV4mq*Kz+C<=|^jh)SQgm@SRZHwTcl&rn&N)mz!^5>(juw z3XR$K^Jssf;aAi%*rMdv&+G9|ng0c`DHO_lEdQC|wzs!9GoL(M8g%@hB%}@<{=x52 zjDIc4Wfeadc6+hD?zGk@ww8@ZYz&cnLK9uS9fhZ?Y&Sxy3I$8jZ%;7T!!0EQ5eYIG zcQDteKxKHPJ4aF%3wk&c9FKK~`sVAW@;#`N-X%$t80rOF2frVrehOiGNWVS=LsPE*vSh=^qXsJiMW zixH8rwJ{T{PDb<)sAy5X)>G9Za|awo{gQNzByqP=mp8Ama%~9(&{sQ+%ZpNBoFF37 z&YAu2Dc@F#*Bmhhfm=+S$A}UjV9Q$<6k-NXAl|GVOAzrSO^t)-A{cLmc!AyaJ^9a_ z!T1}QHRvp}Mo*RwaS3axO=paK+9Thqxpb$vGMPq;EM7L)fy-gRjX0S&7S1_ZZf;@9S;7IVZ3roClraWkYHr) z2G6N5xlxQn5PW#Iuy%Fu!$E4U2Dol5Xh9#zomJ0~7~m+%CfoV;*l0uaPULnz}q1q~J<5y?NF z*D-v$!vOHNe>cMgx&Y@z7C?8$Jz%Fj+27Jcm3EXm?QjsdhfhG<0UGC>^^dZ-ZW=WAZd|!BzUjcuEiU2cJ zb5{tnaI6Vk2I!dn@SCVktrl*Bx4+HPh8KH!?YE2F+(+|Xbale4yOsTs$R{K0O)L&u ziR~AvIJAkQ-?KWG5JH>YA(Xg2u`??L*h0AHCsh)eK!5#4V%RS4|K=@--GZ!YA7!?; zGf)swtV#vAO|1Q?F@cfRNNm#Rerm*})jB}#Y;$o(0p#uC);J}4VThqQd7vy;)SnPZ z^djN2nD(gs&3QDi_FC+<{`|U>iYA^*zL`V#$}d&=`^_+Ni@0}&Alv$mD=qHv$XMn98pkFChW895FX7z9s+Jz4#7sN$qP?tn)fN&JaiaT#iAz|b8y$q=L*h)gBoX- zx;!Q7Od7|Rcz|k3BCkm*RO^*HWX-$D@DW=RqE2;2sNUdtDDzWNQVItg(fxdqV_V9B z%B~mO-GBzyntzCMBoZ3=j%tj!J|Ck%&;>{fJcW(JzNI~n?ay6UJj;Sd6`VLuv4vxF zai%J7Cb#W!D`z(-UV&tH?`pFMg7ONCIQ1Rstg|p)hlALg6j<}E_s`D08MU#I%8-+k zL<2w$_EiYZA?>+rP!NI)nc3(xI<<{t?il}bBFt11{se?zj92VYL~w*%{Nq#fRw@f7 zzrhAPhFFUox_Tu3!1~K<)1Yrt?_{ltf}k*EL$&s+k&)|z);F|=c~1Y=N1AaENp|xg zO<};CG~RHuLwIMzGY09X7Do`p3jqrkcT8nnEe$Ip_Z#1G;(|~Lz8mxEU^iSa`vQ;7 z9EmYvfB8r%Ycx}7Y*NmS5r*gOUVcsr%5!`kW3lOi&GDv=VY4+xXtjfU{nHW#2i`x# zjrXKr0+?!G(;kKcUeH5P)`$IFz1Yz$BaJMsd)_1CJ9Vay+!GLKpRk{)goV5fSj4`! zC<~7Z)wUyWNQ6on29OhkWc-q5$+0IW?lfT&FRPz?Ml~N+^D}(T?dOa7|2*N%Pj0EY z`4#lrsR?P=$pp#JoP+i{%Sw+zAKLY2*@TClVJ{LghRvsH<|tuCp(0f30=u2`n2-Kt z<5=KO`BN*ClEw^u6b06yU0`W+<^w84LTXrMO2s%oVs%b6os#5AM?Dh+u7vpRc& znv#?lq)lvjNgIvZs73tIx+$-&j->EXe*33%EyC;N3yV^_q6M{BJ%8VCDR_(*6k7bI zZ6i|)3>3AZRs4E)Qm$@nvU=|HPkJM1HnA5KBVFo4K0>;C4PV;%D;%=gja;tY<3Ngv ziZ%JH?oTbyK@g=$`3ekYfNnh|Oa!!XV;%z}74 zqHjTPCt$AFw!<=PeB?1s{0HlN(@aapk57{fHDYxbwIl6k1QWw}O$+5b^wXGL{qUV| zm9m6VV;U2kkBK|AIqJnm#f68jb-^m*;NIZiGgtC5_5W;+{!UB@OT|8HyPnyxP7?eR zMS{X&(UO#;6rHY4RskhUYcBi?=`^`)Y+&fCf5TTP{9;UyL5DA`m-tXwn z9T^zv?f(yO0U3eC7R=6vAUlGb89cO=kCG3?N8MARW+@_We6G`5UUr`O7$9uFVDKlx zy<=G=5bv1n9&WP0$vedSm)A=w1o4L(2q6c(dAiw)hdrJMgUpy>EsUeg=TGh)H^xt1 zK6&z`Zz1fIhiQy;P3N57>m|-5iF1{4@%x7#v402u0q*h}m4^34Sa4VCn0#m`cfgR@FC8e>Azb- zF16w_%6p2ye%E2>Pvp1Zi2(CFIqX|RfSjC*i=CDNbJ8uoEoHa5s^G^Q2s$L5o$^gF zeFo7v;u3Y!{sRK6+m91*fk5-e5`a5qx;1enA*(`-z7cSTIEs00NJz9^W?EDH<^uJ# z<(9C};O>Oe<*+UHDr`R3BXWf|jsU-*we_MW$&@Xr^M#x4lU?ftCx8WKqS3eb!0W zj+-0ducX4qiU*s!Q6X?DrWOoPD3wynMXdhjYT(}zeZ^Bswqf=7;NdQF1{z&U`}W{P z8av(8meHl%%LxqJk6BkqyG*6-VemG`H?HA0sI@i!@fk0?Qc}PY;)(C3jQyU)9z1bi zO*>!{vcxG-nMYi}IyVjn^biU$dzfa5<6;VNz0aMb=r{&QEgj%t zpppcHUXgfUWZCmnwvP;WF@L6iicH90|LA9nz(ghc<5Qw8doI_opWr3Sy@mv`rUn1y zh0*X#jaln27EWO;Pi;VumZ%ogTZx#_a!l)EiTGiIclFvL$r2Ns1VeNqi}wzZu@PA> z_u*ogT0U9%3J1_cgp|X9Z*ZY(Nbnm!y)q6Uq?-vEALm}+wBQG$SgdcN)Ftm;Lw|B( zJCt8Cc(tOY`CJV!q$KBbQuex;i-m+nMds&bNClxy1wEEZers6CZ&14t`YCB=0)0GGHy{NkW&o)w z!Z;@W^u#?mwPU6|veqH<{$X%|6h6`qi_ur`8_vr&*jDZB`!Kk5`VY5)MVw7}Fk>T& zzpI^>{r>P!qsWL+`FA`p`Yk!R7!wqpp|@$fu%BrbDwLd}7T5LZJ1rrmXdhG4O7OQm z=)r0GJ*NyvGcM65KiY0l40&!Fc|QwRt{Yz?*eUxVz?5NrqQ}zO)Kq(rdFe3s6%`g&czd%VG=&x${lKPJtS_n~|1 z+G~0J?LW5k8;`oNePg~ZaVyPavJWnUd57@jw}vYOk1d9eM!$3PtjO&>xD?t9WGp_j{y0$H#^kYv()C zU8H=;=ktmE55y}2=UdbhX%LJ5MjH}cQ}Z3_uAGh;^=yDc!8RgiAGwRlz(zV}mr`E> z9H`cjJZdswa&ml0AEWPq0U6tpqqxSTzOMlf{=0K)^|@i>I=?^@3Z~K5T+MGkU5$v=25Zi{H-5OOzj@8_Co8s*8#9bv9q4D#Um`eZn~`&BLkY0g&5FHoFu75^QHwN3eZIw!G!!wBD1XTegA^{!Fk{N{dzr~kEbF?am6el^l@R%eW^pl_Y~?Q*#XV{&wN>Fo;5gQG`f zBJ8>+2E0bywp5F^beAFB<@*=7tj|3sISm$rE!|A3KV=aF1WAGeC{%&&rim&L6o5hd z>TjK0Q}5R(P#G&6n=#go{N$*XKU*$-=LDc%4_po6*mDQq0Rq^Dq_Z?5jzZgu8jm%X zhP#Jgc#H4bU9$&#P{f>Y6oQzZ+PJ?PLuTNv zSpm-LPqFgHOKT>es(V1_gB&KxcLQe9b%_q4oSb-CAev+5TgsZFMv46j@DcQdpTOHf ze7Hkazv%`H(t(wWo-ri7LW=f7F`K zB-5&=ZR8};>}4#~;SM+*aNrU@B;0u)=fHDq^kMkW7CPr{g;OBx%jthecjGqe{$;T4 zzXA#GmPh`7&F|aV?dT_%^z)uV-Ns{jHU1z!Oe<{dH;s= z>l1q8s>?9UD;DE_7Whc07<017L-;#OzaTHhVZJ^|&>ugjtkG(*Og8qEuk-IXeobFo zEdM#WxH+KuCPlX&e0THpklLb417pSsc3nb&6`>)kjvkUb*?fGn6Ue#Ph5k8h`gOTU z_B}(9gIrLdzSMbIWN}qiDrj9nEw;kG;a9O>PSrv9;qNL}t&NuH&;K#pFyd#@zuw9! zsEL=a^rf4dM2*=q_R@sS9}2);w*{GIYB~q^dgZgJ`+YwP4EC(HXMoh0^b>{Q9evmU zHB$if^Y+?V1GnLRvSB4o9??q+F+ua*7i<+`cXS}=>(hL~Z#6{OiR&i7O*H8p2Fme= zlmX84fWY_mH+aAeFBua(2hf56`&1C}vPZy)KDjh!k4tRV0|3toIDm)TyS44b+iGE= zjQ-AwGT_<7K^q#qng60}I{^VXM2lqXCy+?8!7K^L@mhCxw|CM747h#*B!olQZGo8I!$8aJ zTA+WkLW#qP01?jMH@!meM@ycPi|yH#U2|B4!q3-8s>7bjk}xrV&?AQ;ItruQNr?V4 zXc?vazn`xL*EkiGP>>Qwaw6 zpg%dJro}Pbsu+*|)jy0YT>k6ew3iiDzT8LqbS9;>HT2fm-iECKvn|;9(&m0)eE!$v zY4(P~*4Vfz+TTU*>M7Z=YNMs){$=!Q-hnO+#=ORdrPkvD5yGsHz(68iPM)oFn2UMG zusi;|Szh&wW{h&HX(wq?Io4`szA`3?<5kl;>dF#)>SR?Zm7xE0a_MPIUgGfsBUzzf z#-hyxzkYi1WWp)ZID+MW+kSI*A$9Cbmk_nGcX0MMz0CP-s`4beAy=kImq*%C@Yn<~ zlfF%6YXFcw#VNMk_sz9J&oZJHkfos8iu=n)!o#LQ) z_I^@ZE?^P)BTPS}AUzuWXWMU6hkMA}oQ|a1(4lj{q!?i2a-6CS4C*88AS=1&F}$C-t=C?Y~@Wg)-=P zZNXPUBh!di^&P}iB%oo#BE9-{WdUz!_h%-)wpnn^aw!J-#*Eg#^x8u>IHU^z2LpdP z`A(ST*)Jjd0&GBW!xEU{_v5#g`Hl&IB6Z%GbY8WD>T$a|AHs>=LEMScr=EQIk6Yr- zFr+_57! zT&qhnj#_4!bYrd|{E?-l^yU+8HRt2xCnzzvMYseEE%hC@uN6E~?xb?qS&<}x=(4l` zA@i5@g`a52|Hj3rd%n(oT{v-|`jc2Xp!4IWtY}++;nQcZ^<4I*-gndQWjom?mve5+ z9;7Xe>Kkurij}r97&{2&Hr1XRTjO_Y*~xZ{dsONmRhpftdfj6g#upBbYh`N(D~3qH z^fRpk<#J`Myyi>US64PIPYcJk-zGBB{ffzcf9oZw&ympJ_W=alovrs?7z(FE2V$Hg z4SAiecBBjRQXETp6UT>Q9>|dm{l|LVd+_Hb!*BEG_r$;7{Br%D3E}e@W$}$`R%R7{ z6ahNxRfqH|Ow_8PHLyDfvw*J=3Ql8Al_Gy`XhUgcjvuzUhe7AJLBTsGQt*@v#&&w8 zw{*!lr-*tpW3(!5qYRFt$ryj7)T7^I7QH;_|Gq&`yV@3HHEz9%@Q1Js{u~u|lXdhf zt(Fw#0M^KeLCd4rVS9Px%JU5j3ktz^a7VXdc z2P#(J76#qb_h}XZ@Gsvj_Z0z-O`Vxedb=`0VNzbWa)~Uf%M}ZVgqb}-Hw)--nqF6KB3Rkmpz^@5^Y%y~B*sn&`oXn?0ots(6^w?5_V#Fkzn=#yr8b4s2VnEiivx+!Q zWk9i|04}ds!Ng30_lS4P>H+UN9GZ(>Vy53%bI}UpVO?|2`LQrReN&_WEt~RAc1m1D zy%q82UxO?7wx=APK7b-q#(cJpcYms!1vgx_{`#e@@+d8vb6@KTt61^pF{ZM!n)X#S2D>@Fv+A2iZ3A6NO4m;Z8RXbfR?;eo~a4XJK?)~9_g-UgQN zbD%X*AMLt)(M;mWvPKRRUVY-5sOBn6pnXCHSi^GgEj$bGYrOEltLE`J`9H|cnlGfV z_P*c6Y^R#7aJV13^*d>a&co5;wH}b1#BWgp+HM3e$ze`Gyo^C|P@0B-!1xGO#=d~a zffpg!yNhln2>)ri7u|ZG!d+w=@^icm(tVoy_H`KE5=9kz&foA{`S4fdMDDyI%W@2O zFDgSq87L9rq+nB~2;<2UM%RG>t;8j<#4nK)E1Q)A0|QT1JQ96~!%{zdrk z3+Ja8&x&kCoVm@1Q6 z3hzYVUrS%xFc5Iv>re&(dAd7GyGYwWFpmQijUM5AIBaD-NS8I@YlzQ+)lHOG3H3n= z^iLCgd`)PQ3=$N^L!^!VolWf!zXMwNJZQ;Z1|ehVt>Mqrkqi=inDskK%{%Oi=HIUu zde7yZ{$2kL`4{ArpV4JRnAn-sb~~tK{ghB?PR5B98zLNbPsZ7feJ$LHT-@v{gtE$2 zv8u!j)8YeT&DT5KqwJQ9D0(?KiCi>%LIPXql<^~$5R6#NU&Cq zwBa^{fFuX9M!fZT3?QgZ0!b?AC;(22DN zN0-WfcYfUe8Ks#cvOU2Hb`5P^*gm?}b7UQ`;(!gv8CneImw+||meqXmZ*TThnV4#8 zZD|Q)U}Wfn+~PFD5Am9?WcHl0`Q6goWwCWMl^M}841hq!_AQ9DmdqiZg+z$eq5~fI zCdebV7qK#-6$t3y8%FeW^?}}CP*9_5>=OpSYvWk`Sh7j-dkLv11lsmItzfaXj?9;{ z5o5EiYK#hEhz>iIv4P!GRiih#caYlZuOhEO&Dw>W(P8{(#+cFPS2#p4w*b!{Z%9wo zN&)P{)>gZziU~6NBb9_6PxU71;`}_22?U>04P&W!mpc#L9&n>)-l(oSQUAbC?SdSW z4m>P(!k>f=O5V#krr>DK2EEE1CiRv!eMnLN)Wx40lN6Inii0N@bJ4u4v>cXlTlql5|BAC(KWTT|Mu@11~=-sf!1#fFn4;YR$tZvuZX z5Sf;vEq!Y_jbm(;~aCzV=Rd zN?)vZNzpGQ0(X>1MSw!B=HG^a-Oev7YfzsOw})u2vb_68)lSI-LX=e53tlR6YCr)1 zBFKY4sf=V05HRLO58!25wR7?rG4%XoY6CXd2=?Zh1v@HH6F__<+|(zI4#+G~8Lthd z<7}carDn}pTjum`$4M6|adW{TB{522T8&;#-Om{BZ0G_)%h03LAWmwi2d=`tZG8rs z8Fcp|>~^cBb_=ea%U-pqAwr&cSVAv#nSn*a3}t*|^IPm=fdJ3*KU=x4IdZqhm|0g( zNQ*05Tc18Spi^t`OQ^IPanATYE<3-<0fP$xCIH9bH($+@R>OS&R9Hv^&^|# zOud+Fc&;w1se}Hy_YcI8KPt89Z-Y#u6i4;#pM#y3YWa7LpBp=F*YlY|{vAIEfIS++ zGsmGolv2txz*A{>u2SSVYEQa<|G2;X-F0_&U0roBxjVJ-CSH{! z?{D#Rz?v@o-zzB{@Y9A-q-wUEXtMnX8LB*dxh`yyuDP4y=SLm=Pf8DLoO~GTlkNv= z)30iR@9z>?sZCkKtO~kvXK-71fvl{BA1)dp`dSvhvk&5Q4*GXVuFJ?E!B0burJLWo zC8^g9xZQD4<(-KVCCg5JY#aIthQBPIB#tjA5rJML0>Dt(u`y^&0k0s3s0hDiB=)W{ z#Sh)x!Ah_@0AKqjrUO~<=0{k|NN|C$Bw&GyyXIp-g(6eaSBNNjhLcbmRs8(1we8ODrx#oHKK%Z{M)yGX?$5V~FiBf{c)Py?XrC$9wtz zM=WEdsq5_DrGZVE4&h8R{zBgyAm+&fEij&aR7T3zluU%a-mTTV5pxp|6J7U;8e^*1 zKMyz~itWk~#F6UW=)Tjiif?k%WBpZ^Sop+(EOuc1`Y`!YTR09)*0-;JRFX%53w?f`Wli9mzO6WoHb_MMjMG4n3)ockpmMY9Fx^as`O?7a?zU8 zqvhv<_lLQX3E-5C>5UE`!;$`bGBCWJSeiZM*=BJb^^q=8-IYxbfw6zZis1XSDB^IRM1f!Qy4X<)*B30*+rl&p-9Fv-BdNq8_uX?pZqL4^=2p54RX5@q z6E|2=lf?OLsAE^pL7S_eS>a>6_jI7o>!SJ5yC=np4OnSuTC?Jj`)`Z3AhdBkl#FaS z$J0}^&4n$atzheb;Q zmAhz~6F~ZNG9IsdX0sYTjNX$8Uw#%;caX$ezZKm3%gQkef6bOb9XzSj76k!l8n^G6Gei9B&Si2f3V^}roIkkXVVbm!#@9Z)Jf_#mEbWq_+@#RZUd((s8 zW9RAJjF@=%JN1mf;iE2l8&IueWcm(C%riHK%E2Cb`52ot*xjKB0Q?1TV;b~rj`*)A zB~Za|xwc2k3%udaG}1|GyV$A9dxGln^Ght(id+=<4%Ht_joj>13HW2`((m8EB+f{= zb24fkX@8tIQuxdE46CZ=Wm+^xOIyBIHnLd)^LitrOZ;TsV? zJ?nZ*LZIicOz;pO?a4ZKqEV`0+`V7V9&%%C$fe+hW0OFLfj@jx`&|K%3@j_Iy!S10 zG|^`3I4pAlzOZIxiFOGX#+5lWJTqTTi?a;-U+5b0USZMwPH>X#iQaMq+auO4Ry8*3`--V&6G`nTogJ6+^I@mk24rRxT* zi>|B=OLR@`FSqP3jQp)mEhv?9OpontAn;hRnriJtx4>?}s)#vcl3feAk=P_V0(+AT}lnN?Yni2yVoFyU%Z zVgnVq!VzyKJ;?ww^NJO^Q3td!5n`?#Bcu5DEod9g)oA{0VuhdbypI1DJq1O;P>RzI zaT_`(bAtPIr!qt!~DP%A!@msut&y?6>RxBh?i!f}2pL47Mpe(lBHDO;{58s+P zM|Jr^;^zNzL;z`XFDY3}NI{7yIv$;X*p~W16eklV&KQT7s9c*aNcm}eRf{4gR}$4U zm_lobP2_KhO^kZhtvp6{yJ=s4iHtNR7n(8~H*y19-%ahlGEEVY)l>WrIr1 z^yXT0q49$v8@@4gYfM;0^Os`Nh4(ZCT(sPPGl4~~nV9-IJ4$7f}OAjhsfF%V=fJ6#q=aa1>k7?NqUD&)P-ZqsfqRO0Xue;#`6a zEX?^emZ;pEBP?O4v3+qF70_&4++^baQlG?m{$;_A0khI`HMJ5VeA)=Q3bzxSmVW}H zcqXeF{~2e)_>H2%iF&0h>!Z)!deV2Q*NUWp(JR zVp24$YSsHjJeZ@bz>{D_HNln>9+kGZoIY%vHYFik98RD4F*eteVbbLf%61ykxWYW! zDNFSu`y+b(UH|@-cUh8MIqqLgEzNQYb8g)-rmy%9F8aC#_OeC;fr=z3{cokUH5VPC zIND$Xh$pygn*-5(K4!c{Y{HO-YV6}aa(<+>mZJW6c_onb2`|-Ph3eXI%b&0B;~4<+ zap(@2Cw$1?zAx2S69zh3hg$(!SJL|SFFD-@Vst>O2?7w6kKdieU^jcOLA7HB7L=Al z_q7*&Z@M7J|1iYR0AE;ZG{Qp@12ENPZ}JOY%I(<@-~craD6!N{;rE;FCfaZSuk8OQ zN3{Kz&kmIU$1cgQ?UQw2Vf_3rT1(E2aS`SN}U{o+D> zucdcz`Fo+K4LXGNrn)(B(8-|hSin#^>1;U2$apK-8V^mR!k_t7W%TR*DdA8@To7B* zkq}INI0}*n-yvk?R=Pw4R}@$;oC}|zi601@={F-(=KJKr8C&hc7FvM$UCyP^sm0gc zCZ7cg=)E1KTD4=<0C2n1s0kH)L5MK!fr!eTQQES15_YLYsaptH5}?*`OLN;<~SsxeKk=^|V)|I~`h&neS++gF2GL zeT(aaFqf%_nft>)0Z+FdHg=}q^>?yOvEkC0ncinCb~$e=a<$LMox6z%`e`tDRFgB5 zSGs*1lEWx96)@5~`SZ-GOWp_@+3L_@pLmzXb2SJ^YAB?HkF>LrlnO%-H4BA)XJMUN z_Q$*7QtW7pX1q73$?mv9Z8ZsDS$yS1{j4h~-V7iwH!4XL{tkuun2`d4$|W{_u+PCc z7!y@~iv4cmzw%Nd%K|jb0JFbeZWeBVzD`lo^kRuo3!m4;UZk&}jf?FA9yAEQatQe5 z%$d^dsfP;?+tQ<4O@M2MGsl*sD-Zs*1}6~+o$muVT|)w>Yo|rFGlmj`nlxx*BEtoVRHFQ zMQ@Mcp;BS2ZK`v{BQ-w_g$H9=cMKP>sTL`WT%IEzJ+-7NsCN$dd$Ap$rR6_TT8g&n zBI!eQc#n1&_|KtKM#k6VuE&}i&?H>gBm5WIz$@_HDwM}f87*)7| z7ziK<^~0NhXbyiLGd;iVu=Q-$sRk( zh1j?l9uk8?mL{2Te}g45UR%DhGXJDFaJ;{0pi)bevkK4bz6b+m0-U&lj7UYKx#*MU zo1HbnnK|7QCg>dS4GQPtdT;G9;U0z5mEsXxKc{G-eC?5uk|$?^o-K|ZnZ~ZoeL8s^ zD@!JXdXOBF&BsvXc3`?{YV8?~H+v<`4gw};jE4L9TarNJzq|U&qYi)X4Xw~Ud^r;< zK7kpf3Y3d;@|;sS0I#ONQ#n)KHzH>4G|Xty%+3eyuD=`^G^&9-d}B^VI7?sz=c1@< zuyy?G{ynT0Lm#yqm(GEBT?k$Kw$gp_XQXDtYT_?VY&bnv*H7OsCO%0{)`aCtG#Cgd z11WezYkqrjbHVUp$iOJSy{*D=MV;ZjM^9364?IoYe$Dvx?wE_%b7%B!V)zAG52Zr_ z5xn}=XTf?)y!|^&Sqot~^B|9{Pws_I`Fha|Q_W)(e+h(oI0N&h5xQoAWL1Y%8D%SC z$mTry+_SpqwgCl?nY`^~1q%+CN!{}xaAi=_T?`!u$J@t*ZQ$5KNnTFkT*AsDnhlbp z21pvm$BJRQcP$3w7;T7vfq6x4?Rg>s z&77d^#lry&4Ro%!V@81j>G@Tr{dldV5g+v+>n^zV28)JMWDxP9CjP>?WnA+L$M>q8p({=De1mX1uD#5Wwx!s~Gu6vCknN5Zr}`}jP6%VvoK zUfYPje~-c+GOT}*GJdq;vm4WB<@B$nNmf+Sz;3-)ATBE?kNi4x zVp{gUXSHPEXtDm!k7hyZc|CePWFc3_TWWP`1~LURZzff%NsssqG8~=pA~wwEAlj68 zCtvVeWZe2qd|wJ?M0L~Us2#mkGUY{UM_c}3)iQpM-xQBeXE=Kj#`NSU$AV|9>!U-i zfgQP_G{Kc8PmnLTB!0e+VcEO)M-p5eF2fhN8yyEiU%v7q& zhuFiJAh{6BO3%s52HR9kC$-?x;Z8^zipMyUeJ0=ql5ieN;efoj#OAY$*U*6FaE#BT zv9h%^5RliceeGdIJ}|2z90x%zCi3x~S@WRTPbg5oZhNwYy}5bvRsRJg1=6r+;%!6> zdXgI4i7)1XBlxTaXVioG45Z9vt6-3!+{wwwRPqqI+?X#^H zTRSf&oWxRxHK{A`v#qkVN=UxH4so`;`kFt18(@{9LJZaHm=ZeqRLss{&4SS&Kp2!P zVEt&FaxyA9LOrc?K!P;B%%OjSFk;ff+`K_lTpa&o!1GJEd>dbXz?sK0S{wD)r|Y`A zGfW1FxD+UrPo5qQo{LU|YYW~F7kM64kN^^$_olIpDn>@Hvqnc$`u98X+QmQJ9AZ@d~_re>~>{%}C_eS+Tq_K;ih>36j3Y&zcg`_}}^I2;B z*Nm?ipBS+qGQkAsHsv+|m|C6()MsQAI7Taef9xZav!2E9mU&~>_#{a${osc;Q}rJ( zzfpvCW;hBivWCdPuOPD}9A*%U>4Ix`_E+rTnzsP;qHqoBz@JP5ny_5a1KADCwYzZg zLh$`2kqC1>ehqS40L4WbcMdAb!I0VG&?Lb%jnNOt{N<=`kPAEy(KMQ#>Br}+b?MRlZ{&4^y|Q;LZ%|e*#eSa zc3J{GrJW*11GaD-Sgs8$z)*sJn*y~VR064$Hy`6Tzm~%B zST@aCFSj*Hh%v^3ByMRL_d4VoE1p0)dliwjh6HQVeNR4ozZm&K=doF@}mx|v;5~p+i;qi@Uf=;h3QhT<{2N*3i_)78?zZ*H+vArC$ zjS$3Y`66bQ6#c@zWrdz-N<-rWzplw_afclyScZ{vnsD*^Ydupr1}Vf${SI@+O|-oz z$WUdGpG^nS_aYMO+W(Py#^_3qCv#jv40~;|3RmxhD*-c}2_0dR55A8Ghu&#aYQr~r z2(^`6=>3+Aujm&eHFgDzF{CCw?LGgb(^D-8VuXT{?U^NB9Bx!^{3=cE3m(_xa?UuW zIyGI&{#5@p-u1s_jRp?*D`9-19K@Lh>t1DuwDrE|QVVJ6VJ@@)X@<$6`)LiqxjMmx z8x?VkrTQ3=W{X2?IKdhRZJF_70rom7F|Odh-fUz;?ZIiXZ}QDmYh`EA)xKyX*1)w9 zXP}c2+g){YxXsX!ye>Im_Q;a}uu)NyTZ;`NEcpb}YQraV+N=R$%>KlJF==4e)8ns4 zl=sjtGCv%HZxvPwUv;gDZQ6@+$L0l5-!Rb2gk}qB1IFF_>RCbJw-Qp+7#XtXP_n;$ zPG+mqw!LLC9*`P*!PFUxN8hc|1YkywGbd;xot!`2qNIT!P7Auj=L=*=I!QbVeG-xs z6N*&*_9Vk4dH&dQ1MTm$M*cq8nm5_}13UT0$xoQ~O_FK4oS`cf?eY5#>D|EMtdhrS z;Kg0UAE7zQG9OHE>DE>r_=>;D_;uh3e8IM0(Up<4yg4#>vxC~` z7CXBqRw4T1(v~7+p_m4<1y4%T9_B}0NVv^>#s!fOP{y?bRYwx}zANTv;b%*mn`JLcMfsZG4U58B`BbYP zV_e(_BST8U_2cgEcmGkBh#gg!GPtrNCG|ZuNOEXn4|}t$(7=}E*7%=fgj*@*(}$5P zHn6W8iHXs`$%8JQfZy!DM%1^L z7miCNFs#pC=IZl&U#7V|Awgo)c(B6VUPnwdEPS__n15h>$xIX$g+=yM^MVAX6R?jp zT)|7T_%gn~ioYoEWVX_ljuz0*tBhEE?qKxDb$MZx^9B;Um@^eA7G*zIV&*0rpsD~E zh-n#qc{C_{FPPvUEP~}8F$g=i1ur(3_~91%%s*A~qn)`>l?CLXZ?I7YRxP=FlAbRe ztsY3Fd5Wtm|3R_U-x)6EKhMNKlJnCHyZ%)2K4KDXRP_pTOcVPP>DLevgXg-M&k#en zV%|%1*Eqx?=KM%fT@2NkBbjXzD0r&6S32_=f@DnpwZ2AgWX(%IZdNsLM(B4bemU?O z;+x8w{a+ZUiA{IG-;~yBY?5MVBWWS~V3KCDkVQx`LpP*;si4B1>qZ&Vi)BI2@+d{v z@6B^`^%_RnR~$nEk9R)L*9D5=TN%dwJRxT0E!4+l;t5}<$OEYOY(0xB3kUC8H-Vhw zZKCX=q0(+PZ)N|!kIs_PFXU-5e!bleX2U5!-PP*FNWt`0;*v>_f`CB;X+frU_K&K3 zA45RX)>~39YG`tCQG_;^Em{;->h+iDTMM@C9^IGvRs~XTFoNEPwa(&v+8<~rXdx>KZ=7-Fqv%1K6Y$RpGqCP<1rEXHfrESv< zN%8%8`7c=|fuFFXh@@0Q2qv&;+Vkq{@y@(F#UY@-p#fDh0CZuDKBK^RL-6*2Ao7;J zd}Qit@z-~EC#iduj5JbyhfoZ|tJ5+1Q@#2r>?PLJlq$szF&+C-_la|gPj<@*X^Lw1 z%g;=qoh#gy<2u^78d~nogEYG5By)o-VFpU#)c$OEONMX-*TX4x!5x{Lntv6GQY!)G z+2cmU!6*|0^B)rEWoev}5*98f<@tsSkhGh^uvKZP$8$ekvmoLz`jF6?TAPE7yK)2S z6yZ*f=z4#48yJWXGiG^1TKiR!*i-|9B>gkWo&-tlhV2mB;h3^uDCBYgSr+vtSjf4XScGq{NSL(YQw!nL^a=HDx> zZK;suA30`OCkYhP2EUgBeRevF%j09Uwg*d*+c)hmMd*CbM6w()U%DGcY|g7e81AO6 zM4HGp2hPiD8F)5kHEKDiRUEGVrncz9cU5K}c^Y~Xb9LU9mh7_Zq zQqfnTBRA13-V!s;h^3svPcwRLY=Ubi#Z9T1Ar=5(GSq_SsPPnVxh4E@A}R=ep+^Ld zc@#2x3IP;w@~pD{N$2UGK1GT)7%#!_<lF`GH zfEJkrgOxV9YYDu3XaVKK0p^wjHOz(6vOpBKr9HI3ua*_d#SDtjAjOd|U--Iom{Yoa zKS+mQ`{xpZ^UIRr@<%@|_-Nv^^m=Fb-*YUKaN&MY#}@z`3qSqp;gRR^N;*+{poy{C z9(db-bF+8jW_kUw@mm2qK|yh`z2zNNMrf@zBumsfS_}%C#t_|IP<(h~ii$YQgiv_T zOj(rMh7lgRxh|~%7@P$ZxIO!oDP#DK=EdrjjD~UcKZbb%fdTQ5`;{_r@jajA$nj>3 zj7nFpw%iO@&TiAWf8v>1i^g^l1K0+Rr@PaW1^kh!sFy;bq&0bQwL><+K4&w#HniAL zV5+4TB1Cz{l`}=~NdV_(rZ9bwu?7wx=I7w7wDUJrc^pWrCkv6p+L7SmtdHwcV@Dv1 zR*%PwPc38^ypn~PE7<4e$n5GvOyvWgcCH&ci|?o@A$T&RQramyMdrO0H!@wTT$8`JNUxxQ8-EIExg%z*hOW$#WWLw(PvwFX;Y{nykin zi(ULVxsE{e!!TQ~{w={o5C8$~pY2ckr;m=Pq{ZrVf4pXQH&Rs8Q7|@M8bbjW#@i_A zSfGtwSrMqjmFFz0PA!zu@BZ!AQkUwz9VmjP2{Wc^< z_)QjCLd$41-i2M(5Tlq82p`>XvbN3 z(()rGo_xzS@}VaZr>Lkk*lPN*!`=zUfsyb)W{I(P>&>xIeqQ0gZ!L9oqWI+J4&)<0 zmS6c@9{K-Rc?xT}zWSECQp!ScIB3(}6oH+&tX;>}LQ$Ei^(r*T?ZYdQhZyT*07qAP z%xZeP-iC;Wr=X9OhXwD%E8Kqk>`-0OIo${rn<|>J0$kMOWS?C(np^l2Y~-pJ3gFf~j#r>h`w#^5S@ML|<|s zN{7&3oG*3nVFGtv;#1A>Zwb%XzqzGi>1VkD5F~>@G9ECe82(3%N1_GhTU5@NwTo?a zJxWNiWuf5PyWg;lk|N%fzez8ByoAjzrU8PFKYXagzS#f=$ zOAuS(Ku^zi%1;Xd{xGec7}RmhIlUwre7%Xl9T0#D-wXgpA^*y_r*ulk96HJU)^pAA1)NgdQO$+`UKEg0!AY7DClEQ0PtKebrVpPNEr0?-fLd5t5qDqB%I%bs zPS|_Zi_1{y&>iUWpPnQa@5=J^S^w(l>iOe2_fa=~{`Pj$eXO<)!5qOids(x~uQC&A znu7{ygHvhaycl{+T-ISYrNxzlp}dut(K=-PNfsONQg%u9(!^WxG+ zc{F2o-PiW@lISiP*w8z?b#7J#Nfcf8S1i0S!z>i-3T!T+s_O*3|+-Z|pe)`p3@yT0nU#!7o*evt%| zfriK9rRwr`Q*+hjgP?5ZIVI2@=$YT~Q-oBGm73X$8mlrOfi?|sZP_>T8V(Ldi|EO5 z);N@%H^9Cvs8p^+9jOduOfER5`Z0 zpQT#J)VY}%@y1;SB5}BWq03mybvN%#Cbn2dR5o&A`LO^mxCZN2_@gc_%t= z$HP8^{~qtJpB-YD@9awoQ*Gz zMDFdDw8^vXk)B?BxdcmLm$a`VnI-2b|2jJ>#IoZr{C)g=91JZkX_9n0Dl4U8zwW-L z{K{laq4cg)h3mbuOS7RNv9n-$)&C?F_tt|W1 zwzO(3+{TCqV-gg6OWyoz^1$9d-`!w9B=yu=Fl5A0!YjzDG zb;JQ}7=Tazc=A(b=Amcf#3de39PrQ@@asz};60kKbWI-X*QxE>dicY*S_8(?Et|zt zDv4Z5CqQ5#C4K4MPD6Sa>GX)NWQyMm5g%FjQz-ySkmk$+e0`eEo)Z!$K`P2-iAm2n zI37Efq0sH0>hEI~aKZa$c2g{6P`EVy(0TRmR}#lFA4cv;_Cgi7bLOChZL?g6Agv+A zww{1aEwrnH9p98-3d@t(or#I|g`|t4V@QCX=L>Javlyim0r~>Ft{X=Hsy_7L~8oP7T?P(pYR?5{J_VDAxadw zhRbX9(fw&;JUkEObt2J>ZL+Oco^q+@E*lI?;;ubY<;DP7N#>?cf%dK3A3(K_JUKr% zkke$d#w)PH^-mLITK>pQ$1n@7kTyA!7~t&8fzHKx<$PPoTV-mIj@G=(pYVn{8&=4J z0|OE;uw#J@PzMKN<4xQ4TKaageq@>hLxbzX8>6E`7khhW)0Omk2rYLr?36_*8Wu(+ z8%3Y)T-tmZ*H^8-6P$YbXCp@!M^3usnU~3=i*Paq3D4OKT~?59i8iF(?QmtO08Ms}=CNz)0{q~wd*c=K5xt<-)Q2vKS;nBwz_}gskesjI&v)%Z8 z=~YTrz9D|YTF<*o_|-u2KfAwl-JPl9D9WNbqg{yyn5(cI^Elb?2eKukp)71 zexAPzm8mC|*mit{(d)9+noHYH+#?rzqu&tiq}ldGn)Jv?v0(rSsmClt?`jAPQ?78( z8Fp3-iB(3Szgvc0xj%uY#w%Xj%l%&F&`?Tywl&)M19oA^sPFsHHad_&r~v>W3UIIA zmE?5d0=RTlnyyXP;3EJ?G}Rc06b+ftjG{}WnSlCXUPVp?gJuIm9>9>0$jr>l&g0YM z8z0xy641inSvE0*tD?-`{`b>FaGi0k>B{){d4=K~5tQcz}d4KEbJW-3*_9j4}F+ve&a_#&H{u_mf4B1Q(Khm1D(sTZ+wJPm>+#ULvvdJ; z7xkYz#rxan9JEb|?IdJ2ACqF6t6p@1nMmMG@5;QBIg!^4)(&kzj7yrn;ky`n$HzRL zGul&zOtuR$4~Vc!!J6tryyRK<1kV47+&*ln`>m%reXF5@Ev*^n_O?C1jL!9c6rE*Q zlua9kpIy3BI;D{oknWIfkY4GMF3IJkL>d&NyQRAqk!}!>ZjkPd{q}qQ?!P@|=9sy! zI1BhcHF%vI>Egv4K-Mb5*1`)Rc9j(hV3GAqd6{| zYy8QO_;)|}SkQabIsgJh06?9ANJ%)By`RwtKwWK+{R)o4CccovsQk-;0(~g#j=?8xdkgb5xE{Z769$uy{E_Y!oQ*s=87k zIt(Hp#eqQvW*j1$jFbShK`nlIjd*nHqeH-J=~BmmmpY{V6J3I;pDKQpg38Zz_YeF4M?#+ z5U>BBK5-gQtKA8o{c8Obz0(uu6I1<$2*QaM%e>q5e6uFm;eUG)NgH%`>$CQ9-QD$o zB3c~0ur_epy?(Xya@u_wug(t|WlrW$yRQ$D^%0_?>AjJdW=)v$%+A`d;5O2>;1GZm zFBvNF*{@jUk^N4mS&wv;9~5#`#$Nyydct=%suk z@Q+u3!wS(Oq5xHN69s^#2W6skRdlrm2OBbNMC~}(C#2irGg}9((m0b`kIQvbT%tvm zHVYEN@%4(~h!9PGB*f;%L>%0NfS}9#4aVs^Ekz`iMgh=By?Mv-YUe135crW%(Fm4B z9E=&?2>@l^5Xb)j7=X+~+1=9OTWSGgx&1Fh#h*DLE{A|x#{f|G!yL$%Wn?w136*@G z==N*;J0_}p_7@F;M5FP=>2d~p7#QJ;NS%HsXdwu6boWW){v0^)1Te_ZoUplLVq-@= z^Fc^jP<)`ntCb3gqQ8v&FPR9R+=EO9AY-Sk@-StTv7W~4)hH?FxU}}9rqHCle zUUBkBe7fwhm0MgEW!kp%s-ktP-!&O58EeK_#^jp{|t8NEX+;1VyuECX40NCPvwu923?Ver9n&JIuh@MJfgIk;YFPKimF!1x1#3 zghqic;CToAul8D+Z-&`g;BLWUV5=cjzy4AFW7=W7%OL~t%NZg(=Vh6!%QCbA8(DO* za*|DCc1IimUECDa6^4z$(ZM4JMiIfcB&~^WKpwDmS*=1a=QZixUTQwIlx^+4?I(}V z4|n5Uw+lmStDXLj>f%9k11tZTC@2MJv<7f@-|RI@_8Me&`@gI_jWj;>_4)>|cD?XK zWsuUAw;Q-tPRBb|n>`D)a@~`sZ!&eS`n|siitQUYY;iHR6h8bas8G+(S?~Hi)02Co zu>ESxIlb?9gSkQ?H8P@y17gd@A)6d)VvG+?P{lJaF~f|+!Uu8S^m$r2S40RJ%mP6)#fh9ra6&dRiJ&qm|LN(mlC*ZqP}XEYF$0(pIgvmWwwVRALLCL#hK`Uhmm zj`l^JSt&GY7PFbKW06=(6Xznpolxcp070W@PvZ+dY<%$T-@p|R$nfje1zHAdHk{a$ zhZ5eG_`t03Z2JAXez3<0HRux`{*sT1QqbKUvZ-thE1(1*`!^PID}3(){O{iU0vs4Z znn^E+BwvJt6LUs~zj6!)2=>2y+NFz@XPquRR_6WrI@)!PJootv=jVz>-irP8t!lcL z?%-}d@xTBv(X5izn`~CK>sHFokY&HIqqOXqnQy(po&g>zmbZpZmxd&>)PujlrnxFB zOFA=ss<$6}6f8%4Dd!&82j)|aD<*;?wj#SCKl)=55P|mk@obuEyCNChBhC>5jfbx=C6f%Rjz^XrH@@{PkTK1q4ur6OD=6 z+U6DO>uwb>(V_DqVY%V%l$MZxLy9hXuHz^-_O)WJ^mSl(Qw7=3ki-?ndGJ0~aC0k` ztV8|?+uZIS1yB6`#$Oc#XE*pH1=xwG0MC+D9YKMrewuOd%}wQ%icDV%W`t_@3hYJq zk@+l16P`!s1w(;#QuJe%YY8ui7?ZEW7if@s>^>FkoNg0CsBo9j{=f5)!jI4Z?wVdl zfjz(DS>4-wrSGb&mR*A6dB<{ltYzZ-{n=o_^1hr04o=_ zc?7Uj0eN`03j%5|Ycxng{{Vm}_bU#hX}MA)CQwZ+8VK@lV{%e62|yg>!y!fn;nAhG zL?+n`@wNDt@*XsY_gL!^tsy`ZeZaJ$ErTr9P$Q~ zB@j)VZvMfBN0e$R%aOY#=nn>Gn&Jun%qS+QsIsMU}1F zO+|OWb>9fy!10nt#3-x!`KId|Z7%8c+VBtPf2?fQGeD4CZJys#i;6OaAVme50Q((p76Qtz8`V*#m>i zO{vFR6O5oaW^j#HdWAjuEl>4pS~l44-;Hz59PXWV38j&<*dYa=3duI5@~6R)Tbm_3 zr#x@xeJ0>_lWWTaG@?*4{J7A{{re1`ljZx}jP^B?0rgo$4cbZ))B(lIf@H%Svx)N; zsug5xNEsh#ByHbh=a~4Gcpew^F~TiyUmp;2J+MBlGxsw`$z#7EYubuW%WK?BLWs=$ zduQW*N2sLapzw7>+e7SFYtLh7>5gfdlh@q`0ubb;zp~oBq=e9=x{2>o%H_PVjb1+-F%3B4I7bE0bEVf2%gnW^NZL1-r7-?(49yGADTHo?b905TGrQEMSE zzwTzfR@2@}7-ePg%ifwNy>A&^j2Gkq+z#g?P|>{vSLNS;Hi}s-Ubx@Ed2nM~2HKR! zYr>SIq7+aQdO$nhG5MgI7Y6cHfGt`f@7bGajFwJgEMRxf5vb8$!ckaBwr!WW#p90HbnEym*$M_O7XN1U*z@o>9B@c2?pfNS?k)VL!$dI!L(9Fv z@rBP4>j$r0ah}()!O%ATniVPZw%(P!W7#QZv@R`z+`M2LE-^knq`U5hR(>3@^?rBr z&Q;~N?&Jg&mXFA{NsWdxsQ^BH-S<#+q$WH@Lexwk;;yTzwy3^#95N}WP&=91YyOd+ znpcdQCHciyOd@fr%rhLKH~A`O^db2N@@VCF9RJJ*m~s+3OyoD%P1b|k`!78;YD%GC z{pPZqa6^#BxWlbtJ-&uUoZtt33N@zXKYy2&92`#!bwP9_(E}Hsg-IsjA2>TSHsxrUL#O>z4#_^dL)->efMR?m%(M^aPC|4PQQD1^KO8&M!NUX z0WQPdXAvIaOWc8?^EXX8J-sd7CAjtR<@xaPvhsGMUME^M?bq{|gP<)K_{&aA(ufm_ z4zvm40ua6_F-KmH@v0R4^5;7R+4&}=8MozltRD!a649`29hHLGyEI}R>rWX$Exeds zzpJWpIq&k53VznhJv+ZeXkF#_UjagJwK+}XOC9LzPm*1kSd)Nnq`p^tl?Tt20@k5) zJqOo^^QU5gPeLn`DQvXp=Qyc#hUo!-KA3}2yk1#1!igY}*;8YW% z9yy23+jOSwdI$^ME6(yzYW{}~Sa-+A$1Xkpput2(Yo;Mf&$RI{m7zugP$ze8@qQ$I zWp!hzzr3t(b@;w}j3X{K|4NuUMsK2Gb{@?b9b{>1Yg?3*-xzk3Nj+E)1X#|*)(>>s z_5>>1e&5_hYSTw$ly!aY7yd22a%{*Pi=Ar%X7uZ0g{K?^0_wO$DM6lr5-P1r$bb!9 zR)LWheo8{-ARz$Oqrk|^W)HID5Zq1lUjVdn%M5V#cbwn$q^WXpN!(#JHts_~h8{Bs zFr4*~_MJsi0fJyq+6Uuy6C4stOGJqQu+F}d-|44ln_38#A^`z*Fxg+gU>9Ay(Gqr2 zM3gS)x-^EC{u=YI0Xa470KDW2e0AC1(!8IOZ$5&}t_Z#(XJ}lyR8GT{}<#p2lxdZ@gs}R_?_1#OJ_xnq7os`Ah&+wON)^3uQ?^5z3el5ih%q?M9UOaJ|>!`>m);6Yk6K+0yI8>7uU?m%3_gx zmMHRnAR$Zw3;=W!bRoPPsccd)H0pNKD?3eA)A2N$xo?Fj8p4AI#fCUrg}ndoqa;t? zfMx|M889-_s@X0l9J9W5_A=B@EAcz__OOu(_6i7aKQ0hJTmOVlNXCXzt(!-#idFnq zmnZzIVM`=Ex^((PWW}yv15$r)S{f+S9wl&in11-y;d;@o7qbZkw_(c)-yZyD?(WrV4bBwpeX|w z=qE>ruVKK+__*0uqF0jSfwyXLRjuSQtJ;_h{cl%?OvYm(dlvk?jbe3e;1|cu!Qopj zr3;@O{hA*b<^Pz09fsz&UNdOdhDTQ$0?--(VdCMvOv3p|Eeu7kyo7O^L&HT{ZE`6= z4Ai&hJC&WCCGfp@@##~H+SQR~VGNDRqYLEGPncTd$wg<&n>KFb|Ki*8gdY2zp3E;= zqf)zQ;IDz>5$B42VT~&FS?<%w#&wBs@!@sPc#u3 z`?rsg)oZjL{fTF5rwPbMFPI#JQpenn#S5zI?nfa5@t^(M-C%@uN8=)G{9{dx;amHk zO)R{{)?EKIN;;>RJzr_`=4DfLP+(#NlhEZOnriMG+FXNNFWX5c);j%f53484m&Z_j z9>`RYYgq z80B`-9~$!Ut8Er;I;uBl|GCH5;tHCleOAsXASbfHGQOWG%q{<~&FW7j{4u>=WO;EMVZ_Plbg6LEMXTa)gQz8Hzw9PQTklL3gr^ ztb@Vz`Wir=obyIkjJDjZQ8zAOK_fVupOaS$3~Upj!sbiZm8v9p1h=<9)2&>~SQIau zX*@vy#6oGkodB@n`^`YHG~+pPkmZ>GnLmIQiNY>3muOJKW=0y?V zRuBKrK|!Y{PsDMBpnmQylW(w_S`9V+`NTAih`fFhQu_C-&ac#PJOa0Lf?qBMXoCX` z|F*eZj&BW2WUq7vdEELsINWH!gPv{&&KN~u7BxqexOOe?ndo53_yS9xPHInKue$?3 z!*5>h;%;{iZ%;L>D;u#ziv1gj zAM9MA!E7`W>Vo!s#mUEg$vU>Ia-qGs^qBGI#c40e$+Ir-%jmM}E1Vy24>7ZjJLh*K zU;2kJ&5DN~eI`=imns#tdiAl`5TC*N(ti!`*OSG$}GBEhGY;r7{Ay@RQ+%(O?}mqd@B21LMrI#1J~E&Za&gh{i}Z-gCcI z1N79~sCL?304;$Ypo_K&8`!%h_yZhYdw_c%u!}54quQGT;sbU zQc+qHL`64qz&y8n^Q~q2cpLVxlw~@Warv&h_3`dp$H}hy^4~sf+-dfS_`xezul$nT z7Yrw)IPUE&`lT!LuJ%XOtO)<}qlKB%Y_%nhlEmZhGI>ZHGn=O)KEcnAYw)V{0X&WB zts7Ew5C9SsHuzdBl+NAwj_>O)nHEQLBa`x&*l=6aShX!R@$l-K;HRAt7aGan=mE*K zuE&j&6~%YBxLprd&yT?!pFQ@f-FRKu_$rEtpqi*P6^0ZU#vcOi9}iAO(Bf(2(ahjE z@XMRy`b{=R3BS&Ee^N+tI(v9?EWx)u^B$=Vhr<;p{B< zg(AD7&|7_6PQ_08w-Y~ju?#G}(?{N~ue8IT_EM)-=O<{6S31H8ddE5!#@*SoWQO$U z=}tNpR=ckY@|x#$Q7X{5gEh;Xay)f4VB@ft-o3r0GYp8#_BJDxAo_3k<8y@Fgf|l* zshqf;b{Gs=nUx*Z!jEc>cCb0?sxf|b?#qW-;_UClll)}>9qS>CF=u4w+rMRrZ=ukL zrQpSP)*Xpwxs^3(S1f@jHMNP!eSJ}6)!%+Y2A0#OY%VTH)V#j&8Z^f-l3dQVzkAt+RL#dk3r@WhU0J6n6SpRgEuD$HH99%VZSmjA6*A@^d04?U zwU8C<5adm>u>kjN+KIR{FD)G`DJxetM*z^^e;a*$D4C}th$|CPi<65si0&G`M8#^E z3KW-svvXgA$@5v`w0u4MMOjMJ7D0NLzhgRyA-71^*OejxxoYO~&XaSLi;Q)uI~UD= zpmb!$WcHVqggj*HZ&_HdF=bgxYY7roBKM&%#jnMJp=K&IMN>;_Q%2RFrW@@lh0))N z5apG|sZwF<2!))M1R9!cZhfw<|5IxraGTD*Vb7R1FaRMN7?`=U@i`^9**Hyf$8V)Q ziTNVbg4n;r@|ufty<8?&9BOL;=PS_}XIXc$H#e(d#^PlX4;zs_E^9A{E_unst?u9; zY$5mOhpfZ-HOaNUs6W;3nJ3efr!pC*wRJjKSz$fOH?2kAQPmoa8udi!TyfVryq@pd zo8dt(r@`Zki#yfYP>=3mvftJNpjH!gbi%z8v=Odm1GIk4(bqj&y!I{W%|TY9;; z+)SC>9{ISy@8S2s@Jo2ds*w2o(?WE+FqD!WbtBZ%b5ZWnidR*~hX7lG`~2zmNF$BT z?Gq`@@{Ja{W3#A|2$u(lv$=C`lM;6lRKVw0< zN#0F*%UrT#L=}igkbj2M3E?B(TuTeZOc5>kRth*?+Yhm)DOBWWm=D!&HB=21pT}K@ zF{D7P3K57eW;2E6>i+Wxb~{6&-iT4l8x@F(CCt=)VhL>lJMPi!VL(IOW7g-`b$6wkvut^OACCakX^ zQbG6ZRY+K(e$bjKOQ=lHvB2ZJ!b)Hu(xPTTRU%+JIQ7G>=UL3z=hyJoZ*t8KLZP1d zc&}OZZem4+#!=@m=7Ku>WAa%4!ZY6eyE{EPuIw@lzm3SHx_6Oo^RHIb>Kye9U zp=Oyq0R0Hwe)?_G+mWE8(@4h2oGb%CONzcPHIpv7nk>=wmU+k%sj*9->-7Az{F!N^ zlEiAY|NDh3Ha6ax1V6Dnn|4XPqMaR7njP|-%7Y9m`Whi3#q>swDzkRR>fq?e)APnl zB;uZooTW2qzzanuH4ECPoA%~k?UJ9)r~TYX0iN}L6D=#PKREhB!1zEZ_3QHaFd08u zEgQKZ)lNLD-$H{*O_IIl`!z@)D8dDCeyH?OUlsw{Mt{`$KxX_Pgt5`Hx-h?Y?9t#O z`0iZ>BXl@q{5y4}S3wa$qri+QPGB4|lN;a)fYnQOj=tzv(@qy+!thMvz$Od;ehV@b zkNXgg3#>nhJ7~p1=%w{bUjqPb1-3aN z@cZ{Jm&gN<=8&Uk|1wOBnUu*d#;BEl!z&b(Hr@+>o^kP zRAkKD5nH!yw8+~JMOG(nIoUrYg3gankSpVJPC6HQab2&4%mS|l$RI9_u6uo3w%lJY zyGB+&>b+JP9OdNZTKRl)G$xttd~DkNvU@(GxL2Li9e9Oq%v)UC1vCYpy>!FxZ-XV! zSV7sfw|2Mtms@AgjTk}^2IF8*FvZEoMH3C47OSeCV6}*nn)YxzGW=!lqbQqG25w1DP7Pir z;Hb!V)%)24h|_uZVXicM&vLk|NFXV`KiSnCMLLh4!njb@lEQ?lULeCLgN`dsu6)*c zhJXi|ojO1CdyyX_9Iv!Y=lcj;PEf+{KB66eDeqV6 zTkD46S)WLW9v>Zt=+9bImnh_rd!QDnypJT?S!BrH$Vh!T=mkB`@oz=yHiwRa>^v=e zk9*da-OqDf0c?n;rKP2Bf%<4%abf24Q85Kp-|!y~3ia?2|oH?BX~ zjbsEcfD<%7=Y3;PZ%8DjE$1&H@7jHe(vSB1TirtiL<9Jq;=&8*P1Y!Q&4CFf-int) z8&K1HkGzNeFhNsMTiRV*hn;V7lw+iTGqQR%w3a5;c)C`(q;=-;XBdvntl z%?3>va9M?EEcwINB;iko6B9HH&ntm#4=?w4D+OBx1)o1{UcKjfM8F@Dgu{5rqX$cH!(Z@3I$Jhn--7B|Bi^6pHpiM zS|_EaT+?|ln8Bj>uNKp@tW>E(Z#~4a`ta!zFjGcct7Bg7GVPJd?FjRx{_!#Nm&nI0|hCW0e=q>FK z4UopFy-10ekX|FDFn4b0IlGhNyR!wDjr9YQjrVyoGxc;D-$S{gF!7mke^~P6(0-L| zaIM=P+qt-pYK-dHd~d!sJninrxHUG)aTZc_<4%_?mL>N_mJ7&=YGVO}T9Acdj;E zWiku{T1n!cK|oa$e;RL7eh|mj$j0A}S&dQJVDf!`+qB-7T?r1Pr1O4a0QNJM1RY5V z6Sh95bM%kX4e?^IA-L29C7!!Ubk3!tw#DpC>(t?4|M3xM(j*)&>NRh-I(_|qKAazr zg_2={f$-sn1^-%q>uKxa8<9YM#1>+KJ_7=ZYDVEa7-Wnd;Qt7}k(+Gw-?0LKzQvP% zpuh1RRC>+km1zgH4(Kl?KD97;O-?qSg-D{ED&v&}jt!Ubl_TPU{O}Y*l@JO(G}8t_ zD?w5S9v=uLgQ!|DpP(W-wr#G9*@@R_Dt&Ab-+5<7p!M3yA#Dik3B;XU?XO}XLv7z* z(dU{4YB-sTuXadt{9V$4_pCI%>OT#7bZ#!ZSwfUGa0PsgLvUdB{3lksvVw!85Pkgv zmKwo9k1r?*Ap`MiM_p^CeXi^CIl(~>c86XY*C$!?l47)!<17LCml1B`IC2#E015yl zfK3=-hKC|G;^VC0xQs`~ZFp}@pErBhilVjq&(n$rcDs||8sIfGQRV-f9{VPj3N8dg z?YSFqjM>zD8Wh=AMi6pO;>Ppt;4!L8i-^_uCunMQQzWxn@{gs?rj)eN_y@&L3?5qy z^*=5|QLzE^zckdB5jq7$^ZU9mNhPXRnDju0 zQKWe$Cd?9Kp2ds_OcG-@{us-fJY$sIM*58LQ@-{vlbE|#N8In}PLpmq*<8bd#h4I) zTe&4UJE1epG0%Nf@$h&B0pv%kA}uj3*-BjdO6ATuMrQEAnH@nm2tS#ipm)$};2iSy zHiF1CI)|rt$%B;;(J4{3D2P@eQ8MuwfKb9%QW@M!GyRnyditsyd&!;`X2K`jso__e zK_@pOsd$?gv;dU8!#rML;*zSWB_uxW4ZIpGW!v3PSv1rp2%`p|D>lzWtzmWHR(2|{ zLn(_0=XYHBAIQ5kBoS0T4EpGy?QFR~FTr&}w3^GV8HCFy@Y#EK97kvZknyjGgfa?Y z!3qK{wzoCo6sPhLZ`0Z3(VePZDd-nkUx{zIV$vx`*2rd70`}X6IPswg;~;t|-94{Ewow7^M!{wgk&MT7J3!;xKij^{1sUMuUVgcV?wb!mJ zn0}d=xD5(?JVGIanr;r=uEMV_C*p&HxeVWpCZRA2Q&>N(1 z8Wt+CKjG1u(4c6uuZfod^3!2^@Q0nAnLWh&aw53v`7vdmjK}dsQ$H1A=Kp%9fw%FQ z+LlPre04R&_>$cb`$DKolsP$32$KTwX4vFXX=0;F}g6Na2tvU;mR$kA3gXpoMqIXyX*IoGx zM~jJ2-|-$o(Rg%I07FF`c9ugjrcr z@|lMLH9ao~c(Zkdp-X=aEdXbRKH%$3x5xVN`u z09}gmG>}%o0H*4nQ%~;w=rp#m65swMvtJ(#0x(Tc`RIaCT|KwOL`B8K#f2MHm^#W9 z^8C+*$iumViNOKLfCGLEuwKs8TQw_kfY0P6`DVuX=z#zGf4p(O8nZ#pbcvHq#?+#1 zB(};mH0h`ZjY!AdFfqr#t-Bi{^qpr`ns1YH#?R8)GnjsepklsrH4s>8&Ue@flcQRAr zQ*}D~-^(o{{MPLH-bdeYjO|p(6lmA&t&qB0>kM@Jib$+t*lL=x(Zld64cKS5Y$f5B z#~Gvp)fh&Z-9hIkD>Jl+P`PZ|Djn+G{a8+R=kxtOwd+Cn%BASMkA{Yjgrm*B7IkDz zk|$7k_D7*8Q4z=4OdM?f0}3bVMCGD7-85lP(EP>2u?GCBmY z-Z2!?vUt>zNZpyOD*iUCEzVZdz)wUiu=FM)!vkj*HFSLXT~pIIJBPWck;(GU%wgUp zf43?`4@cIA_MH7ozs|9@DkC~ShGo%zWM`%4L?%GmG)?|vSWfC+K+?b1@pTg4w3NX- z!4A6Hc3BI0JkEiO1&Vd*VkoM&F?=+wHh&*jn$iLQ7g1q8@S>#&Dka4%PS!JQP#B%X z38u4|?IX<4#oxa=IaDYjb#S-3(L4yw%fiMspU8yP-TX%u$c;MqH)80O@i8#TrAs(Z z&8pHynXu@s{I`khmzvN{DVigB;~YjcS>91cET+n(4M724tnYn&!zONNstbx&((!*- z3xDMO2xiUbm#^nArLHi_te&XA(Q|zfXTpX8NI!)+eM}_bZMTTqZg@J2+ zVgfL%Z-fj;-IBtkx}4})oyVx6@(v+M=Il`jKBVcA1%HyDe{|neS^YF)LITnam+v|F zH8gdOHBE<)M6TmDc)akB@zV79m6E^rY!uCkYBR~|-4GX&Rg%Q3BZ{kkTgXOIQdk{S z5-r&5S^FOuWZ!9?UTTEi@I&9D;Au9Zg(-1!G;sp+XSe(tB})8r9}!QCEUpj%-7?0H zypzXsX`jvW}foE`Ys z;`@YnBl+x1Wl<`!umFFHq`q$RBJuWMYrTcpJR15Je&v67LjFD>`g-HoBXsE_h5zju zVy&0JnFp5k&xQt%QBnXG_GA8L<+HDMBP-ztMfl1unJ#wIF4Yllab-;{&0DlTc&93h zA-@>cS6o);X;oOA3%b4gK_zlEE;2Iw;iI2-P*BjdFJ?X<2MB4_f3`2HS(r!}!Ditj zrn8HQx>wpvxXpS6Ka_;uyN~%l*K26?7S{ZkV#RRaRh(^eYyIohawPy|2yz*N znB|~}o)-e3%yDm_OxM3!iE#}eo>Am5;NIZr)|O%mp9p)HL>26eZOp#$^EW4lGcH^D zVYT}dzKwM4n^4=F%$#pXG6(4@i3?UQuUGlp5`PP0$gc}^3Vy8hA|=_iQP>rXUWb#S ziu$5|(NHOaV)w9rFyuS2ESssFSuMC6Oj`)B@xB2Ag- zNPJiV!$on~A4MRct=y_CPb%T=?TOsX_Q1_zB%XO^hnlCPpt(BZJoh}B$(s!t_smaH zJ`;&D!iW;O$qbQ~ID}+6{2GZ4fmG*h5 zk3WB-7ID#`q^M)|;r8r(I4(+h#s;llZEGe$LFh1m3nc^*uR30(9fszZ&_gkSRMGhA zL@_DN{t@+td*E|g(zd5^eln4ksO)xGlF0i(KzAxKvNJucm#buIYJq2l?IR)Os}CA`6 zII9!&ks(4@>l1M*p50i3z9Y`Q-~si^k(2?LDhBV)ZhNXgFX=Wo-log{r@{&&>R%AHUm4C>&oe_GCNa6eq$@9T}-pFFLtiU(g>RJ4UZ zJ0a$Z6XqSk=P&D20bbUQUZ?Xu)Fq<9@W+gS3gelKjEU7S6(g18`rpOt9fjAj(n^J> zn*MQudIHqcE7Sy|-9A3Da;CziH|bQBI6grS`zzHh^r-hY)u*e$*RwfrzJ5O|<{z8O zoV_Upn%o10Rc*vmXLk`jfrV`hO%hnR2Hc3k7c`sG{o>hK%8E;yvgvUj(&Ffrxe6hi z(~{I?iHwyj$?WdNW}ht*Iq>DcyD#NWf1C*3_Xg@K7?4lTcw&>LLt520?1y~dJI`2b z6mfPO$KG-RC8fwF4)~FlxHOHb7(74#-{3_MYTwci6IJiTH@Gz1B+(n|e43lrR+;^@ znF_#{ZtZ`#1%+qUO;!#&uM|YY$T-@b4WG~4O~_i#jj5sm<#n1jHHKyhg@6r!d>1kl zNqYY1QN78lVDTzERCIMZ*Pe@G(=SVgz$chn&MoRLk(ad=gkh#BydOWb{IO`rMC4RR zlrNbL%JK!!(C(?jb7GxH&P;yY{zs3wZWyq#0?O9EPbd=JxK$mcmzVTp$?+F!l7QA< zHb3byb331CR8^-|n}VS6gE+~;K=wU7o2+JOi;(G2YkqLxf&oRQ@$v7B7B99X>_~LS9 zpV9O*TZI!=@scT}#{TasHH7kp<-nJ^OvzGIO6nqk*`97hW3V4W{JCTXsztIlbh82I zy-TPmMv|t_QjY?sarCE}uT5J7C(-XAu#RqpmymY@YUuk69QsOn73)fU^OQOZk-ncAL>MR@U#`-q1dj7B*BEeVf4rQZ zFafy}0}Zf~+H|=818V&2>}6%A=-zsh4o#AsN?Gax5hcx9X4jNy)~e~t#{Kz9=*+P) z!qecrP0MWtKSRBqz(lK#Ka~h-*9)QBnD3ShI^S$Z%;6;c2}MOqi9$Ate@3}g`!HLa zh>(w5?!JFFzP14i>z@m_c)0GlA0nT%Dt{~2^}JEtuCXTeJTNysbU*TV??iFpw>mJB z(i?O9;YBine_<4uN!mXD{ySJnR@igyiKo1dBH+A;A)AN*~wI&SMHioJbo z>Bs3Ehh0o_(#r*|Mki=bSHL_iAzyQJRP<42pXF0x$z^PexUkO93;?Z{yLE2I`>COT zaY+BGR6_IDv&Dv=hbmfL;uXsjq##F2sSwQeLJ z*Oh4?k8wEU$Y`i=3#G?)AKjn{&%<7jnyBU18iSO1e`-X-IxNkZYi##aV zUP7*eHC`kLtSSpiSmLcrz_(;pj>15vJ_4j{;_8P1xYoa>z7s;%WV-H>+^=%zIGFBy z#0Nv+YZJ0x0$ZhzjG`{nv|PR^7L;s6gyTaQk(E$$_0ffnn(Y0a*a2h=z>^Lh$@4{v z2uKS1C6P&F*o9jWsPF~MQ5iR`PhCvB5sBo#vrZq4A_FC)BDCg~hIk`BBXMzyLI5oF zgz%t|(G}rx1bHR_5lEeJtbA#SC+QKjW~17wcb5=oOfOnj(SO4Qw}jcB@A5D|tX7mse1CE=RWJTo z4wGt;#MoC-J9)Z}Etx&={uK0{^TyucUM$LcB~+24MkI%c+I6> z(W$3@^dFRym*c(L>NW8okMyp~&HehliM1+)ah{rb_)S&ZX5hqv^x&QI#H z?EQtUvd>uXH-!-{7u5&tu0e(uCBh-=p^nF9k} z5?2yOAW~eM{=vkVE)|J`R^Xq#b`BBp*;2#j%>e__f@D%sR>#)_ECt98Teb65v(EVO zj?|-%o6T!M{$!N->lxfZ?NBlg)-<)@+uM zMD+KSe=}#Mbf|Bg9F$u<__1H6&&0$V_A};5nE-&ohWh$6DZ;g_=OmBN!>_*gJ-7DB zV}o`{+N?2rvP$TMz7jnT-~;x9 z=EEgEaL0;O3WgUQjZ_3nc#DzGm93owFRE}_Nr>#L$#q!lIpl*ETYMW$Z76ikAV z0T>8)KQ7n+uP-2Ez`zH8kxk9d%`ssHNlqjGR6_!0@XB|1h=E%Z5OAMVYb|98ijuut zdbjI?r+y6U5Qg30k|Ok+DbZqF>^?EKLx&} zo<+*wG{JB&IMI#QENW-8@cMY&?(*_{c~~DdJN*38yZUFV8~$*!Fm(S1BCf`Jr&^!f z^1gH^{gtoEEQWg9Z$np7@fAv%xBso}Y&i3{iZu5|eE*u3GAGCtP5C^z`7QV7His*9 z4u0b6RArF%8de$`|Kn%X{RsyAqhoQIORg&}PI*kJyFG9A83xKNmGDY?Vm5|gpYGNW zH+XF4z42we4h%~lBo@cZFd5n$r-_Rio;m+1SL=ahZx!Z#cc&>vemf8uVZjm6D;=WN zUqiIP&Jos5;Z#wK}uc~0REO6#T-*rCzMhS3~N@!ZNRQ#O+S75d$z@Mz<~<^ z@%N&J>8`u_`xnnx*rTArHe@sx$D=gpkcEWBMP&d2R}&7(9IA9GE7 z$XPMwYRemZJHNj#8sDsI~8q1!L_ zW%=rGDcC~e4E}Q8d3o1AGZF0j^m209EctwOw4ZSq%cbh;Kftb{q7v<~Tbvl8>phK~ zsLoE8{8u9%vBa~8WGKW@g3e1pRT%_#*rx@f9GZwSNGDfHpc3si=Yd9`=S(*UZ|_ja zDk#Y7;K%4i)h2X&bv5hMS(P-ET?M5E&WZi(GRmE-aR$5J-tQExC%a$!fp($s!Yu{T z{%@;HI`gQf6C31_Odkzq{9Q8)9+^D1;>)0=ra>{nzd}}4?Cd2Z60ef@r?j{B)-C^O~7xWz%S0V>Lw&B4pC@^7_SsQ;nLZ@3HB z>p_Ol*WtmyT^J;9@9|Sc(foTRjHOnCoA}-2mJ>N_6NOo(I(KX?F6Xim4A5i+01JAL zWC8{fKKqhrpaTu*~M%4AKu)%L=eG*4AUDg<{5fp86ky_HqKbPed{)x{HlMIZ*sbi72CF-=3`5x zMO1^L^}WVXZ?MGw0NFq$zfZZ!qMc5|b7DB`YnC9A06!a3$S$IQfN6w)sz!twS~4s9 zxOmj++gV(mFWRxvA&_kWDaHpef!;?;6cyF&WQSh;{`>C?7MZD|vlHU}apvMF7=`9p4tQ=tezM#eg@Ngv(nR;62l|+}S28f;(BO_(I0N)#FSXwDh$N1WyOS*3 z^aKDxIn*}|s0e_Yy(S;#aZA{%A(R15F=wY83p@A0R>?#R$w$8W*QlHzP zhjM&9=pq0>wx)al0N@a@Bu{7aI^c$JAaiBG&dRAkkHN<`H1RSYAPz;MnV<{Yp*Qnm zKJHHnXMcYQ7rgDfBT|S9n4=Nc^`USfb3rz55rJfvWY=e@6%JQHw=7U*s(NPKhmf){IPAhK9g=;N!>GH8aS&qd5DeOE}5CFhOXCcBsp3Th?g2QwS_$*<7@%@uLl&sJRpPvEj}F63vO&|_|s`?qD!aKQ7MH0fR!ty z6dKbAH2@HVl40h^jElTZsw*Z-*O5p7ASI@8a2>2x_Sqp?DDhn_i56pfR~sYY+H_nc zFN&%xnj&HI7=-+&(#(_~*5Di4(vk$uQ3f-PSyv9z5RL@{x#jumXGrf; zrM{WDxtRe9u?_|J$;L%hAt0#sxdoJ%cA&S8qA7i*e*<@5`x1^e5z`z3!(;k^A#KOF z4HM!HxStRJh@R~7t*1}ltgcS&?t-Nkh&Bzx|4Wtd%BiO7NOBtOPN&^|WIwZLI<^Z) zk*Bu~w$H=IPp2$6dZ)J95rYG3FBE|CcT}f~&Mt{O_N03E`gb<-?wx}HN5ku%(Ek6| z|NiIh(#g{F^i-kp{_%hPC9fZ_l0uZhL<|=sf zo8NuAOq3SEvWv^pH2{D|mcj3nJiso{g?Tj}h~HO)Kkym$2iqVJN9h%p5=uZUDavH% z2sPpq8kb!@Bz_Ga5~W^?(f+#}1a=)z@5)zGs5 z186@xZIzCWPRUUsd4N>v37p;()1r+9SmvO}2O(a-gcdLWJthHx0iOV>&LM|2&f@WXJt&S?7-pM006)ZH^>0&j3~h! zCZQw<1EwQ9DTQXv%wsA591;K^4W?x41W}^IWx@iwXpBR_mxD8gtAayz?B!_cWfmBP z5pYUb@<|eh;sc~^3zUsMoH-=z{c?I>We2kwrSTi5-@D+K`0Jhawid6+XmpZilnRzz zWwzFcO~Q{@Oi4PxyPFLLv&HQLx`+!>(x^(9#*!gC(JY)y+J_sj89(C}k}3lLze?Hd zPV9fQ4>+B@VB|KF(dj%pI^w0Hc(No(<}+}@5K{|s&gz%A0SyX$oJ0zHPj6`SY0oFM zq$imK5PEGLHf-71jg(R;IlkeZu}|N4`i*4n(CoN#q2r&>{{Q=@f4(WscF#>ujg7s3 zJiUB#*w3|Fn|hkYQ2b$OacOC8WMOcyK7g&fsVAH@(bC&%##}loObG&SH%kVIBNZO> zOG2tdagJ`3&9`x>?^8(nud1_HeNVL&z^uWXi5A>37V^NQfBM5`VAfNgemXxjfAeOs zsQUk6ahi1hPoww8guyYsnpd@+1u_)zF-k4!lg zNVoZD?r3AvpAurp$qOxg6wLTgi36K@Vei{&<3;~X;uyp(B=^2mda54sj znSduxz+(w;u_ZGTP>!B{lPilQg)G6K;SS3Y00@2z09X_JBLi>|?SDLLG3R+<>-;oi zrpz?YDFgtnsg?B-X#~vWu>F@u24KK4Ab(W`jMV^O!8S4%4u>P%{VOvEyVD1i^JVEG`oTy!DHsud8n*0gOTrWtZAWETVIlxX#0)j*dt;jExCc4?sf@1a<2I=Kfvf zbym@JQ4$RyDed5K>sHp6>v8Zby|E!GB?#!UddlbX;W!nlV=%D`006)ODUDKEr4oh^ z!V@w$(N@CkPqf)=4)umYjTV!pLDOJ&Xl(EtY^ENI##RkSBF8uGeq)i8OO+Cm^-PsC zq-b{Y`s#piv^@nBf5lvxy&mSXnS9h8$}GKw7~rR0|MB+i4~6PVrLq!!eJ9dA;G_AH zZy;))BYWQ#)MIWf;Hs^PBr=m&UB)_U3T_=mus%rz%4{r#sq2=`J_kJ>iA2H?4tx~Y z|A)2BNtZui_VddC0ATb80O;&7TYzoAB=%RoYgugt zf=8EgED^E{2ARfILl$Eb91wtjgw1KQY%oaZypTTJ{`}wn{@a(Ef7{#uU;FCi%OSjG zqhJ0`Tzj0US0UH5h#{?`0!#f+Z=7xcr%l-*=^H- z(epB#5`l#{R0#0?hv3w>t*lwu)tY4O$&qn~@y7uurpbHZo1OVh1+kU=#{JPrI z@`u9{a1k_c`P&$H?}~}8YXAVBT&eCSjQk9TuStU%saWn&b@2un0yr>64m|0!jUNpzJXk>G7kGn*KvpaTN$82^jL3bH z2s4=CO96Sq-Q3;po*S77hbbY0i*vdKuzvzZ-(TW$N*w5`Oiyni6taOwM}umkWI-9I)2U|ax?V>p06Vb%boSKYJ|5E(4fk#ZP3 zJ4Le6*hf#(R%&Pa&FBAq_wL{S{`cprA z20_Ny>0Sw+bT_x^scHo-Q4%z>3>Ha3{39{xB&7tO_H@ipc2^09dSg zV*2$806wAvH2@%gF=55p&$j>oPEUI+2HFWM04oaMiXf6TXL0Q?CIK-A;3J(tG^qlj z_CgH+Rw{EN007fd$Gh|MWJv9NJ6yFmIR=a3DZ3+!(^FzeWdHyG2|a_CjrAwhxiQbi z@Ar`dOacSIsbh*v0}5g~%|isBuDnx9CCZ6wkjYLcaHiPz>Pr1j`zL#kY8lL{7cay} z+F*}_BmA+UzVQKwsE0q@g0Wa|G8DuF$E`sn&~91ljzzJ5oY2He{_-h1@4 zLn);`cv@sQsoop#|DQg4cH@X&nsJOq;m2xb-6fii8VmylbI<+C-0Idr<&3_UW*c>8#=nj;F!~Ngu88n zilEg*#{OkDcjm33%RXEO~_6i5@VUy5rSRlLjS_J@L0AGJS ziDKBHiOgUR+vN?#Sf0&UB{rLlCu1cADD*&lVj@ujsFd%m-}^eQX{~OVYu+C(P$1@) zeEO(k+}qdZ?TNLFyT|j%p|l|j5dlwgzBHU*RF$H7n;gYLsI}b;7q3HOv)P)gbr4(X zsWilp0Cp)U3oqCeU|25h&&nL!#Uo$< z(^Zq@>@-BRn<+m-HrUb}Hmg}LrvWhuXh#Z={iqL6WdLI%4)@H=Tw!Z>3iXtaw<_CJ zO+zomm9P@jwi0D=KL>*&|1FL6S}=c)M?`t#V6B2D?(7f-P~SqewK+)xwp~I8&5#@b zUn95?Y$^)-k)jWn9229?cW@v0ui1|?H z86sP7ZK)KlaYM~90syX{AfqP2!0xwiPEQXxiQbvRbo`2)u^#$Zl_(&z$O7(Gwv9*u zzEp~tv3JsKf~&%V)0g1a;iv|#sX#3ymXt@CD*&j?R^pm&it4}|*`r5y;CQr$W5I9U z)JA~ezDKjiI^i+O+yEzdYUgW;MIRC_4L5J@J-Yks*`r=s=Zi)Y65rn1Qk58f7)v_l zhL(<%aCaoU`}9f4)Y{VGjdVzXTr3M=Po+4_4L*3V|KP#GV8)yAdWVMtE;f! zEv8OOXP?H_Z|j#+vTcw`N!?h!9}CafMi$1xI^?DzZ^qRIXoycr->c!-HKm7m^x|lI z=;C3LZrUjxU-JIx|N3SM{Qvx?pH3m|FJO|77&_+uNbVmK{y-N#dKB}BzyRvhpahGx zLh8vAm7|m~N3K}2>5|=2t)6Utqq_e;!HNF;rSD2|-Ie|7mtVlazxvm2zabDD-hzEB`VzoJW0Ve-&^y|2v&LRMO{q@&FnZd#Q#l>hopG=PCCnu9JCQ6HToK^Ri zfWzhFbQ*#Jd7U&6L>mlFJrgKnN)GHXkoF(;C4h?cHD}=HD0u z99-ZmNdn?pnX3@s8l0&AkpsXOfT*;HqWygL{q%VgF986UX`E!l4Ifs$HmbE^2w=v3 zJ#b_c)&T(kVzp5Kbq^kMI0(TQ!#Ry6Tn4Z^HB~`T;C6~58-BT!6}KN70!k$SfCQL- z0DZBXlwDULqGtmUV8hqgLJc?v+S>=(?RHcFB5HyR1jZcRS)AR@Apt=DFOb0(B6IRb z2=I_p@D(GzhUPA_!7_1SWfvj<0w0;3QFw|59?pm|ypDjYwnrV+YXU^XjtmfyF&m~= zGk37P;_NayOA5GZzZH`JskBoJXa@ksTo{ac6eZ;;)0UE;cs%6+CgvB@2{3iutkv0t zKnhS90xrKFqJ0QG;N$^&oV)@6!m)q=NF0A;v^bW4?%!O!5(KFY0ps3^J**TSZ|&aw z;qAM3Z{J?0_`p@0cMSj{@U4%YK7RbPf(40ug5h%^W2YptQ8Pv9dpa$rBbBxOj!Log zq^B=D(yVcmQsqp7NePLamLvE0`0&oKRzh;f_@hBr%&TdjzfZxvkd4QaIMMEpTeYTU zueSpt{_t3MY_7RGGBQUA9T8irk4HT))f#KmB$XW_4Sb^y*YCwIk}=*iU6c5jp8m_< z{*G@qiYH?>Zg7_bfFX23I?2M$(Kn6^4{0$IZGYyKX`wz;KzC0 zwBF&K)cPe>(t0z^&Hepe-qMdiFxU?`kg;Xfyk6WPo0SEtwx_v2T-gdy5FU0#bqUvV5YlIXdzU$VmbF{U!cY<{LpZrT8Y^QjFx;C~>@d9ofU9Ys6lur!2mk`=3VM>B z6p;Z$%zVjMM^jh@2g{?a+7^eS&Z_nSV9?6?)oGxtdb`#-5a9+ljwgdbdIZNaE88N#Y<7 zr=`+R3pN|p*5Rln1#B0t>FmSz!b_n!rwI^95;5P4sRIEk8=jOrzyU`nvoRU|Q_xce z0Kw{{Y?0U`D{w#rc(c8}4L6j@0XfO?CBB4hTSx*#adKy4BZm8Tc~WFY?G#!mNm=tg zVfiQoN@jmi7jSwDtAl&T)5kaN{^f^vZ-4&j?Yp1f9#bXHM{^#Cdi>aSWoz&5okw@K zF>#yI(pHC~UV>M}n$2}BosQnq=g;RiJ5ELfp`o)=W7G8Z@Z;eSpAxh>z0{=50x@Dm zHZ$U=OGJG>ZM4;5>FjI(9@^j0-#t?)fX@$K9b;q7org0sBO_vINgZvktcY;_&?GSv zcr6v97mu-xHwPXx=}+?6-mTr)Y1P&L?c2Xxnt#pufg9lD?=eCEQ305`$^g!O{=*MH z;4lFS0~hMQziNL{I}THqM*y58Ud@uXNC?1}_i+{M#)f>=mF#Nz@~aR*T=jQ|bn ztv+&hmG7;S{=u);nY)+@?=IwCLtJ(zzWogUhV z0X7BXbn;4T+qd)kMcUbgia^)`Mcb(oNz&myX$AoRKJlaVdoYBDo4o7^s!`QP#!r%e zN|4OOkj`&4@q!-`Ng%cKp{izpiHa=zG9Cy(Dt^fEL{tZSiKmPu6hTZ8uTFZrJY?y% zC^h1s(i4GnHsX&LSgeP^&&P7qdRbjo8bk@wln^uWe9SKkJ|6>7Jr@1|AmH0D+mGi@ zAgT(IRcC1-(SKUPiC`k3i1h$^@OZb@1zf=Q)A)zHM}6SEM?d@w#{curZ-4&wKoV*v z@zVZNJ=41oj;*dfzS~RL>#bJGtaIor680+5dP7T_)qMJ5c5`|<(nvjtni@2kPV?E> z?zEE$l(a2ceLC*bYAq&B^GJ9kf?56W0)+V}3O*7h@5i>uitL*8!$W6BQn(U!2TI-3 zwcvhf1I{Mc=l1{u5TUnXGOt)%YV##@zIB@R9&F7X901!d9DnoIWy}FyfdFRuX4O!C zO4R`(#$QzNVR!WgVFXx0IM8va+NdEJg!n#H+l~mq0K+ROSFmKCd-<1dk^h7J<3{MT zItYMoWs>2$c=g+>{HxK~@1}G|N3DlP?RNLc%Avj~Qt6mE)%1)KPoD43gs}HX1%P`K zbA#ae2?6+;aDWAiWyOsnLn5}HlC4Yd_v`q>WT(l@$h;UI9nDgb!Z!2|W*oEAiNQlv0-=lmYCLJDv8l z44jQehc9}9G5`i|#VG*XTTI82u`+-Nn~?oJRuJMbHmS(!GB$91ER#wpF^k0e{eFek zrzK4Mi4fAr0*kk1I>w~a3Z6y3B$)>&kyp2IqZSHBG1zJB&t`CB54Ytl!JpT!;mzeS zJ$qyC_$_X}{^_TmU)l+LF~2mka|@i0aDRZ<@8f0_V$o-K}y4EFEneXPc|R5-7m zp1^BaU0uF({nM9KJ`(8@?f(yn0_+SZTx9_f7_cdzIFqhve5v~oQkPp>aIca$6y0g= zU)%ZpPhb8H=I=uQVBxwm0ktM{@8Q*v|Ln8DSm~&8bGv`g>+Y%NwzoS>E8}agChpz4 zSR3u@&ot5~ImpLQlrld!@$l7;7i5A4|95Wpeyb^i002x7?#+01{XzwZY_*B^%YHGN z)9UD~Rk3t~kN3vpn7(-sc>myF);hSbpxQrqUpw+bSAT!QaCLh{i`7kkTc4w|!J%uc zuW!`WXMqE=k|CEx05EH{jG$Dv7IQHn1prSv_sEth+|&A@{Z|zP__z$HC~D&rlHF^k zh1X+EynvZYq`#!0KxF*BtdF8l!`6!S&q%b{YS0_VIso`3zAT&*UpDTuV)a76nKrUY zp7(2;%qRm=N_RB_5C=)k2sKKG9+M!5txjBAQ|+B7B-d2YJ?fxvV049p zSCwmSUP&vd04w1xaYn=j-_hl7e|Yg``%qBKE2aS1v7)3DI)QtR+6@C*GXQ`bND2Io zQ)0PbAdcgmHQhlL0?#Ba$F$T_>9`*VrG%%8X8q|DuxcT|@G(FDKLk%?*G#)zQg||8 znN2txEqz9Y)9T{u>v>zXisq#r=y%ceqVKI#3R}w;wF0+ud|9R zVD^#7ASvjB(T@hjCLXn15lbSSVHtU#zjQa%oI8*6HHX3aJI8QrZW3 zIk~n}tn|CX#p0r~Bt^jV$GXP=)b4)w`DN!{v9@{n5&+Qpk82%*AKHTnzffZW1c&10 zjFqfKC$_Ifh9nYDSbpNwP@3b?>Gb{IcdR~mpH3#hAX43-T6@Z6ukY9E#c#gK-zyy* zuHMY}NFhEYIYYgT(ZtZbd!r+R5ST+G@4Ezm+~CB;+Jje^9V8J#{&;FmV;P>9h$sE) zLoq&|%LA`?IFwADo>Fx7-o{3PlZ=%0&~NP@99i&YS%0<}UHxElvKj22^ji!rTtdJ| zq&u$w0Q5(Feuj-<*1>P-F*TSv8?^(C27?vJ2xB$Z@svNnX|-84aJDj>-ggIg65zDGR zyOSy@3<3Zh)il_eTO@|Ec22H;ebEt#j8$IaTm!D71qdKUfJ8iU-u?XTi$@R}?7q3N zYfb%+ILMPazx&;H-@SNz*W^oBn|G|*`UD`50%suL5=af1rt0A88QP!ZGuqz7&dce# zd|p-*#icN=RO6Y&>+L4oKQi)>m z@M&w1fPIhgxPH!Wo6r=NDofid8&b)j`}1FKFCw*pxPZLC!t`YxkYoVB#*b@Cz(rCO zz+j<3rh{fLA%e^Wz@tl*v5~=yKXoZR1hnt_l%O!NhY!~`HvjbJ0d!|FE`v{}F*__3 z7HYBpu6wnCzwE8Q?hnc7e|*|~BNHQ%jsyjQI2dqmbYU>6s^Vj#(fY%$@5M8t5IS5S z|3~zA7`GoBxR08{Bdz)2cz$wpZG2)9GXh`dZM_mJZ_IyQ@QY_oUozd^2X;Q}mJLyN z=6>dWWwfz7toB<)1{X#m@P1@4GBSt&07%f^=V;WDphE_#od*A^F8?A2Gf>B}N`TI3 z4SaxM%PTp7OeC?u!{HVs3WllRr3L^of{I~VHGsg%TpjDWk|6@`&n9vA zAQ0*_MgjQTg8q%6_|66yOmK;ey~Y4iEXAP+h)8k(B56Dmlw|S}08~vdw75TJ007q* zqC!solT1W|f@EDi?gGACq4$X@AcSB6R%nTI42U?Kf$_e~K$yYC*On_ID?3`;C8%yw z2`XZ$w6Ju9s|qxY}khm)vjQTR}Uce&t>S&xez%m%VM8)i+*fCT4J!OAb-sKQ(Q z0*mX9!Qz1e_>mdltRz+h(d0=|=r~uQ%%QUhf4c#E7aUb&E+o;Jisr^jg2i%pU?NU1X%)`T(_lldXsUW3>WZ_pf_uYCU9lpTJS-RzGAQ({YI zd^nS|Te8XZ!{Yc*H?jVa1t10sVE+*{70}%yP`3X5jC&^<%>e);V`+yYq0RYdzcq)m zBzix?>2pB41Ag$pwCopLX-3V9i?n(WVNyj2Tpqy>3fiinVxN#q#;38H$SEL3{V zcpr0q=#E$=>t|x{vqT66c&WD+=oZ8TCCWbpuW(4yaEclk$Kh&KV^hU=s@3kQO=GgO zfgRd3)$?jkWyz<8b{}h^c-X>?Tk|((r~2WV$tfO{ohcwDK?P1Ek6k2Z9FN2#D_Bl< z?vLA?a_VR#RU6##(BOoC{EuM(Faoi_V0WY&j}AB@DiRUn$G5b=)t~@(RE~ zr1a#-R;>_sUFcG`f}~PZw;*U&2GRk6P5ZcmN@y%x^Z+~5znvz|AG6&kohwRAZt%W;DOBTY z>(~Q;O@N<1c=hVPzFQIbp6PrI6)y5aaxnSZZ@=AN=#Qe{AOZlm2LOQhf?3YShhAO8 z*Ct<$K8(lNfF#O}aEB%|IJpk~iG&sF>$y&!_>Z^GSHPPF1D1|D2+D(O!ZDu7(vFS8 zlii)e{#w5^uGt<0=O0m10R3K;>BDSbCS!{tlK6d{*f{8pCUBEPG|`xqtns1AtPr5g zTCF)L#$3dadk_G8H30BqYN=MQBeX^ak0leKAt9jB2rOBZF%cMqFp8`ZCd(TFgfro- zp2kHWB>#z$_rO_cA4RrRV^}~@K7at4!G>R4{8%c%_e)f#4om9SzV8}IKy-G=PJ#Us z^AEBRiIr;;g6W;1*v`&6FcFU^wQY~|nQWA*)*y47R2%QX2_H2F@PPq91aMCc0sco9 zz%>8>iR=@?uY%uJJ#NOgRrKxwNxV_@m zRBas=O;^BE+l=zbwN#)M0|@8fBDZFYG>^Q7#~cJDV!B6>6)ZI{O?FatObeBMzh1)Y zceGDLj__ak!Q6J`c()Siz{L}A*x-P-{T1(;Nvz$ zC@F5$oI8T57FfzROk*$ilajN!+qN=i>$Dl`>GbJ~z5eZ`T6Xi2{~r?)K&-q-c7dn> zQ2dZ4oSmH_=3fgDi2W~BI+U6WG#T{}gw&Jsf>FdZuxXSH13F9YGVi{eiezfqfcJhQ z0Pw}#*;lWQztbdh*<3MmfvtaEU;OsltGt+c2=8=_X0!mf`D1Y55dge2T)|iOFLD`- z1TKca&j~v#V_}`SFHhQkz8<=`NDi5DM>l`|byAc$nw9HMa&$H(2LqgLyh-XWPThAp zz{ZL12j_p4|5HE8ie3GKk&f>0h?nF5e5R3+k(t5a78KPR*6DH#l^e?C$|5)xZ7wHf zv3mt?UXSSWW!1km5P%!*Qo7@R95E8^sQtV+3qwBU!8?&u9==J#X_E*9#K>3iBPvGxI{|^9=N+PL^)s%r#q+;m0S{I^@ z5WxGG7$6lxTpPp?^}2upMqIuZBLNjr5JcrjAQsqFLBc59bcdNzEUMv4Qe*?snFvbB z9iTG^0I}Bg0UxW#u{5qAR1_ht?}E<*0TDQyRy(kfwnhQaqfxkcdWlyV02#HXNjq?_ zvA)ldmKDx!TS*&z62$F9Diya=^!A3%{_>}(!c4WntTikIJ@W(ida0AC;gusFx7 za^ysK1e1c+KCQ_3fM>BlIOURp{do$rwBzyU7RhkfHP<>(1bOx8orV0^>W-Ap=5zat z7bLj9`0dqiqj|Bs{%}ZA_kscdKnQ@E^sK_MrNr9sXnt+<)y2@zI_}|InjIT#?TwDE zuO}xj;t&PIM_c*npWcniZH^EnGhGKE-%vISp$XW698abXn=|9%Yin!R29!Z(KS(M8 zBauPi0pqfy=?3Z*4&O%xK+_E)c=pE~D68!lZi)ICYtEbvN@VLyR-pu!z``90oRuW# zu@>mVXRID-uX!4WeQO&%fq{ah^St_XOdB9*fO*rH-NzrWq1WefdbD*begSLQKD(W= zSv}S9*M(I{=~&f_8g;+K6^4jY2t5=jUF zewqgc!1bP*`UW5{ld$DL{2-}^$&q%0`DpsHTerHa8h3-nG;mb|46u?ST&ra_zz9{f z(g3O*Q&u>gIyj%PSBNo6D(hG2i>qHNsHZTMAvsxVpRKdAudf*c4x6UG!v=pkI=a=A zV0cW8{ne7>ju`NJRrkHLG~m-Jl6Z7|cBxW1e3D8jVM5p~YA+QFKm>@BDjCC-O1CX+ z>)zg)9$82*{&4~T(;rfR>OU}lV)8^BV3XMYbm3!^fNwg!3;~u3gaVjTwMs9}A*H^r z-Y|%1Kg*UFKdLWqX}F1`jfGHa$t8u&_}a@cRS0M4+u2<9)9cAxay>uVQ@nr>Q|;pY zdMF_W%Q4~s{lEZl7y#8m*p!mp$gE9dGH!P!x;7DWNwI)!E8&xR>&72GOb%TPJzQVU zyU$MEzFlAs`-X?DjBr#jHD@xz)_7WxX*u39>=+*Rj*ojY8DjgS7D&PY%nA?ZR}Y_=Sv%`W>&wBE;0gHjM0b+lK$ciD zNeQY{UVSa_)`u4C^C4Db;hS@0f&ky@b7=Uu(ms*Ey}(+n9(-0^XrR8nz7hU)H8w_b zT1)}ttTGrA#OE@m(s(#Q2ohwdj*$}X4KkIB^LXg};iF($O#*=b-$+8-zm9D;n03H~ znv&GMJ(yDjg;IHSnN~vpY##yyU{wHk`1OY*05$?$yY@difr0(93`qhqfy>k!X2rm| zRY&QH?MP&2GWkRZkf%U$U+^2jqOn&&fTs)Ys(jhwPnNDg2&9UHkqL@2Qh5AURBV$_ z@uz*|%+9b`>r6ToV6<(Ifi6vaW2@$Nw^dP zqDzT%0ij|aT{=3|cu~C&XI#|V`kJSFH_ZYEjU&KpNw{8sVzV zon0dgh6Mni`BDOGkeL8l2k*5$_1t;bVYq&(@`JW{q@%C7->d2C>%$%uOaewwSHRXC z>BcGOv9TKUC;C0DvbZrZkV-bnFVpA9C*zIA8ev$)s1w5Bee zA$Qs2WH1=R!Zx@{aFOs1CqB!9sDl$UJh;~C%}iw0vTW2%CEZ$z4mrnBV?TKTUb`oB z_Uxz6;(h(#*N4rHrf_%n;LM;mVFhZI+!^m{9?W15;DT!Qa|<|PG*?4_k%d902;ovB z0~iqQ9?sG&UZex#76gEnXk%1QWA09BbPNL^G_q}R7u|wFf@K$F#btEb9N9{pvw~rp z^#l*B@X2J3%=REe=z)D23>&z#4vsw9J6L;Nf*j;Cz?o>&F|5PS4fsu%{T~>p!w(x< zfWMa2m4poA(?fu3)?T~>W(AdTJs00VM*V@OW##K|2R2%xI7 zC~?k{Obl)uN{(ZyAV8}OK*YXzQDJZ#SmA53z+Yn-pnNIt_&-7bnT=r&J5lSq%+6a| zHjo`-Zzx}!5d)X)N1i~r925fT+8{-3fo7B-k`(X{@-?zhSX~E#YHLDG`6I;wlED*? z`{50ktJ4VhoLH@>|6wIl zmS(%e-~u=zsahfu%B6Ja=*-#DR!hlUCNR`oWQuYEpSNK5p2=ivtTPLq+Qu?Iody=> zQFENvE~}D2a)1E<Z$Ru2(JDLuK=C8XzRd0wFGK&U_9yi?zH9|jp`xgk0SfROdW2e1h5$f` zk$Hb0esO^fadIH2;QAojjGYe-*ezLz0o8A0Nh&ufbGX60jMm+dlv>5+?h;M^I%UV!|S}-7`9i@bham(9C|nuUtf<} zsPmuSjrNQz?1$$vj)DHjc_chHw?F3{$TEs$WDX4;9$1{4Lt-_yI7bd(S#x0of{ahL zHfP)>Z+I-?&SqPi`@2U*##_j0twbV}%f{ekb`1~XhTtgMmJ2ci$ufgWLPQ_fE%Dxo zL_VKa4SQpw=G3)`#0MJ>)#LXyMlrD%)xn{@fdhUIgs4#m0E~`AtN0V6JGjphzT_Zp zv!vx7!*_jseTcF<+bF#uRA(5lAKJRGHW(m!1eczeaCQ;X{>hDvWO8F2?Hn@`*m5i~ z_G(8ml@ciUx({Mkjyn>FM`m#JNBm<5u#Q)!Rt5ND3~;S{)hZ|iR2vBTbS;cW{r#RG zyS8~0U@t!{mr1JtnU++=qyRt|nG{Fmplbg(JQVQzaGn&X|vbJ4=+) zR&8Tsdu!Jz(UM&|v$E138R<8*v=|4RcFt^VYt!qq6z&6UgmVs1YKEv@-w0GHs*Bc3 zz%GCRV4n%up~N!~ZnZl(sb<5q{zYKbyfx~O%x22aWNB?}?Hy>>Lxfw40Isn^ez0+0 z;M#{BDVPjMDO`$dw-A5<02!D>NdClA0-O$z@1SLfk}*9&Ij&N3KQoDdi2hcH&2`|rHX7>wM_P_L@Pw+z)0ONoHAqco5Xsn;P zD#()20EL(D{dRE=6SSq2OK~Y&Cd{!s>(WYw=_rc`kOYsiKL6=LGWv}z%i~=$nVG~5 ze*gjMy*8PL5P(MxfV3GaYjEQUOk!|xVWGjBiDm}fnhk4%Hx}(PTYWsO=(Fp<0pjaJ z7W2`wpFTBrfR|sG)7Tst+v^CHPsipW8(E3*g0T~$N9&*4M+KI-Ikn#c9zrfKIBW&e z4mn2V008K&PB%cnc%uU>Rf`sF1pts7SQ>D+8(BjxDCpxcvX`FOKocGwjiTA2E$fEW zj}i&|0{+{E&^pywppRL=2LNyp3wM>R4kM@L1IZ*nAUSu&(!fly zZl}%!sGM2N>tQiOhTldc6v2RQtCJhW|XcFc>UnKYv?R&9M_3-G2W5ID0$Yw-iP2wEi=Kn6gR&v>Nq zEG{LLcuB+*DmpWP@#~{K8j_3erD$_gtFgD;NpZLniey<4z}A731Zf{zn+mR>2jB)1 zhygheQ>8}M<0)B*Si-d@N0%HZ2t0-|>vb)<9ygBrXuO#8@8|&E-%;xUMgw2Mh9BaH z$-16`oi(dH080e|0O2MYP+~jmRA_7k+@yf8HNgjVbty0q#UMYnJ=R(ZL0r+WaM?w$ z*vF`?gCzB@ZvULFb^p||3$;Fh4>>{z6_&5E0cw*Vh5(fj>ccij91EbdOnNd2o?ihs zBa8BpbBPv(k~tup9Tmpb6le16rN3r#q<42{e001!K9s~IA$TMwVlmta zME+qX5H1W5=*+_6{(i=r8P5!6HV&hiY_2~O;uuj0&>^zHem&YmSKt1eu|+Vz&v-SO z!G^8w@Z2Jb{||O$8M=FJAI==^0|X!-c8^~hs}LC2-(Q#;S?JFL>u&50ce`nSU33_T znOh5Yj27c`bCz{^Qe0wm6wEj$8~kOLJxePk#YKT(6Vr{NDeI!dgrhpuwsA98&7Rf6 zL5!b#fd8k*fttxzT_)oU?9jBm5`bqzS% z>$G+4ZF()`V`w=*YwI|Q!d)X`fbkJICDKUANkm{7+bMnU1jsro)IC8tb3q{b2X$JD zrL`VHtB|DuSMr!lJv|nS(OD`{O^r^PC>LY>w&v`Cn81kDT!$3$_#CVi``>sFu6NTo$!lnUJaVRtz)n|a!LM)^GC}k4LQU%Z7e^g zhTa$ez%6StH&vFlD}^x$T>A9v@)8O-3{OuVN?0Tccm!Mj418xe$2N_PdHd(Etq>7l z$&5+?R}%o!ml=SX*{kWsCF10ZmtnvxP7Bo%hL>f4&814XH+2~Ukc6Y95^3(+X>V)< zvfJ3za<+3o#bSb-<5OnIwj9U3F!3C_^Xa#P@!0fil3HK?dOfxtUz^Od(8(mOp~VUR zXwI*;0001FdCUR0%5126VSjOdA>;P0X{?ju<5Au_n29PxkGTi)3hTKXoqhfD^N=kw zcwa?;jHWWy4T1jR(&FNRmS)=#4Z!>X3itO>g^?&RA`l=ez#w1{m_O~ORz~{WS-%zB zx5GWGCFYOK_gUT#o=NmWXrGfs97C3}R$zgiHne|Y;^?)9YsOu(dN+RLhW9;a+m7M! zaksn2LF^n%ew+jpv3>pR_jZbz%JPS>i{ zwm4SiI!v7d5J~W^s|ppql{YrlgXx`(_(mK^bxkG(Cc4BVC%f#YwEAV@fF>CrdTL$209B9!y z^;k!-atssKI`kzWC0g~|5oM+6G|m)Jm_w~^Z`T3t5A>?A3LCA8h160M4Xkz<;` zjGFpall$rc7EXTo@>QweLqwp$#8RbNB1K=)dIZ;68k_p!_DG~b(`<8SEVk3bE$5SL zj+NtDftfT+_hRYAw=wtmcek?TK*g5i)eYW3Yc$*MV{y*299+M+$2-vYk=+vp5R-#t zf$v1Qw6TTp9cT-hyYD7rLbG;nukUuR!m?K5`+`3R6;`0~qrefV#@ zul7A)>hAHrPRtkecXOXx2gV=OHy&;edL28c`b(!Sw^iT~^n+=agFcA= zag&DChox#-s+A>qNf1xH;1Q__003WhBQZb#KonI)FcCkw+70%JIuD3Tr#>N*!T&b^ zfSX?70;+X5&xrOn$^;7v%ZZ+{-!IE;qDR%y24^YQ&q}PnN5chWe_9DBr9iE#ED&_X zHiq&$iMpY+4S*`^;mAm&qrbnov&ByP%uVeEr@jp5N zN40Z)zu%fpF`V7u)eRVp4vMDD2D2Fmo|*O#2H*gMKzqMWt4qz*MenqEpv8VvCZW%SF-HC-^3*)Lszz!d|y z1cBw{t6V_=g9$O9?nNN80I8IDX?}iwX)$clG+Q(c-KWzQmL5he4L6&LGE18~D>HLb zpMQJj?t&alRcSwi3q;T()N!NZ3L5}RzjyK5y(eI+JZ)8z06YMIE2v1C!N@{nY~MYR z$?@K3h|dmY#?3UTCW}$rV95$8YsdSyzi7buqx}zhy*-)c3YPjG;E@^C^YKAU0M^_- zSUxiUxqUExbvmGb&D*z=NaXa`hRm=McINm(003~S!N;+5x^q~UmEqjVfMq<2w6STL z2`E0bfCurt8qKRjA8oe>9Xon>)yJzA4Hw_*#Si=3-ac<1xC8tg4c|-1K;Op)eg!x| z4GntY;GwxjBm(Jb=KiqLFeTcJ4giqhXd=<* zFglDaKz02>;MA6*IO<}2V`nm+i$+m8004l90^@ciH3Yy-ZC_t909R05yXL921Yf)M z0RYN1-x!c2CT%?9BaB1!tyVK-HtRS1E;R&5tGmWX&RozZdYzPgz+5i@ zfiNHXEKVvNM;&q^%A2Z*^nPUWJa_qVE#C56F2G5p;^Vp?^1y`VuQubMS@ftBm}TNudxN@XoEwApkV!o5$2ReaS9RMAmn@ zTLBKhds1Q`;8HSZ|9};9a~37H;qG6kwE!2!IAxF+^{i^#>fp~3V#6f{fcIb8EG}IF zKyiBZaxP%{a+%1|h*P{=Ljdo;ccq>*0U?>4T^V;6-ptmoYbI}V2SET$caPT*hCpF3gBAbG|Cg_S|84uu zvPD7C6iG?+ml8=)viQ_je1)<}@gY(&(EvkI7EQviBUPM=90*WkS%70Bu}-xdv?}Fe zlx0*OrxO%#ft0&$O(pjn+;p9u3*7l_aBv0~;QnQ^*V>;CrBq)GZAlcxH^2L}_S$O? z?&dQcaR%Kk$Q3{QaHKxqgxNnEf|wDM%waUzk#HeVKOfzRZeN?vLr@k=EjKfLqyhax zKDjLyZbvgKQf6)!65`Hk`xAU7q&vZ2;qU&B<}i-zu*43hW47G5?Bd zt|ML`JK7ZcTtrEcf?ueNg1+$T!MZ;d4+rRXE__u>)*%MhNAkZ?>TiT$|5=ML1_G8v5F*se071HeMJ zC&V3=n-cXUHegl}wZ>LDGi)EOWIp_GGqY7%F`DHBj)$#M`)t%I_4DcILaWqD2E`@* zxs@(2WnYWsH=Ig5yt=Wvv~=y-HQ&xBcM6->=v8j_<@jzqv49*m<NDcgr2_DEMx(va#cg?b&~y7Gv%eT6?4jfho&`^I zp_*K{bNjs~QjV;mD4B%)%ab~QVcS$&|6ujp&0T%4oGBS@z#$5OEpgJ{>V&E4?L9gVg<$tpLin{ zlmqhnTiCi}18|Xm0^k59vh3YP$5YOyvLgWhQQs%GC5&NJ5vBtri9SC+zk7}*-_FkM zQZK#DU9@+R0Mx}WQQ`0HxmPS43{SOtes}cznS2N5b0C2L5`}WP^#L??>i{NqzQyFhYJ|+Rn-TLodArg&irT3mZre0-f^?+llp#Pf}=4 z^2zd_PwT;X&%b>9`W)~cxId+#N&?}lC*PBj!f$f(-*C7E39nL5g=(J=YsR^Mk~g4p zaS@L%pt|QEPCK3M0ieP3!`fC_BK?!w9~QROmM<*@h@k}i+W6eFxUtZxb`~?kRy!XI zF2!AMHs0xMY)H^w-S7pY=)Gk&KfH6Nj^=RE(+7iaxhft&lK9N6sbKR-zI}M(nsVzt)A{iF=gZY`0wXBR{}Au>3xDQc?e|ZPbT`hc#{H-=D*D zY8q4O)jgeN5+4)Dq^pO%+%E529KcPM49_|D^Zfq4eLQ%Ys~Eu3r~8)^18@SyVIVPi z^S*ufm!@U_{Q%GO4nw<|WIWrgU2ngSWqsRyam?cNLEkJj`vr-j(a-<*sIV-~UtUoL za*;qzdeEHJ?h4++BRBvu0jjXPt>vw1JckURmK%WUekcW~9EDpYWT_|}M*oyqTd6v^ zZoG9EaAEN7S=Qb2iiP;q{uB7__a*Ctw{`&F_$~avu9O72m1S1}LZk!5`EuE5GzJYf zYwG#YC~z^la)OemWYhUV6p2w+mMI>nHu>66Ga$pZ9Wq$BG zZa_U)cm?=~Gqw53l{A6Q;O>?fNjN3vm@1^QMNjA`RR(nzS|lb>H36$vPWD@pT`w%c z{sYl;YqP%*zhJXMeh+Uqmk#as8Wr z_;J-IdHmWi6W)+xpj2(=Z!O(e+SphME_m|M{C0Y{h2GoR*5+;Z%>vjV(d}qk4!EK3 zVr{wi80?%!%jJB()U1x;dmrDs8*C@F`t|_(ESt~pYkLq*)X|O3U5VJf?W@BAR02;BoR&u7- zLmE+)G&NJI-nx}u>1D;J?`2wXmmUBO!(!GQ#VauL9Jv48z4*3I#{k$U9MA>4)og%` zK%xSY2z=~GdguTo>&7YpHEZ__200jLEEjVHRA+;`Tzp1@Z$N2K#Rf5B=nl@nJ_fK~ z*|Y9L(u6a%1E=d~_Tkp>4eBPG!P#ZZms3NXM1|gB2YA{n!31E3sX(;6)QL(Kj}})U zQ(87LjTBa~#X^2zBQPBd$Fsf+8t&y)aRA62LSnHyTfOG{z-rj@q)=!na?JT+TLPpp z7NC;gY4On30jMI7a`QR-U!9&yg(}`kXtnj>Cr~BbDm6jh%@jH}%7y&J#bLX*^~onY zY2b%jU?x*txU^Cl_U`=V#~W$f$#5&Z5G~0^nrtWYW%T85m&EAPJJ7pb&g5IYX4O** zi38Z)iOx&vztT*`<9}J(%y+W08!7RKfxv7y?7N1BpJ#q%As_YaZ0CC`|M{-O{j-ld z;sC_@U%!5RHqs2B-!ENh-(z7xyJygi6&u(@U#iIyf88s^31D$R7%Q$jJc zC53-Hl1jx>p`za}rtg*H--$Vm1pXNjuo_jqtPxI%{`;okN6dd*YgfH+0Ga?i4fi`W zGIdDd5eIPd_Zpm^CZ#GRmvRkL9UDJNt{$o+OQlTq+4_A$^SyTHzpuRC&kXbTD){&8 z%J)k`;PXlTFAviAGsWjQ$+bFOk2n7nf&fjg4lV9~_H;MgDm63ta7C={eB*qfBY}FW z_tWEU`ojaaN39nrQ_9f5k|QnJ$`qg`Gvvm(wc)e7006nQ0TXds%fo!uK}2=@RpE+?_N>iTBJoJ5vnc-}Owtd_k?9Iiq!SaT~fgkh+TxZAQAN=AZM*EgR zLC0^!@LwN~$HzyCgX7*hl8-Wf-SpdP59os+}(j{l8VLs`ZL)E83gr`#Z1 zLSvwrfq0X>$%(%cC}5T1U<@?oGEcxXm`My8ED;==31shx16YVxv`tW{gnZHP`ND!P z?2CfsUf@=cmCN;TlBlN7!;783D7q!@T&ukiK=F{ONIpsjQ2JYZI_%j3p-WfX^^7Lq zAu|AR2I2rHHKsCQrJhv;5;OjQP~OVg=G5w3=rrU-2C&t~09_g+{^emQJyVtgq1rxd zo*muWIb7zLf+X&p@-n3)Tg`v}&3|j7V_w3pyNuL};s7vLB!84tq1Q_8Y*&YC%aWsr z2guyMw=J3a_IAE}*j(F6E>_-$<^4&muFh^uO>NBhg1%^>D$mz*x8u7Q`OfbYdMi)v zGVF^3_!#Cd4&Y;PPg|G;DA8A{jh7C^0sOtz$uAwiiWzEDiUoY8nSC_0ti52*zzjNNGx_t8 zfdaq@I1&etJcVXp*J=mo)F4&FVCl8)arbhZcs%Z^vXCh*a$RHpjRS&?bP{sp4Z5$p z009O}5ui}o;1hXG2hhMpV)V#7VzB%{!x^X}P=A0d;7+rZDaQ#8RQwfxd}sbVbs(S6 z(dj8LvrsC~$@hNInp*Cqqn&(at=B5&&%Gl2=p9*-EEwI1LHB=bv zp#vC729VUDSgs4D1F+Daj?0KAXywDJ0Xf1?Pv5u1vRYDIr`H$=OT$94JRDXfVNchO z(>+cDfe5%WD@DOtZ)JFR_p=Rv|7%+u)GPW*#KDoGmA4fg86nk>I+OI?;I?)lfLuLy?Y;j+$m&wPwpV`-y0>9 z_g-rT@NpVyWD-0~2B0I2hmsTg2f=eQ?qur#tRk@vfO0Sd!LF=HG~nN`kk3bh*QT#W zBB=;*KQVvs{RsUhVvPnOKPrD^{fV((Mt%hUXD257TQR_c^5L8^1mw@_bOV*xwSQ~=@~ChDS5ts1n?nr( za1w8^*aY<_I>z1KvnmoJ_lM>AX8;<&Q1R?fEy;sC~P-05Kt+)bFk z{8~Pt=9B2w5TB)jizT|*(z6(~Ky_4plMcWKL_R2=Y7z>AYt+%{HA{c-B&%CP`51e> zYvN0~IX6?9VYW}yzwY<%47YB$$j#rv%DlCC-{2Az91Or1l0!nSjYDMgu|#IVh4ba4 zSLy<`2;mjhkH<&H>tmxOHW3t@zzJgf{5Ph+SPm}XSggA-I51g*Ia$Zh;9de1C2zd8 zi9`c?VZ=ko#SYzs{B!{);HQ|s|`?ePCOJ_fAjRSKS;g(?9Dh{ z*joOD3s}RHYAe4H2uS)*Hi3M59}mUro4wvbU@GiU1`Ba{P=R$fFsOj?Xi3h*w`u=F z_I-chT}oo3(dl|X?$j$DK$1F&gR!6}Njy)t^XazyTI#jwJ613kyfXB(n~+9rwznJD zy(pDWJ1v>y z{LHj(XXkGGX8G=`_iayS2Wa2N_mYKHDZl47x^5@A{qf$}zcCw-Iyp0O03S>oK)diC z9e~v`QuA+a4GqA$^2E0wPq%f25P4Z9^x6wz{eihiBvedE<}X!$MI*m<{B_jV?DyEU zv%ZAjY~ruUKSV%P33xr+y1Bmllt2No!u@BL4q$%h-hZQ(`t%?zIaU(`S2xCA|LRx2 zJ;sP(Y+<8ua^Ix%udB< zBR9lEVyot?f!JngbN7b`!GJ-g*IalLzd_4HQ7LEkc7A0NBo08+dbe!;aU)Ic4>Al9%afD5<`DFBsX z+`GeJraY(pfxW$WwqD3k#lPP_kGsyn7@yj@6E1A6X-Cc+x@+lkd8g4kNL5myY$!w& zA+Wht5(hBtS+P5=m;$IIm~IE4T1gvo^I*)AINLIn+JqU?+YARJ$4#A{t^)j*8)5kf z`m?-)WV+wO$8Vs6HRWzB^p;Cv_tm6tqtL{HRdF^Y>j0Wktu*DLUK4E4e08f-+KhIx z*^P}2PrL02Py42(e8Gieznw2Yo_4vkdG{hd-MM==?kjJfocwwFuK4mZDY@zZ;m{CR zY`3@Xo=Gks3Bca@+RB#XrfJLJ!>bnuP%TW<&aC3s2imT{_hClLUJe2<+&|d~HcQLH z6|4KK8Ikx1?01H_o2J)B>E zX6^s8r<2+oJb?D_*rX57o;AwACf54-za*bOe)-p348x3%j^?AzQ{w@i28RJ<`rGvX z{sj9wd8#_3&rWtP$B^%9%K&YF=Re4*b{N@}YN^;sxXLc5>(T zW})BThPc9$-rt7Y5_9;hCdt>DX(Nein9rA5Ti8L_!q^uTu+o2d_St6PHX{F!TX5v! zwbrypRj#S$)K4!~9C4{`suT58n;D4PWma>*`eVsbIxB`uD?4*%!2g`Bf%+fWO<5)EV}-gd|2N3>?XkV)rtVpwIfXBFIWDF!JRK$d1mX zxUYmN@;?=q1m9P{VOuJxKuE9LLGCYZ1B2%{EJLY~KNbJ5v|J901Ngx7Up`=z!=mg% z%mb@nhB$zJfw~5e<%D1nvw_}Z40JiOw2Gmdsf}_I;$E#*FVo|4C{xi+QpWQdW-|j>ebo~vTz+p8(oDP=1Pk4Txnfd1LW40xZ)@w^DMZDa+K8C+>`%+?U4Wr|;pwEXLjn|jB z`KI@mXP&jvlgp;ca(EyS0v5ov=ruxYL^_Sx#xCaOhtdGYgF|ev5 z)FABYui^x3=*L|%1oN0;A1gJ3TAKfa7Hkq7pC0JHBAl6I_k6 zzR&J$XQcElq?eKVH?r&JL)|O)vHy7XbjG*y9HYfu5S*o~?U^x7O8VxaIUtwbafw4#vN~ zpYc2!%j3op-k-ewY`7s0JBBCmw2(`fJVWOScXy3LGTTQkw~??k(0>x_ljr~TNmH}` zz*vl(Ppwv4Dggud%k8yfOl~`n>Quow$H1*`@o!(O~v0aM<{9W8hdQ5B9$P#am$`G$u4n(|!It0}`T1_wPG;&1O$ zp*zjxRxld$SVjw=eceyS<$y_h5>HTQ_uGX+ff_nS@|PqaXn0HTY^!{pSa7e|8m=s- zx8MP`Hiwzb_Mtcx?Mtj=+K3+?E<7o1l`;RDX|`6^LpMCP4t^{H2Pn}2v4 zq-Mh#OOo!-EQ*VZ=KIJ3(@D=3WdT-JHqUps^>}S-XS63)o@IAqD%tid`FyuJ@Cl=Q zZ}|@T|M#lk0m$j&k4NXrD_fa&;;Q3X69St5zTIH4v?bZd>nDo2IZxiE|% z!8$AOz~RuzsiJtzo2fRJ#KRoDdh~qiaK^E40}fX+XM8I?%uTFcQvDN}y>SB86|7(G zB*5$pebvdc-R;cnyPK&0`=_QhK^LPo4`2DOy_r(;dTxDvkdfrC)Vr0)?f<9W{(RDX zf4??tlM-JEXG+Z|X4`cNuHNhgbCTL&Q?DGxqtV61>HqoY&;4^>vRVlG&gJmIhWDW_ zdH574VRvvQMiFUrW3JeGwo|{0-3uU=tYv}_`jB))e2vReef?;C_lLK~U7M(*rMLdw z51!Z&eZgCc1<*oqeMS1&^c%nd$j`UGECZ04+wLCy0shK>U!xbu)y6g8f3{|r>>op$ zKO*13SgIKQ_4rF_$3PnX$w(OfgO-6)8&Z+SfN}eQvhH9ZFp^d z;RnUE%K1XOb%h?g+@pq&6x40g{#!0(dcEz6g#9%vj&G(9CFJ*D9P$8iNwYV+cUoHJ zc9fFj85v)Ae!=7ZxX{Cbu_jbd=P~{?E4h1T6$%{rf_O5lUx{WUlN_cCNMeu0DiYy|8c?M{%w^+tPmJ!jVl zZf9foqXMjERc)6Vhw+kg8X_pnyYGmQBC@$78$u(_3y z@+`ZDBT$@iy|sLC&~OQTflTxII9B<;Q;6iCRqN&0pMq@t4;t8 zU;_;Tbst@mCg8WX-76<11!Mt*HJkU_tk?5^zSDR$BtP>xd7KBfgmOI&M6~i~#e*ot?C#;jPuqCr}46vn$QT zKnQj(;ol4M2Za!4Q=QCOGc#;?Y!hHP(-QYikuHLOYyp<@<-CVldXoI-^KGtxUeiHN zMC{h`8q`TK|I==@hHGn?W^d=yH;3(XTC6<1aL7D*tKE_ZSZg9T+&OJi3tg2>ldJQd z>dw*o9#ocl65x|Q$pxo2W^Q=${>gT7g_FsJW-FGjK-nY^| z$2?x51}8DDbke1pOh0xni;b(se`OPyW(L*^&}li?$l5C{)vjDmo?lY;1I&9f&9XO< zOP0jt7v)i3U%dWd{HOn=IGm3S-0Y~1$*Q&HsL_2*3XPohTD|;u_lQQ4J2?5nKcDpb z@4MaW+aUjUw}AjiHRkUo|Mq`8J2@Wbj!$-v6Jpu5V?1;tos1^aTcy@v+}F;nAN>Fp z-)67sT}L}#uJ8i}eR3MaVfFX__!bV}hfC=HtugPnS6Tyb0PX5(caqE3u5|n~E1;+_ z3xWoH*T6LZfxN67viZoA{E1LWH9kK#V_qsfr1^nl)Ul|5y|aG$7tR*+ z4$H}c3T(Hw`c=DN1rul3fl}&iRofnM3!c{6+Sa+aGI>Gqz${E(3Q5WNJL!o7kgDKp z@4VMbN;0r_uU*}N6yq)5PP8u$V8{&MA3lH+!;2;Za`^-8=E2@4%g`@rfCdB9a%Oo8 z>kq}Bfp@srPCvnI4~wAt#7Ot?R$MRhcz$xK^WBvS0GQqwA$x2}XtknRE}>gNkt}p2 znH*zF7@AL*%z5={fB#j!9eMiO-+uYx#fvY#`R3bq@4o%r?|wf&OnvjaKmKq3`~Uv` zymI>EA74CrvtAPqaGuJkqv6V-tfy6azpjP2e6P2Xj-jT0TRVC5&HpSN?l=i=C4Mgh`oLPSi{sO@V6g+*fm~Y(%Qe$7U;qO z++dFXRuhEou?^1?G*D}R#xEHEN-b{+J$ zIUYRZ0y+Q*Ay9QsQh?-Y*0cpHett(f0Q=vI&&)*og?xTfVK*GHrFP(b)=U9>ZG zJ*xxYg0kf;`O8Kny@vI~$+X%|DLphagF8QMTz~)Pt>&^s`<>bJussz{&d(&?w0mo8 z{g(^*!iR;;EeW_=TWcK0g(Wtju#VkhZTZmC>Gnb=?z*Y?Tp+Nz;hVWDCBM{=_kbov z%hlv9#QM?R2qnPs_6Xo$7309cd>;xR@~*H1_=kV!NxCr;lu~j_d{<`qGVa?x;6F@~ zeyw8u!6pYQTKa(Sb8Y&1IJ^m`P({lRe!HQRAIa#=tiK$Gn#r>s6!v|NhrMes?#0{`^0E|L*&TpFVi`t51LR_2(xG zr9VF-otJ>VY6%cZI7VG@nrwWf6gqLQchF0LG;904I!KhLH|3 zb%t6$&aE$HHt!~LnugdNs_WUNE(hB_d3w;_$~Us<(#qW*BpEq5y8qSVQ7b*aGnb3G zq0u%E%INFY1E+DUZS|vr^R3=id)65g{qb@u6Wzn{UoWIPvBuyNT^cy791pgC`or$- zu_OQ|VofJV;eQZ!wA_w*=7IUmZ+jZY>MoC~YpsZh-tvBn&n1UXJje0wU;meSFh_rY zn199O-zp8@*u;2uCRv@C;C{CDm#95xX!$?E0vg>*+h-v&cmlX@xaa-;>B6e6b< zo&P^9 zYwk*~*DCk|DWnKu{~Sp{uh8|nUW}$>cZR*z4W9=n%CL_`Ijv=Fx03gpLH+M38{!m&Y%=SXL&2r(eO$eY}QVqanGTBQ5)5okPxtkCZg9BJA9p3u!Z#%L7 z)h30!o8ho5^Gqf*0*%CocJmg#Ed> z)$nv@sxJ?7XO9CF@(zr$@r`!!R(zBV1V(jV6l8;)s6732HJ`+C<{ozxzyWN*0mw^B zO0_~U#LTj;JH0AP5;>A7MmlzopEx_lT)Tv_XVvTHhN z`}-8tdi3a#nD~Qt-+%X=nz)`2yhbIKo$cyYX|b{Y`Q=G` z&evam{UzptQ3yPH_UP+RAAc>E$us@>Yk9JdF~R@nsR9AJzkmGrP4)J@AHIJ4>0>?N zulLt)aZUJ2Z%&+l}fGEu8F?g zu6v$A+>nny6Fncfu(5hIU5^b$cXSN5-*E@qe}4S=`tkb756m2P(TQ8%X|5#aJ@ZHl zV%vw+4_ohn102sadr4RBJ_q0pp%-;D4vxxy|KI-VxmH0#ec(EVAl$vo^fk99hhRQg zjk-3_cTG@NkI={&A`hzo;~D<`ghwW?9 zpSaQlFu6b?R@@v)s9w!xVe-zY({PFHV^kUw6av@qpCz2C=*45(sKeV+~>P5Wo{@9`{1UfypmuNBH`7z11@?JSm)(Zw?K zL)q+82Y@JvY5Tw293T8y@(D2KNCMG>WUbWNIsIEQ_v0ZNMaVgMUY*S5p~teNFT z?dVQEx^sRGiNR~vDn}>#ul8S!se{(L`{3QT4B_^4nU(YO-M4c3=*5f2&!0cvezt#L#$N~H%l8`K?aW3W^?LHSz_vy)~-0Gt(zkck-q0hjTaj50;#jZi; zU|5wvFuJqj>ts0qE2iqJW`geCsN?HEB!A>|yZ+?TH}`8&3DMn&12{U$W|mt^K~Fh` z`|uoA%gw46*5eeGd*z7h#^jinxZoZ5(Lbzr;@;o?Z{6fgZJ+OD{nY9ZM_|f>x82%h zhQD*w0Ze{qM$omAL)IrudWEj}XCO@@{o%wkoa>xmBN5BRZ1%wHU#z}?`7k93<*wua z;EWHyRmkxX9qLt!c2nTbkH16{A- zFkMYId*ZmZ`uV7s4m$qg&%kYLZYkKD$#6s4ZESCeo}c@u<4!5(#S(j-TForqn5Oze zFgPEKwyV4E&%naW$`z4Wxdf0Rq) zBwuXg9)I=nr9{`ielAA8zmL0l#=IPLdg8!4d$>i0%y6sJ3Wg%7Kpd-UBD0ZVDh2t8 z5H#r`amo6M*&@c)eZFWgcxz^6I#7fX2^5HqHBC7_uGtexWnDphOnhxTNn|DvA=HJ| zi`Cn6>*_)#7{c6;%%FeuxG@P?PN0vGvcq)sP-2KL>zMHQ@xpXHnyyxpKKax%01wny z9D}7Xb0*Suw|dxti6wH%vzIz~#^e9yPe;27svk?_JgcrPPpt-%?TA-m-zU@#&my|J z;u%T{S!Be15a?YV_1UcdzyI#9x2h3c!r=HB4J#_Mq1KGSV6FGX0korM9OQYcSbuk5 zur056*`Nr=~Zq`4-zagwkTRrIp@o3`p*!(wGa+TFr!PWfU$aZz&`i z=tN7)8J};eGqu>RMmu)jN-&xhb04O$dM1gDj(wgm!XqcHa{yu8=sJtBpBG9-YJX*= zibH7Hhn3O&Or|;?@_Mt`b5&wn*@Be(1oWrGFRV&}C1HR0we|+Z{^PIy`76nwKYuP3 ze#HJ!Hy3M2a3EHQ#-N0I!fa#HZC|e?z$~hyQh~tg^^K*aYg166tH;BtIvt6}i^Wvf zck9;l^eueDl9_4z4HHS?3M!x`aqAFS18V3j{g1D-|Nj0op#>>5pf+EfsZB^i1Okr} zNsxn9mj7~Hj%deUtV#2{H;Dx^;RdVNH7tWh*CZ|$vHK3_eHq}o)xuU*(&PV)g! zV~+=}uTa<-?a@2fMtviheEaL?2Mt&|a`}AG8+YO!g}~Lu3HR_n|dJZ-??~0J^|Z z0Eh#yM1Xvm>NzZYdzrlN{lY@IUx)_7ImrPId)2bF|1_3;qMMM+51TvBe>4tY1>1d@ zA+J$_S2BY1a<93CmSf?3^>5dI|D&WJ=ixNAC}*S;K}g2x)P-;=6~~sO4em&~A(4Np zBv#wPYUP!+yL7ga0W1$Qg9hy%Iw|phIDm@kss{?aWx3;KFK*07qsczDy;hS;bJL#` zN~TYcal0q4Bw?2mCGh?#g)*e~|8QSfeSqdb8qJB3%lWa=134+cX(7jHn6}}%cJSd0 zV6iz)gn9jr-yh2QL#|icH_4xvH1_)OQ^Fc_V^fhx5mawPI={c->cS!QO9{EdW9^|` z(v5Aful)zxUpJJ%H@i0AuP550i32d5fZ7bzl6!v%WvBKAu9%f7VrS`?QpiF>d?!ce zo~U@$;wWC^%x>3pyJx0-$x#+An8@4Fw>V zvmU^@gKo|;1I_Q|yyJ=;NZy>!PmSV??WwpN)0hBiEygQK8Qj@gUTLqI{-2n>93a`O z)Cs5nyh;g_!zNxBHrv|k^T5_k27p!`u>C&R{6P52Xb@snr&-N=f&tIQ#&kH4t&D#x zmxf7?r2_X4AxE&>8%BNUX7wN6^o-!OVzyUhi9myG7tywWInUqPz(_B zPVsW`daQwQ5hvbS9*$_p@pjTj1|>fFY`mPwq%+%bIaI01fOk=jHA45|X&v)H;4`rp zZ5JrxMLv_64$q)Z39h=_N#2(|Z9iEv=|OK8N6`l}mQ)Y04@E!5MD zp@y4}TpYjdMf5xX5ZcX3{JhBLgVFOkbabs`E-N>C0jytaXdo5^{mT#g<@YDg@1Gp+ zP3JSq%Y7+x>d`_voXv`b$wxY!-E)mW??5hKT5_Un5yUJ>KTiKYzaH-luce^9jccT| z4CeV_5y=1y#5EFvkR^5pCIL{Sue;82U(@_cxbN&zux}FZss90iFc&C8R$06JvylsA zzTe1Mv*vLE)=$HX3n>Q}M*|%MK5nKD79Mci^GvC9$Uc2=<_0zgM%yhp2>Tl#2Dz9N z@JR;Ho=u?+@@niy3y|p#89^8f@K;hhnP$r)5x&hlpiKwBHQdm&sHWT1ezi~IF0|7~ z+ff#@lCy-Q@Qf+G|WiaF)x{SXL`zq9g|TEe>K;h-2V6faqr$;Us`H}>h_Sc zC3lF)Hd3vXC)eB6#5z5D{a!p2T1eh<$(^h$_IgPgq+5}+Kz_vh@&P#BR2j2p+xWyC z6=Wk+L%tq7GZ`A-=4cBy=Ez+`OEDPjS1}fWH37rS=C%*39#)6MGcZSyDbI#f^&@5% z{95*poD#4@d_SO>^IAG5$kbfiKtg4LCee7;e!pUuwrtNabATJbU3603S^F|ESRP6xZP>h(?_YlQFUjQ1*3$i|8c zbp~^zQK6O2@9adk3+d8wDbF<-@==Hv=e6Q!4;k_O(Ycg}t!M!51lx7?-u&j#X}&d| zGLpNrdw43jnPhuLzZaLJ{IKp@$#l}SVE<*vKQjVg10&+4R{_-Ew|~$-$yen}lYiqO z@YA>mP28QyxSwedcID_p|6h$>+(UwXK|2RA$C)$-dD@dCptN-;sWWGF#7256NdJL0 zAS$>!!`{l_YRV{!O4=_Es`{jG0H99wVZqqmbr2&Him`t`Y z2i^qdui3n{x)BdAO4`tF0>)PdAm4_~G^GAY$;IAEFN2X}iU<==|G+HWDwTSrp?pc1 zSU690`lYq4`9NhMIkhp99EyFn+tHb+)oIL;rJ)7bL{{4e-8&;+>r+7S?AZL)sc*?T{Re`JMS_87&kx1*Zckzjwm_ z&s~o481rEi&oy#}CfFcRKzaY<05aJ({2(F*Z{E}_O{Fv`o9K4b3AyL3Eq-~JGhL}V^rt~HQYld8{4D2fJs zQb37IZK(1AUsZ!WxGsoIfFIaTaVif99Dr>a)4sdtm2C!11Me*6uxZlA=5k?{NJ41%b$)W0Rv7G!817Y z31_&-t*AIS6~WlXH(;Qt18DTxI_yfOmm?hjbL5Quyf*X8hBFceig%-gwe(_}tthg8 z0~a`zolI}JwV@zEg+Rcm;R5L^4(V`EaeLS_t$u)jZ5{Qr4nY2}fKFgJ??H#4w3aS= z@`t@)(s>`8OD!$5n;-fX+O1aAOansO10JB7?-_yIW%TWhC#f!?a6G zZ_GkWqEroz_Zay$oB&HQCmWbv3I?&OvXwz2!PoI&1@_c*hag-e4%x3z#S6J#8hCCJ zSe1>-s0q8!kd2nE*8!&M(2y`KfTRNkZkYSEu`#}*iJ`8KSwm>8fNu>vP-A>F#>V>w z2T^fFOrILVUnkq=DgHfu_46y}g1_2VL7@8#?hqt?_NPDm<~NVO`VCHO@XOCfj~@Nu z4}W;Pd#vE)3G~YMpIZLUQ#&qH&B0gNjElH1$5N4I_p4M3QvSbe;M-I|^H-MhZ%9jNjH z9m?-ScYxbO#Q{WpLErgZ-_FhsmU(sR_oSxijOv{{q>(a(?Hw%X6=Nxtr?3^H!=;^Z zyOdMB3m!)tLNQh3A}&A}JIQoAbP4Y@ere-IPwX-WxXcl37x1cMaIK7JMN!hviAC|Q8s;ebwn-^C+1Znl$$2w`hrt)fhTP{p^|^vo~J&o3^@{{`a!5T$!!n9;?>#puG~g8XUGgVCO( zCt4OGtlmACor%Y9{AF!39=I`~0RRRZ!fZFoO&yFl$V!xoJLBDfM+QD-G<{`EYCN?^8rnSZ5WG~LoAzhsA=g%*) z(CUe?(xT0RhI>4Tv}nv9G_aRssuRpZrY2oD_jRVHRRul(p>O$L&Jw;s>qjD=sG%l=_Px2S;{W3@12O7vC9NKbweIFPGlY4e&;FN>UVQY; zH~;+LpUKKW`|z)iu+8)N^ZRB(aQC_VO5)F>FTei!%SVqs|5BHNJ%2^#QX89fgxdR` zr~{s;u|tx8xZ$o9Dlsc{(MG^d9eHhDZzb#ZO9t&i?W9{%vD2Dj7Km!bv8fw7mKSp{ zI~qlAd!qBvsJMXbq%HxdO2t{NCP4}SE{lCTcL~q>Ix}3h9mOm$Nom{UrxXh5YT@>F z{?2xO8w192!@Hfk#B=WecR+~0bv=;Quw->qAncwP5#W0lXxS*buU zjff>a*$6>`>`paZy`I%5e%S<2BA`kRTTgI##xaG#6%S$32D$-c0eeyrRJaPT0)%k1 zH`+UX`&xeM{_w-kCzDRVU|@LzR$cgDY$t?CEi}_Xm<7ZV(g5^l$OQx<1Wyx!SeO`J zYe@`#4@AIJ2%9KDlSxq;B;-;Uq!NnX9(oqA4UY@)+G_qKx&W2{W%=9Uyj%l++KfgQ z+NEBr8qA)qZfq=+(#f6F--g2p3YctT6~l|5%QmHeuXtjqh`<+i>HfKsfA6R@3Rtf)Bb?5|bp}r9!IsgBeRj-ZYZOjM^`Y zewwqC{Imnf_!0QMnjo{KGf3np9;16IDGKoFnTG$`n>UZXdhwB1_wRo9JAOTQ`SPD7 zIDh`xdd)Z*qyPx1EE}6$H-JUnhM* z{duZETgN_mh*cfyt)eH9sXLaOS zFUZE>09u9i23Lw=OdcK}n~J}OMy+liBxIiQRnY*{|K&F2UWfs}0kqj4#55o|LcOX- zss!}vJwfOF#=@|blyl`0XbJ5k=|oW3SplE7twO_?80?wd8}tJh66V}odbx*ng;M8> zmuR7o8D`q)W~N;Lo|!2$mze?dGsqXJgGM$x-CkQ8qB+@;Dx8w-$#79;S zTb+aP0S-qv0Ox&PUJFJ_j>fg)1RTIM7DwVHL*n$qP)7$Ub9!p>FBJc5&5e({I=Yt= z>z@T&g+(WY{Paj6<~T&vQk46uhy(CCsE5tq&rc@&|7dc5L~$_sIL-6?)lYB}v6IIy zBq@IR)mPvHy!huAUp;?>9o(e-Jbk)uvxf2d7&M$?-boH$UHMz1hVT#X!AifON){L& zh2fJ}^h}Aw91qZWgMB^y_P6Lxe*T;PQxg2gk3W}xdXj|ytFPpFynONEpTGKNIejI0 z#YZn*$p05F9z1wpo~u0DufBTp*z$nocKG+zSGdl@hwr}qo&4kJn{U69lb-!XzJ14E zaml-HF~s!n#h2g#b8q+Ao^O7h8X*$ZleY_1dI9JGZ;H#UVjqu|2ivsrPjpx34qX9u zdGCAz4-%BaaQa||l=sB)@4vh%Z76FESDHMP?fNEF|6pLnqr}%@}s?BYdEZ? zu~n#?UIueW3A$hcZx%AeS}2t)t!3Ks=*i}C=KQ>0D92+?sJ*tbNI6S0Qhlf312Id7SRJ{vC_7}kYehojfy1B_xKZym_H8t%+^lI?Uq z+Q|avMUPQxDOLFP(wJkXF~@sq6hD6ii1_NuuRneHso2WP$1h%}IeqolzkdGtXZN2T z#Jn-76I2j(AU8tbWB_I9ffzov#Y2L$0aXQ(O2iZsxyFG!9?>_DN*n9Pm7a`q=FC;PvfPL#xjnydf~kHBaz#xP^hc8ZkCcP1-qwZkTnVAXtI(MTxSkDa) zAU;6spL7lRmKHaVu8K!U_xlC-1JF0%2IR+%CJ1W&u=8`DTqy2D5{o7HT?sNP0_fUfwl+d3et)9HlcNC|f2(KmF>A2$7E-HG9MC)WQTCjhPlf5ig^==)`_ zGzvjYu)!BgN9w+v?E)O#W(HD{fl-!wh}bN|)l^eL;y!$VHUh*Ws2&ivkVzGRG1$&j zhxLtoo1UKPoCkwDg{|IlGg@};`a^gMqX^V)wEzJ~<(|*? z)=D36KgBYaglrC%wGJo^o7fq?SuT}^(GCb5rRsJenJGv?R}g1Xt>(+=((>9?2JQn5 z_R=T8;r6R%p5fY7{;&iGa2R0c#&J$JN-IzNn7EKi{e8dHIxKgjxWv|<<&|{S)KobI z0cQqmtB5s8;+<=Z4gCU$s^n83i%BH_A3KoZDvtMcN8Xfv{yY%M8X@M!;P~k1=_`q( zUw-*WVjzcQ9)12Sr<4CM-d(EeAEA67WBxZsQm54a4dQ*O5oj_7vmUrHkqdxc_#|yk zm<}R6fE5y_PyoROzz2!OeKFbp_5Z=h`_2FQ(L)K>5{|{xKK)d@(5GK3BB!UXar*qr zM_(!iu)qIme~&Bef&R9PDVI#qd*3ucXc;lI*pxtk`3@b zL_#l5atTm_f(MzwPEgYSbNPSnyNg!gHd)@@T+Loek&qfZi4uSkk~WNwUfn+$A01n`O`XS> zklodj;(Kb&&n}Dmqur}Ley#Aw$Nzy5xW0ag3ZQ^rpWLi=`DMonRMJ43py>VM2b(LJ zSK;S6XwD~(AAh}%+T_c}FJC_XN6PPG%y;09I&ygb@Y_!eDv(xu5DV=gM=Ra{lTFqM zfHMl=;PfUwyKOBLwplO$OO)eEss_mdqQns8z`wY-=-lnxyH_85{4s;(U5$Xi;I!|lMGZrudlMFOG90T zeo!3b{z6%Q7U=<(3QEl1({;ky8#vD=3(-zIgw{P<`3}b%D%rZ^H}W!&07;#bkM88P zlL|oe!+aitEcs}@kSwIPpvNVdb(?CQ>L3dE*Oxp%vnBA^b<~dl0HXL%IY45h;smNv z0OWNPxC}k^O(nq2kT$U?m8G==K2Md~3r}tF*=aom-_ACy17xY!(XG9Lkko znby{wf4g_S)y&*E_mxX4YabRi#WP8L(dy9wREH(`V5OQ+dX1DA)k9BXVJMeDH#{15 zjRSCR^j4k}8b=mPBEX85qkm2y=^DIoRXj8u~HzO?k1 z!aAFDRg8b96CcUDkK#!J1qLXClwdRX}<(m*17Y zr`De^`ag+W!fpzw89Po`7%it=|C~Ai^#ERPBO%86=F!Xh2i+R)!gL=dINa`~;jaJR z0()0Z2Dh`*LF*Q<|2u@xx{BSax2s)xfl1dRG3kBe%xmKdNGOA$D-R>N!TyVn9)ADD zr;k7V)u)mMeEAA8TF}zRdfvhD4`&d&6*IuB1{Q@?Sr{N2Kiiv8CE_{Q{EPIvDz+y^@6a_;W=jyQk}R{>-3_xTQh14#ou z4uQYu{-WnUsqQaN11tkPb1$(xK>ld|iLIj%KwQ3JsgEEA?XfQaQA>3IAVDh{X?%j| z;A0FPbO->R?1{j9#Vr;J+7jV+RPqJ1OS$HI3p)e&2V9#&FCP{Pnkq!kd8@w1(tzZa z#sQR?Tcxd5QZj(~g?Z!@9x2m@=fnb04qB6pnm&;S1K|0bWtz0quY87;w9wTeAY&1E=%WUINQ!*_Dp%A|W{Tzsu0 zmx%+IC;wUAk0`|L!PAAsawGUJtY2>r(g5wHdu(GUkoI+(1`udBx?;T3grg0B#KFm{A76d(;s4Hdkg)a)# zhCAmMuRs2fjlxaw&}ps>8(t|1wqWvGTv)M2uP{_^E+1;Yu+0v^`3?hLhYG}K26T)E z=!kRBgaDpmhX{kp7oUH!!zmG|4Ms(*)XeTFv+oM9&+(ti`qDKN4LNyo8trAXlV#!j&S8rzA`A-pnciMXRZbP|FTC6fAPj*bbfvzzi=oH04}FIY$g}v zVai78du0tmoj?Gg9V|hdUIFI&0e>uWJM=IA{%@mte6-p^Q81Le zebblULVFLqkba9ofN&l8?fj4{NGZ$LTUnC~U^);8*JlIaMY-{2T|Sj!5xY*b5fX{V zgURg<27t8F3yxVwF%tY{{OagJ94XTNd|^^rN*)P!DaOxrY*~I7ptoRd!ZnLM*~a^c zyj|Gg!2#K}YI|y@Dg~&FAIG_5?BkZ`-gu4qD_hTMgpurJ=NyGi>Y|pNh_`1yYCmf}kR_qhxVg1T4MwMLu;dUr3 z{zHK#B9Zs4ICTZJKL+cdKAjuMdHowbB#GGLmIzhnIsLUz+BhX*E~OB(S9LYtRkF!U_xv7TV1c zu-$*TcW;zUm3zx;n-B+FiWC8N+^YBiw`B==pddZq*6>FMcC zbnAnaEnk2;rO50GYewgjvWTDD6#TqYeJU6@4fDUaz>M$3g>mo_`mqBIVSf|-=R)2W z2R(BDz(0$5`mC6}A3HuAhv7Ts1$J)|f3xFLGhLGnBuMX#*;1W9VDii!T-Ue|Qh_m< zn>(nP_W8JGO%e*8lUN$F836hPe0)5WP~zaDZ@+!`&;Kk}#DnE@+i=%d@h`LX-OKoI zeSncDTQ@|-D^2^$pT?k%b%E(GRKq8a@ZaUE)!wtTVw1oaD8#-ya8EgfMjD(?GH!xX zxX=uMyD=imQn zZN=)jL4B+|K2xS?|eUbR$T?Cg6IAhlf> zK-M6MfjWR>frABYv;$cXc#u2Dy5pU8b9jikt&PJHMlk>YU@g|#R_nuP`VbyK zTtGCKA5uwYc@50~(zcPfFE=wuVtSI&`+{Knr-vWjy%_mtTfLP{c@K7pCgfj9O9r!ek$%7m-k1(#_czsd`>|W*& zJYxrp?8FdV3T_OzHDN42ryxx?LCgdW6k`J3L=2+HlJMitIjIgHtu5E@jv=6L@+jT` zUm`%B0q+T_S`BX3cs-E(`t0@Vy-xj;yW-XFesV`jfTpTe57%0SZDI#7{KD-|KH1qW z+%EJ#yj>`K*e@hKJ5gU$>0sy4JLl&YVha}+@qj4`ro=jwR-iQlkb+w{6Tqn+Pa)v% z>FocRMtNrf0_Y$R!+M1?&D>-J&wwbeyM1Z-I`$y9^_xBuiUq&lZva7zGvWuk2dElG z9CwoZ;qBYe$gPlC-f%8=iUBfo#MV#QG<4|?%nZS41JqeZUC^F(5>tqr0Z&`^e<%)s zZ_(&N*#l{e4l`ycn#1A%s&~;iIqa3RyQHZ(EC=5n-B=*m#}j24Bepyf zjFvI)xcSL;HMt~ZdlC)DtsQ()(_#6VGR;t&^3`e-eUf@-av^A-==RjnBR!vw<^e0K z^-J<0)*tPRTxI>c{K!%Pv3{4^Jz%%W^t)R2Beli(AGOAvsDQIZS6AWC;=P6=Xo&+r z1}Eu5&HT3&fFaiJ#5-~I8A-#t7TYYV}o3#0<-{P6DX z1g`t#i0^;5@tRpjjvhcNZ+e08vBvK8%YtM)sR#bv3;g%1(7@lF44c~E7JOsa4Uyc) zNjQy(E5&u`1F`>1cMEef`O!Le?I8WHStJ4XB*_=zUV_IxGoOiQz9Wf&*dasPY@`?u zhot_?nw(HF-qG8)XLrwDpWVIl`scS7=NBJ;e15*&&ty;%^VAtJ9SGk&m%QMUbBN52 zfD5x#i+Jin4T#Br z@`)rva6t{uderKv{!2~}Dy8XgKv_kYl8DD=|13s(R~|atE&-0~u}0r2M1A$nMmwG6 zW&laNGd=MNTM+vZ^OR7(w%qGAKk=ZsNWbvpzxv0(g06b7F>3SHafTRIx`)uMQ)0;b%7$KkB z^sRarPMi78j@(@#&9I-&^rhTYC#-CK{uGh2;Y(db_0zn-Bvt$kcY5-oZP5OR3=5?IkQM@50WrQ0GIjPXW z{x@PJ%gG1EJeM}~a#uGT;GI@}@ zdZYD(%_;~C=nSDj0&;9kPVmc^Vj`NW#3Tzym?jDzw41B_>DS+W_wM_5S8u}zA^>M?BvdfF5p!Krb#eY1C$EG683q99Dg7tDCb5O0SO+#;NA1{yEw+Cu+r>f6#eK3 zzGiz67Og^HXEfoxQAN`Ox(3w)(V{>u({`V$(MG&L_6+zS$Az-SDO4;7nB|<}$dHn> zRge|$1j!S!P}$Zpuh})?!KevvIz4sy?KSfQI2rlvRNow_5O7ye{zoSu?_qnP6ISRT zl}!~vGRFUSyeuVvIDk~3+|2m$eTsjrZ3lz(df3-CvjEBYKw0ekN$JTGNd%TRw@O<` z0XFY;+C5@=D_V~)_^tLMeOw|$i_p+(D1c7<5 zA$GQ_=}b$zeNa3pFop_C%|C~V+Bx``ahg(TGe&^sT@czEt zKW$uDK5|*Q=$<_JI~o z@S)w9I+V!}bOZj8Om8Qq0-BDF#=whL@G+WV*oc0>v8p5sFZ5~{jX8?r89g!B_1*xh zIR+Fe$ZDkCi^u6H$RMn|{^6~hUQ0a?W368SIVj4F?0pol! z*>9NyVYx7abR@kc{$)AS2~X9h!}UO5dOEzpJ$#v95Zj7d=qsW=LE=B%nPQ9A;5tb6 zQ2(F%I@;(F2cV5psQ(wV%fVJ3s>5V7;553qad)7D((?BCu{sgseK3Bm`G7Si)c$h~ zquG%&DSX0)e$M&COsYT0_#M9`4wwPJhC#t6LLCM*HxZZo-%H+)f@ThdeY_3i{{EQT zIC_B?`S9exRM-iKMK^Le+W^?T+5oVNJ#4t%UFU96E&dttIjko&xPor9GZ}{_S_)JoxhWpJR^r8+e1q zC+pBSR&`9{Kh2wHD+)MAnmgYjyw7gKF{73o7llGM3|lc-vH5`nchGqaAsTf^Kjyhiu!jdB-6Bgai zy6Er2adbW@6oO&a*Dz-7QFS`EIz%4t-b1~bjk9N6zegt0s2&IY2nrxZ7)GWM7#OkF zO^2Whf&pKNmB%lb8zc24-*3t{GZWO)dB+#k79pB}lJz6y$5rEm)FY;MFqeu)QfPtW z0&xK8!nAk>v;iui`0NsJ^q}ueX51+UMp*N z?kT@vL!3j2LcEz4<^_TOiOa}$It$5UnfXQUJg_P+7cm|+0)U6iPIziDSxt(4-7rSwnHAK4P5V+ZA?<|7TS61? zn&uihKCHKtJRdDYlp+Uz`c~rqw=a(lY(65-p=O3~bCdkuX8*gtv{7x{Yx~dA7R}}| z)6294!NkYu3s)kdkrlhTHR$T;^3f=5Ub}j-lfm+HvMpU1On?mO0@nB6ef#jMuP|8j ztLKltdimh1$NNV)Py={n)o#%$s*#v@15^Yt$69^NkRs$QgZ4{hw{x3<~i!O?pm!l*NlRH%C_X~ZHh5BR$c>-pL zD3H>ykDwoqKyxw_r>>fOUp*WiRsjiy+r#Rh5t-^hskeg*7EJ~q|C_z#(s?He1+d{x zu-rTokMgHGjT%Edwa(?StI>JOTf{2L24@Osj={G8qkv!^bBN z(L`=c0UmTtB(nK=|-FCj2OxZEntp zdXUHS;?96OSkKdfaD|Hj=iDenBYrl{`)RP?_U$!$%mBCUa zVTb_Px7n#hGo_FgOlQe*Pihn(0fG+msK~ zAIeI|a2f~CbSKlRSFd!g>1fOC=Ix_>8pj55qZg z;ea{Ck%=>a$5RzQkT~#J#5+t+P2IZXo0^)LxiQ78Av`sEO&j&m1upRFtF%eMh_{zgfsgy}?Me`H}!*p*Z-4;z|g^5h^s1}V`@KMIS^7t#b3S6Uu8Gmt>2Un-SDzn?55^V?)OD;H8q`eXst z4jw0~8u2K7llDvO?^n6w{5-A_%~-1BMOQ=|KsG)bNEze5MDE}Keh~;0HX50v)Fk(Y z>qC&$0c*%hvOn0ruI2-#lvnP*t-F;nXDV(n1{#Olz0)uy2SN=H9W>OE5|alxnsga{ zi8Js48P#jY$`Z0&ibPGjdUzL)yOZ_+VNcvhe8JK42QR;T{_5!oR;KMgdid_$*XtBf zBNWIv?XLO{+*GCX5{&c&2Wa>@e3dXo!o*Rdp#m3VcD8B$(6F2=)4OS~-cnNsu!zJ9 zLSUpYA$^9SMq}JEOtK3$-i)TIuM?n4PT$Sc@v%QrK#!8 z^y<{q)M|K2GTqfxdWWzC=6EG`FdmPO-f+>|pTs@=35}Jp_$Bg#W&`03IZbU`+t^sz z*znDG#QaAi1bz+wm@9?@kXk^_M^q{)fIXhu^>93}aX7428eR};>gom-4s8a&(!GN* zfDb!$-(su8L7=TI&H{6GfEs3=LsEo#y>d2t-cL&zAfZ`D77FctKeMv3){_Kbh}E!$ zaIMW%Qu$5arovPrs^L)TGwua#S+yy*}~qaag5h3 znGZ&a4&m`70f1%D?M|yUCa#YTx3o~8Ya96G3Eq)XN`D(Zq7t>S1uOV zgjzM=0UJgw(e!BfF<2|-aAwT0zs0V6cN&T?DEEuEbgb%MMfqKhrhg52M}ByY!6s~V z`IKwZB$L@4U?Wcm`2y5qLQ(^8OG5;wc)ao&dk+2L0p`RVMCKxqIbHJ-m`$ZB1o`H; z18;UVB41+vv&6l_IJ+7S%m1nH^=Y|ab#|41#J5>7+lmvb;oaN+3_IeUQ)_}#pC?D- zpQM17o3G*yR&TD}obsrWfSP_JKw*KR@c}6~Eb?o`?lV;%$A_E4oy4iQ0w3_ndSG@z z+kk0t04pn7TY&4f?n?Dt_d!S>J^=Qg$>&?z3tVpYFcT#CLa8u*db(JK&@N(no^uR^ z)=H^>PM?_~1h;UDv73T1X%;RB_S;Gm@SWf7kng7*-~4SIb=dLkRW5va_=+m8FW$k^ z$+{d8#6C4pBK(q>6$7hM_!IRHDeAAfJ&rXo4I)395kG~T9rm%b&F>{%fZ7j=!|WXL zZd^qXU^h<8B-Aw5t^wM+wRIH$yaWNw6rJri8pf~_`^k&=&zby(>#>HCiM4b$7NO+5 z^w7VV1hEpH>OKZSPc=M)?oz4Ok)#ivKd?+7!Gr8V(H@vx7b1twZHQK zHQi2LQ+lJOPzgvd&<4T$CE3qOKovCs_y zPn>L&uQx|y`5m#>#UTOFVRFpmpu++O6a>(gL70ui;TGoT3FdGT+n?n*iLkS?@B;7y zlhZ03g4pot>g;rw#%z27{`h`Y40cvrO{#K=vC)R(4ar$lCvy7n^uV0l6z|Fds3L-g zj0fO<#Po}EwiU=h7`Y@BC`%2nEtyIkC_ue&T8VdV1byfmEVK;>C`QYQ{)1i~t^2w! znoJL|2xR4h4Bs24)AG*tKX}!nV_x zLo+re3~K4s_(t&p{xYym(Z~o-7y>mrP&lbT*4b__kii67RBzg4Ba{ybA{*XaxqBme zf?atzeJWz=v6>m`%q7OP#CWVg0(Yq8sO5n4C8ouR6!P7ec?o#e2LrMAfn)MCQ`DPR zi*F!T>c^?C)9b1mv52mMH@1n}pg6a}1J1ET(<5#6n0k>egQ}c~H@|=U)khEH^wCGZ z{^+BZFJHX)>hb=w1MDNKFvysaz~tMg4N^!9#0y9g5P{LlKXC*Bqz1FZ31R&J3F4>& z5a#h?HLT7+&0j9MIThB60&}wwHRecM{D_47)%VW$=;?h)2_z?Y{`u#B`0Vq~_D_zU zjT>H10OBA4)k%$~=JY3bO^xCKR4hgG95-6c!z%Y>Ew-9f_t}qeZBW+Z3)Qx90KMiW z;{OM0nF4g0^8MCUiJYBMS^}_yciq8-6{f%qu&r;aAMJ-4?>B<;!F)U299G-awjz|c z-lpl;V|jUo7=A00fvmxHo;}EDJPvddvs~wpfx$E+jGqku&7Je02i~BakNV(&=oL1Omh zFHQB5a41$|^LwrKyGC$ez=yP55KC%~HN5PJB4Zz4HGCWGkHqqNYb+Ue( zt64C`*q^>W$*H|&X;~wI*xsv<`djSYp}gc&+W zsB{`HPB_$rZ)hXY+|bes96~qNL0IRlaIQ;$(KH&A)rC;PNzch4!)@J$ZqOJ%+kf=p z;YZ(y2YB%Eg?Iz;0gs+PtvNWxLKXD@#Snrtg=NuiV^zRVjB~TIk_@l_h|Izxh*1Xs z4wy6m2;r>S{dJT4XlUa zZ*7Y!H#rV~<>P5YVa^=&`}o2#0f2ME9`+DaIjfxBT3cRjRwV~WTb^#Fwbk0fa5tu! znfNDL!HYgb*%XU0N6J}qCot&(Iv>2X9BK~QmZiK!naT{6$wn& z<5C7lWror}WYB+&+yH$V37Y}XfMYglfyss$9LH-B0I#L|*@HnZZWwBivIugCgdLOr z<>-y10jB!rpAYOHT&`izDL9wL^B4Q?YmdMA{Ip@inQf-!lt5q!yjE4bc6CU%W`=)_ z{o6i9PIXF&^0|6=yRL>oG8tUvt`1WpuX;MIW+wN&emPZKQ_Y{_OJB2Yu8=b`LYMpr zZQ^k!0x3>rd;;XZvE@^v`pOaKB;Sfl)EaC?m^n()ZXEOxI5LDO1C#1cQZDO`RO%rh<+iZUGn#-B~$h~{LlVrlx_edu_N6ui>c@GGw@$}>b=)?FK z7Q;)FJ$Y(d1P6|x9RYPt`4t%SGX+SY->#>*h&^GkBhZ0J1Sr7VEE0n`Vgj>s;kihd zRs7WK_3$)ee|UEFW>~U<)oW-lhSlmX*83Y&!X3e-R7CrRrGnh_1mbO)o< zG%WPR_*+9A@B!6Fu5(errl0QdnQqccp$w~}uw%1xW1~po1oS;aT&l$|PONXzW6#-s zpS6C*MyIx(Q0Q+G0{oX5fMxzR6h$PYe?_uqVw2V~O?kUsGlKy@$|U}YVS?W#hs(%Y zd;042nHR|ftkh57iGUoZfPNYL%`E>U6}OKxw$=~lqAv|7mr0G;!*n^lV5XOu5pfK< zqOb;;jRRIdcI$~%4;f!62#)!n|GhRTLF@zD@Q-l_1v)shBKO+RhnE)!Sp=m1+TL@_ zI&IUWz+#7(kr%NWbQ?$nv~$sol|Aw`FJ-JR| z;t>p#OB%OhN(MY@)&QC%aMuQ?UzNdYckqMGe(`#_H{uE43qC{s{rvMUpMSo;d*HAf zP}>iMA|V0+lM|qTxH&t=!T=ZH6j8#?ELsB*m^eayNAbZ~tp&o<;VB@4VL$|m6IkSc z<-zQ1Ds_5_=dDk{Q`f~7ivO!PndAc?D8;iWjKp&!02=^?E$rB>hi|k?TkY_hXKzl$ z0+-4v3fynELDyg2N@pZ)Zw;Fzoe3yGwF&IoRz|5fX~t)GflaaU)=)?P)pexB{#CeS zv;U!4+FPLIgT-)uhXf`tAHW2v%X`u3__opV1HC~W6&VBUznZQV^1%S9VBQpU)BRil zDX$pfA%SZ*HX!S-eSg1J{!02Y(XpoU{Z`F}SR6aeUPXQ;bN+M!E~kA>Kfr=?8pRSE z`_28b|9(YD$IjBPBaR>EF0nVAAoLgGvlVy-M56{_EHo!$Is~K`fR6XBoAx^!^|i~o z+|KrP4T`4dff}Nuy&p=hq~updPg(bq|7E3C^g+OYF~)e7MV03t_la4 z%!3}a!R#NpY?HiNjNAyPh$VKZq^P||2ONiPqRX)*YBsMQNHJlE1W3%VK?l4Enm!Oj z^bCn)HcX0RM3mVlWP>1)o45ll^4L90Xb-`70G-@sKgedlV)*w_%CAX$O@abwnagaE%se)7cY0h{!*5{PH* zj-uf}CAqt5dMvMPNw%NP zG+Q>Tr~8{G9iVB$*fY=t-x_8B*o!ZL1Asb@cnm503#xd?u|RT}EahLPf1-ISj09>i z%LYXMb}~3J{O5pVJ&^Pe%SXp06%X9=1!rzXH1ad=mygS~)nW-t^sm`pP5qzK5ntWK zDW7%Bu?SF1%EyUm#Ju*X)`@?s8nFHg2VgDL{-O*(&i3#ww8_AR!d$>BN55&r`H9D| z!JV_y1SZ@w*S2}Sy=-@VV*i%UJL$c}0F#;cW{&EZX@P;j;3r}w^^!M)t;7m;;t9Bp zY8XzyzVU=}H1RlN#RS%MGH^0tXvEDFUNvu$Y2nKQL-kU5O}Zaj-iZCjpMLtQUw!#C z)ySc<{NTa(o&8U%3%Cioz)iS-n^W>_V|7Ek!YcBC)!Cck3_NBxf@ucERbL|z4@{gwJj~z1 zQ_1E~%K7Ndxi465aZW!I-sQmn0RR9=L_t(ssYDjU0RRN7rX}@nIs%lf?mh^-9qf4}m#NUJr_{s%`}3=jeG<2e-)lgG-1Ok_s54 z(>C)@wxhm4)*Xmf9t9+c5%-i7PB)y33E*RWK5X@ zxu$O%Gsu`h{2BQ*Qkbbh3#&EE>;T%^>xSlU40QMJ9D?fo>Jvd>taEQ(Z#F*bQu5Dn zR1Vl1vsYs~F=tEh-CZN@Wu1cB;_g*P!#uoUnwxlm1oZ&&Ct{O!7|}nhj&#|&PKESc zLYln$6tN)@Y`>H_wLIhF>o-6DCpkpYSG_d7UK@Q>pvg={PVFY2O!}9P%vna z7HiA z{gp5@e#(_i`}d!oKzA9c%j-EnPh2V&s-$dRKpX&#-4X_1_W?Ko`3FFsPymztIbZ?d z>ud^$`AY&IC4X2u{0RQ)0H#(q@N+|*!KyfcjScfh@dUH89<_N>0BGb7nD`&AYw!<* zu}a0a*ltO5$MU9g2!yXmc+XI`om9Yny0w)~Z!MP)@+B9Cu%J1KGbjy_{kLozUrc<9 z{Q!D{s)ha41AHi%!JRvI^722LzmwM*Ab-jCyRDm{F!6I6+VL%Y3d|~!(UD?)?g%_) zj8BG8^QYv^8%+i0gE!@UOaXeqRv&Hn{bbo=xr5k&;gO+IG`9AW!G(0l&-D0ML1_+P zvq8s_NzAE{n6&>dH8<9c`lrRkL<7vbam$gnqc^BO-YnvFwgE$e17T|dQtB*c<@x_C8 z?=U3Dbp+qNd;DtGY$q}svlFoaCxTIo=iVL`Bi&bsY zZyXGA%-b;`1ril*Shj(JgI=A&cNB@V-oWBO22o}b;6If^#YN6IAsOmsC^H@ok{9{t zbOA*CriO#Wd88XsSt-S;M!)x(>~v#FB}h%@NYjWOZH1BODDmGQ!Me5 z9rHJha&5g^nn+ACJV`1fH}G|A z#OSXsfVmET$W@Wkm_6n?S7=d4(|W8~?NM9bOkVytk+b!?W*rlyRhB+LKE*V^_G#2O zCXkaX;NiRPCH9-OC)i%{?&}k?!DTYY2+$9GOg3;Dx=EWK0Gob;*q}Smkzchw%H)`G ztzo4aVrEyb;|#92bjP#@Cq9G8|FM+ykDmXWb&lWu(Di~vV!>eNvY||Y33*gMD=H2q zJL9MOW?SRuj~;!A%@WU_KOLV|{7MKQ3J|#p39>T?G$1~wZNRyJsp8QS2!{RLjU=S+|KX6; zHjq9*eI<11+I`c!Yc+d4iTse;NLxc^)lV=FYete9$R$`CXHc*}A8Yxl#QfX03no!0 zC;$M3_nmFtGJiYzNpL>8T}aAfR;2>SaD7yv9L%~vYAV@yeU*aBAu_L6_6IssHx_+2 z=j4seviCw<`;r$%JN-#vBf@{i5}A){`mNY_h521Y{f8g9AFY34n&Lns}P*z*^Sy**WxzDCyyV#`}W zsK6t)b*Z|a*X8?GVZ($6M@9PsNSz1}VoqQ8NxBEm_CLp-HAMq}3xYem|LmZWB_n9A z7y(7idI2&7X0@!?Qn}r^>G3I);!h=J%X`eCwnBJga z(sRpKxq9r6YZIe1DjW&Z$IqE|qc*=y0wB+mEi)ynAvjiIF71!k10TL-2t zQt{kGYk=MXOOfDZ6#U}-a@dG%i-g?w0C#O5NFKO;KL#Y=0W{O$loa@Kuz`f_YT7oY z<>_CQH)alU1^*nY)1&P(3)lQxN)2Tjg!Em;`z&6^^>PBBOC9l7zmhEA!OI6PUp#p5 z(Srw+M11rLBh;_o{_yk9m}2Zox}WH>r{^FYP=JwTzDhLH)>SW)8XyhFv>{ED$A8ra zXc&TmQ=zPY*KBmv!C}{+0*>(_1`u*dmozGnt{yP(3Q#=lIuw|oI}PE`X&l}E{L$mD zF83_r>(TR92S@_Iy!~YkpkQE%Ljz<5=zIWq!a6A+X+Y$XEF2G$ENpxM0YCm<%i30K_VESd!g?>qwj`j<| zPCW%dwSccvA7$}IR|x&F?icHy2~Gv@28L2B`e8{13|1qZFnuZMZ?Nfvjf)eGF9n?$ z|JBXIxBE>S_;TPsp%mT3FPDU`|NW1g9Wf*--bk3Dft>@m5oO>sv`_>GPtpy?Ac2n2 z6PJIAtxs4sq!;I|GJtmtKgX&Dl4-{WX@ECFD`fKG!iRZXTX>)X$QH99e}q23(4_nh z_|gMMhZ=22$Jbb0+B%n;{377zj=5!EOxXi

CeI$Uf8Z)!yVnLuX0)p^){@cOks zQ*mSH@d3Q^4!S4=9zJ;R;^D)WSTZQ-fSmj7I44i3d)yu9#J}Ya#Bx9X@WT(s4U1}+ z&Wwxm18sepPtL34Ugh#?OW)CwKq(Qlc>@7yLowE{iGX1Ucw-X}GU@&)X+mYZ0Vwd; zf%%)MlZv+?r2wjc&!qx*{@G{G_d(2gb>syS5HVz7bpOTfiz(9`G&CW}2IgiX^a6$+ zpvFFDBfX>peXc<#G$I)}yk&Nny|eQ^PG z*7NfVY(MY#qUWX!I2rBFWU5#mxUy2}`vRP{&W0h9BJXl2We^R?qOubE7jJ~|i{Hq5 zDdg`p(;YOq2`T;;TpvT{zcTN`h5h{q4Q3{WOR4GWRbE{TN(^C!)ytCD`C6;9Cib;}2~B*dJmjbTL2 zfs=FGgcjnicQORWA{I_GUR1JLHj7k%fF}mwN!`ua6hIMg*Y!*3ou{Nllbhj0ZCA3lC{5Mzn& z|C=Z32Fv04Bs2M-X>7AN*kE`XSS z09`-YJo*6v%LbS;Gl5gVg&V7@op3!~mkipp2RBv?9x&ku@Vw`8g8*g?!erM-**`Po zgUAn;s82oVxW@cFLI_|%c`CoDeb+rU{$ zdv{j3$|M|y%uyH{OX~r7Edi(!2IAA54x5u<^!u>H!&<+B1Dl)Gt$s2XAYmC#o^Klp zRycqPmbe6_uPsdl(C?Qoxk!V**Rfn0JMDuL1a^KrAR9o7LSy-k3IqBtTfB&ZAx()5 z5x{R|0oXLjTdvdOwwt2b4#VAKt9y zqE3z#iiTiQWgHtL>rNVTm$}EfHUx0-v6>&9&SVDGQ8`WRFtP>^i_Bs=su+@V)DKiT z%cd#b28982TDfMP!sScUNdTQk${9}bgoQjddqT9_A^x3kENT`qjfY~P=aI4Mt`wkQvfsYcXH%f zO1}Yj01bMxcQ3{#uc5P9@tQF&2ibx#ank|d8foM5fmOIBQs8&jaKars85T3*q;3T| ze9-UH(h8solSxT@5K&?WHRw3LdGqGkGb~>{d3N&bsKI$5ObN#=D-an!3izUD zsiek$A`ju!&P8~_J_x^m4_IRLT+)W9$FmTPdXV{(NeB)9e7T=4oOe=YyfVe5=h-aT zR7Lp(cN&%yi6kq$-zY1U>oOVRNtk9DQ*I;dYl5(m`O=ctJK=X2aF@z~Aznb4ZZJR{ zWw$5H6g>jG8DP)ZP-!$kC@T<4Tmk%b(8aLwyiP(6t4J|~^)LQxD3YZTorH~u7bZxG zP{Oi{^o}*wlWcsY_inLjcQwO--50Zgc%)b?A|r=d zLTKXj@cvneAISS()&rL7V1s{6)dS31fC=BR&N?&{YNMgp32J$PA#WuDg*_aX@txJbij6wD=gBc@0!By-^+OiR#I?QZRSZ$y&J+&u zk^ITLxAH%$LIK1Dy$5ghIk>N#0qqbxf4V-F1TbaIf6kca_0{X)0Mh`b{^5xHH^#5f zfA}WZzpFPD@3;03`2TP80TJ_?b^~sDs5G21DFEgIVLWX7Z!E2fInQiNEp1G#PWi&& z4d|7Lb6J(jLJe9jSZEDfnL{)Ow}TxR{KnGKw68Pmo1$R}w~XP&UdIUP8C;vu3!G_}SpWwBQWbwOUJQu!2hdK57*z`vLFgNJ8!;38 zbJ%5+;DdDSykl3zd(=UV)eBg7(Cj?HG=R1p$)n=i3{VACKnP_4wS2|(r2GyM67|Pk z%mZ-5vO#GeqJ%KTfp{-hg784SnnTv4;_hmNb6z&haK)x!5IBv zxdO27IK=l|Gc>G-DUS#p3xKos^8<5<8cileVD6L?^qb)dyuOfT0&*aGbG3uW zhXcT%-sjJW3OpSBae_=?zH+ros`i`g3S+)Z-U{ z`1{#aZX)c60}$f`5?Fv9|Fjr+XH{<4(cm7KnEU(p$j;qU&?g2M&OODT&iLr)_;`GP zvj@;J!l~Ohjbp?oU&fFhRlPE1ZDBduiA$zPpM{i2g-gVNMBU{Dfi3rQlQ=sW2R55W z5_W%)N%A#Yk-UZO(H$iC!b5BX!uM66ULj`BcmQhnTq=YOj1h^6@-)a7)INZ=I{7rr ztIes@$S<)Es|aLnk}yO2ei$Q0o21M!d;~ax3g;mFA%u$uF`EF$ z@Uj+HHj@H+@Yv|Q;k|<=0a-5#43}jFNIB@tJCj5odccmZ=&8Vsm?2M8LfS7dq>+K@ zfea>Qqp5*hj~za@%ZPK)`;NDt}Xbr8p|QbCZTkYIA~2E74o3_K!l@cFA} zySw59A`%ow@j9mQ=jJ9eg8_^F1+LGbE&vH|Dm*=H>A-eSSjmBs4}@=;#6N7*L0_liLbtpY z`9|42v1r4&XdIv@GZucdKJFUrl%peK^w<%Ji^k*A>;>z75BHW<)4IJ6WWTa6EB6K6 zMX&gv5Jik5kWj);I2pXC6rn(l_>-Ke{eCYiPU8e9(PN7N3*&h01h^JzUWKOobl3Q- zTut9X%#}^#q8Pn85_*=2sgtgrFpGa038VQD)7;&p3sy+f(dHMtl^nc&sr!((hy$Y~ z%47RkgVULz0jDaS>l1?ECG&@kHY5IH=Y)v+i3g~uLZ(-tKjyB4;Gm5}xm$F>U5@^O zndSn9ju+>o4iI+~PZa?LautPN*>c+N%$a8~zHA2OCL2$#(4T6@U9EpqSV<}fZ$bq% zGz&0elzxX#f~^)j7_3+P9{grkY8QF7#SpV2EJ;BzW`^>xG2P((x{<+ENS}Kh#p2Py z@vB&y;W!!7^lMG$wVgd235}T52cm7W<^eQd282`ygpK_xi{m&6s7IV%a}W43TQ(YY zup)sld9T8JA;PPu_VHcPpT^PCeK|dS3QeK)agI~xKt|ZCFKR;&bAV_C1`HT58xDl2 z3lf-};?m;|IDjMo1eE+YeT5s$bJJ74YZBw-%uNsEf2oU`rIx_PM$m`~Z%B?0M-lF9 ztgcEd049GS_CLbD(f5!GlGHtmZQCOzhEJ)~1FMxJHji&QBn|uT zj*rRtkn1-p7rw>CdG-mx^l7y;_V?56#dxE8w2rZAh_44&8vz+Df+Hp8rDzGyMxg8O zHy`b(elm*u|D?UY$-wtSYe&y{Z4Z}L&JQW&+OsGnLLtpc;lX2xm_d0|J9vWMYcI}> z>5Ov*^Q!6WjQN|T7UE9uczQ2>5(+8`5#d(|$^u2^X6URxMdG0xe{d`i1F>SF3bVt| zwI$Ek&%ZP@@EZrf89)tfG2`eoQT5_AArVdSH4?^hVQG<6Wi%I}1HcAmOjL&W7U+6Y zenHlI68$yRnaCd4*gxq2FkerYv?rl=#4#fsHDZ53D|DKJA2HbLvLWc`gn&Fl?r&iE z?H%a+Wy8R9Koksp-;kO7s>@1z1M%GD%LC!S?^cw$?^o3WjzVisLT3TVj6|kTVKNV6 zuNb-?uBV@ho;DeZHCC{Nm!m;?l-V zk6U>kdY_6*rT~lYP;5|2bvDls%q4$1#kP@Cl2DoXV7ZW4OM1#?HDgLc?cXHw#|Pc6 z3GQRD{qfNo+W(s)WBl+18u8`ZF@pbCtp7Y3oyTb*ufjfR|M`oo+oji#>)@@JBPW8@ ze?*(Y@nz0GY56sb3Rgmr{~Uljuy`Gdb4WHXbp}+f$@IrV67pc) zQH+G7s4C7u7g8=S7CG&L1i&~{BUz>?KyD(_g9kgdq#!%_fM>*^eiC$k(%_006l3$< zjANVdDNn$G-z(x$v3I{42^HnRV_it|SGZ|Ya}n2QxH2reKnj59ag3KAIXY_MEqDX@ zpauhnPQ{OHVOf_GePldj)i{;vK&lvb#MtQWSIk~ZcmR3w(Am)i7X0iT|M0`xN|t>+ z`S6q>;?+;ixB@@HB)=h3Xzp)p-^gvGw9T*50QT~%_w=U&4vpAYdGhS!>CwSyh5A9z z1DaLjKVU>cS=LX3Fqg~+gHzV<_2=~T6wRMmz)h+GF9qip78e&c!~vX2T@FWvMPgVk zh^+`})~DDZsCqgFjZ@IvE*HFZ^zRL|czMpq=&f~Ys=s^k?CG;-yGP?U zngZC%*T?312NL^(3kwTy0OfYy?CwPH_hcJ-UC9qpp_IiCLGqau;}1me>Lcnd&FGr) zW|0FpCPh=)0CQDV3134zgv*h}$3j8S+{Za>LuIq(Yz_39MBsyw= z*G#UV9kxrkpLJ9BOP-+6UyL;enKuZzIc@!;dI{l865mdOBMw@i-eQg7^SPQ%g8y73 zE5W~rG$0k%*IV(LT`4gGaORA*w=OBq@xnyyvIM}h;dT#5pXZE#w~B!n$x)UgF=bjH zHoHolcQs`$`YVzG124!Wx(~^N&!*7;#%Sxun_A9_E1HUl1ZuN$Knad{`Fy0Q`Im zQ{-!!FP|y(XN)dj7Kwk#Xn{zL6@m%?%$xEY&iEYEZ=A2zky% z!YuR&_=&ksZv>;z`@gXP&G6}IH~=aFgf}o4s5k&T04D^eHdZ%0F`#lrGV}m;f66d^ zitOt2;MB%=m^fqnL04n^0sbFPfIHgg7#4jX*U$^raU~f1`kOcW`}!Mk59X>gf2{}( zfcY&(7v|@K^Z6vHew;FFN2ls>cZ|FrdP>|>2J=_3GqfwhQr?6ER8+I?cco$`vUE7d z_Vi8W@8~!^%I`+Z&Twf8F2@NHPy3l0m#yC&LnAaborQO~I7GR_ww14BS6c>{3ytiy zqHhfatrE5bu|9?}|7}pR>LS_J64o^c^j2Y3QiRRXN?#k5d@g*3iHM&Y;C;-5G zgG!tYRNJIQxQ4z$i{tP=OKfdX6$A0PVuVs^krY#fiq{N1+jJk(eK8#|(jwpFL2CQj z5!LvwOkfla5b@E>o%|!OcGB5bca>~lY{`_(u4H6r2@VS|c z`~0!C2%o=tHa?ZAK^uYLxN7}G)CmM;Z%Y0=zbHk(^pt{s!~h8l-h>hW6M>r>Q*s5S z10FH<+`;831~;I#p1@DnEL@ag{s2gxHF%o&v4VLMCxAPcxP$RQ?LaTH*T9IV)(7+s zng&SXzY$zu>c3D{>*suLG8lIn>OaO^m%>C!ib$oXoC*if*_aMswqO3gx17~WDwgw; zp=g?sT`da8+IDn_pLZgC6;jyUfJ4BdNy(zSc0&eKU_;(9054=jO#RO#cc!OkXns0* zS2LvD#N=n<3ijwYrUWpJCZzbsC-=qGKYslDgvF*lA-_QyVrKcx0h?)woTc9->~yix zZSYe$KQy`q=L@6+a49mzlmZP1KYLfi$vCiPpeom?IM^!*6X1axfNWwB@r$;otU<@I ziv(i1nq78cDxE6?NL@gXJJ!)XWkTh2fS2pSy+&14E9(%+b0N)`SuYb$Gh<1iqDw{3 z20urb9UFXNg_KECY?C0R-X&hJR{>VXHM~EEb4~EHEVj5+Z!|(aA-~{7V}eZg`=x+- z0vt1fJ^1iV3IJS~3xCjx2B6490(U~=rqCQS`3m!aP$+&eVkK7znOU?Di4|}VS+=l> zDi`!(Ko&KE)SnV(-QC!rDuqJO`<6qX!F zON6d#RD9W@j7YK{C)s%#)4eI{XH!OEF{GSduhR6bri+!jGeVv=?(^#k}|+_vZeK@4x@{ zyKle!_Sb)E&FhNM%Xl@dtQdW$`4uLV1xA2pN zZw8!TdXk4=z#U1jB&C{{$`_zX&~g9*DtxK=2=3N&J}ezYb~HvfffuD zTlRA0A-8&(H_~rt*CYb#2Qc|Gv};BtG=zY>Hd|nlz%t)u6+y1HzNy{{>H}1IuD1I9 zTHxyx2XT+Ad;Ho3)++YQX`M=lnE<6CvqhvGB$C4Wu(8Z5rBY-lgo;X`E5cWxRE!&Z zPoJ&cpBKK=UwM7@mR&}rs~RalJ#3i*;67LhFa@ArU`C5_)H!&wuk-*d1ipOq`SX1U zjz05JL<|E#kuc@~1AqgkI~yGOAvho@GDSc(s01QKfx>}6_CL9kKkuLmfQ8{=7V=Hr zD4d6XKIkI)A6ym&Vz}Z5?5_vf+oLrf>FLd-BT$bqF?4kO@NdlYfq#LLL$~3)51%j2 zFL*rj(Q=_}m3{h!XrR*2+F{%{%_ST><&^ zg!%&A8eme7SS?0L0oN+HEMpEkw1S3v{pQNQI2vgu6Vp79f^f1cqXRn zq5ciV&Kzj6YLw6Km`{?a(dAB6JeNVjaG}gjU-l)u&cJl$U|EJAjBc*$W$+~pNq$wD zr5rqn;;v)7MMW2UGrBt=zoS(h8QM(I9m0N!5xWC-4~V{4)|cOX_p6fu@1CEn#&^bf zKiV+j`OqFw{Z7uAz;Y|FZV1g3L#cR(Kz)eV6bt+ihHD}Q-zydgP96XJ^Kbw0AAft& zt!UaGs!ZenCkwekss_e#zygC2%_T4wW*RMthVnT$rkDaNNGu{AS*36|hknAyHsWUf z2Q|U0?hXnVtstEW3ZI`B3i;^Nlw<&>&M7cIh!!i{rxvfdy)k)f`+72$+CqP9FTyR< zElA);-pZ93u-{KH)IT&yOCa$w5IO58JKs=Jg!UF6{ zfSa4zxHc7t82E(MDN&Y6Naso!2b_t`tMz!TC(!i0!MYJ1-+b&Ttsg?#FRq~Zgx!vQ zFp9jqdza04S9@h%vj-w}WhTTb2r0ps6?n`9d>z8AmKnO3g6(USuyTQ&Oor=*C%J@3rGx{u zF+Us$wC_+P4t>fEWCS5rq*Bjw#sA?;_k>a14zM?jW z6v=Bfmyq9zlp#{_7qgw6{9-sxe3D)JEQ*p)oPiGtS5g=&DJU#8O~C95;qm~A1JVaj z4g_ic^hWp^hx8;X@FdXHb51qaXY|M3>H}l`iOW2|26WB*!SViaZ2wmKg-_r$YcP8S z`{WN;$B>K?Hb_G3(nw*N}tqe|r0S(5Sn7Z2Zw z`G52B%W;gYYS&~XUKOd;q+4?gRrjuG08#fNrz&dZz| z8NXL~2yVlX02u=n&~3oVr5*qdBGw>rHKn`G`NVbEmZ36RlA{?5)ZSd2gJ{gu>14m0PD5OE>0&^Ai1|<#yVf`u+S=%IWGoHMqtUxee?-V7I6TSj^0ci^Xui zq)UDpeu$}^!OAq;n=&1*4Bxy)zkkBpVsy1jTX9eFw)^2o2;d;4gJ)1x#!w{7JY&j6)-*}GAFZ)CYS>vEi{P_IgH{X2w@Y{Fa z{54U@YQ4d(9Gnjp0(j|Th`uWa&*anz?YbY=Ov0hN0&2$c*`d=h!-g4fB{iPozp;v@ z;LJimufgyOsrh73`(u9Ng>(mwW60ErSj;Towcv(6BMd2<1SMw@fZU~mFWm&+5k1K&EhiO5_*&&nKxTO6j0{Cu?cK2d5zVf#DEWvUA}2EY;At^+Z)uwYg%Xy%f7XwF`M| z2f%#)$>7T&>bHBf3CxcP#RGTwQj0C6!V;0$%uGc*AOoNsVgz%Pj^{TkJ*hYoR}`;Uq|z+*OzXry8jUGk1enqe z;2dYxiV>-211W{IFxF$0f2raX2diy1fX!eK`pTZYe%l?mTn>hOo&!Nx0M6C`28X!& zm%4(VfA$%&o}&Zjy^;KvWPch%d{Z|!FdneCclLUGbea-JpgjMZYX3Jqrw1DDV+U%+ zIAa`vMEKL#siP#n7!iGvIzjNwbrI~v=&8-j>>;N`!11NU&yF0Ef2OT*1F?Pp`dtV1KT5njC>j;9RS$3*VX`;hn4&gFO&yz|21E$*+C;Dm&Q z>S@v}2qBALBH)JxYu1!pkX}G>0RT9n!YsgmR2kg5oIf`i zGfJrOnC=0&JuE5w#4Y4-K0W&U!NYIAfdlwD#zfcew>59bZyFk^az$s-t+Damu-s#V zKU$?ACE`o)k1+>F(qK2-Hq1aFpSkNKxXuR8d_aBh7$=U&Lg35~t0_1vRmGp^sN~Gg zM8c*dU_KjHpj-&bgBa|8QX{BPvbI1~u1f{ENwIeQECbvui+NWIHKhSmIKjh>&h(h@ z1p~@25+!w=qem|uNCovh^n$RY`_W&;4}1z2&@eP19uu*c?FqnjfYJpY!xTW6qUz?zT-Gi>p0vfSXyM0*W`043{N1((ZKhk~ggjuS8si?H&geDlW zjYD>Il%A0yHmk;uL#}gr@*9@_?;d{i<18_{R4C47o*&qr!K@JS zQE_sRC7vCe4bILaHlH!MkZd0?H~LI|LXoLt|8ptzkh%K{;T`Zl94|?*hQXs4e!@GkOs3Y3 zwRa@|@-IphbU4!vuR&Q2x*>M9= zmf(01TdgnB~jiPRj1gQur%PEB*LcP2Qqa%cmIIdNqUcX=_l~e3lPPlF$L1#gFj+7!OeUZxHX# z2BhfM*dO)K`uj-;{A#ZsH$P~2#nA?4<3`fNV1A~H$yCXtPL@AmNY}>9LlcFr2~XEg zPAg-IO0h=xPYy5JIV_TMp`~1r@GXd8k_kloO8p9Hk5E1|%^Msu9TIa?1IWP+Xg_)&IxF5@0n|9N4?rcm~^tE|2_D z@ngOG!c?@BPnt?6z6=Oav4NDU$>jGO>E3OpdA01<;M*Gi*VlwzaUt#~V_2jVHQe^##G zsTXPf+S`{uRZ_OgKVcMwl+hNB*ToWjn9IB7+o9@RaO&DTwC{GJ!E3i>e77LWwL?XG zpKoe+U4j{I4f5g8>A2(0<>PV`{&}cf7`&E?rd<#N3tbK70__szgKy6 z2K~?k(Z_k~6NvaJjnB`~EWZw1M(mxWtja1i#3ID^BS<_Uz)DCkVhLRs;Xs)7vun!dK*nsk$)ksB!d2%>UQ#zWwIm%M%x`tYVs8 zN?f!RJ0IyMt1e=UTYIH+y_oYkxK^FHtkX3oJPlEo|`JL1pA{TY{t6gSKQ*f@?y6 z#;0Qzk`n|W6-Xx|y(H}-Ww)Zbrh*ddRZ5`aa(Xh9*oWPJ`sndXME;i#UVimLiUBDF zK0k3NbS$|!puQ~ee}E25_c7)n-+%~&K=8>z&;k{JwuKn_vy~JZ2Pp-Jb!yVcPiM6+ zZ%!x{B1-Oae?DzM-^S&XI+oGXFVyQDABDDO<{@<}P5>g=ojo@)b1}Mr?nfXVUfsCn zyAce|P?$S7KR+`yb*+Q6zT=zf_^vJbwjq(%kA`Ql$cH5}xWH8NnL_42T?aFGm5abO zHm+@8L7<$Lu1V;}Z+jN!J&gkuJ4V@lKxyRzt=$tf0|l!2AN**{9WbDpKC6Qq z5(H8QYDy#^m?#B(7-GVCk8=FE4KO(qPC1PnMB;!HHrzP&8=_G>Q&-?^g%l z#jv{^Xhnv9v7Y-cC1$;QCuQrGZ$d0#Di+T$C?*jzGi_*6dgTM>5{WAUfcyrz(YCWzQTsE8C;s@VweDbh|t6_5l}AIIxj{%(w% zes(>)X0dX^bVVA*5#pVIj{uJ$;pkp&uaWMEdCjUdx!g6ikxM2ZJq5{~9$x zg=qj1fe>nb@cz;HmozG;qvFEq9{eB5)Y9f$620Yu9FG=n3W} zTcEghIWK8L9jkjJefD)mBXI*-+Al#1nBtpZl?CNSFmqa3y0Ia5CifIwSPU+D%#fd& z`j0FIE?a;H6Oy0800+66GJOvw@;>DJXcdt&#_yz7m&o#2fTDO%yZ!BGClw>FfK@{* z6*fgaG|Q$L_h;$F;H_2xgaohzg~9s0$=0oGg0kdoTVBI{MdST@YIk_)sNWMUWZ?I6 zW{A52irlm?$4vooY6@h*%~D6fN`vyEA*}{M7EZ^RIH z2@AxmRJ2uzGIFpX{W{%$@$kV*vHq_6?;Qj`F zkC`hnKs$~$@2R?D7|h@-ICdsN-YYLt9*JV;S+ne6jLm}Hf?g@vH2zS`h+e3!5_15! zJ1!Y%sE3M~HAar@5}4gw+(ky^71!%nSw+Lql$yegk?g0;KZOGP%9{7Mv;YkbFEVfcgnBZqwil6@stFUCjlfV{+FY7|#J zIlrG+Uy&0#r29Z)I-ccNU_~b+H9ep_3d#O-iGCI{IuMja4GHofyv3;UcJKWBVl+ak zAcaai``M?Te(~vV&+5Kw6x+TQz_2^zu#OXhLD!0Ch>7GFnw8;9-Lh8Vb5`r0QSZr-^&8eNQ_vpK4FEH#pw2un5FIX{Ee_@k*@0 zL;>l6Sbv}%=D>pZ05X+z1!29uabsil%2JeB9FztFFhu^$=`0&Mk!&Txh5(*1qa-@mT(x`?r z!fV=}Fq*?PJ9VTy8>Jm+j?get~E_n$p|wp$Yu0vGM+{yv!jk4_p0(n#y$ zS>kdC=fz^^q8=A#N32b`x|Bdd{$Gp~!T!r;0e%n@)+wYcOEt|wwA{?nBc!nB7YQdY zJ}&f7cwj`pDFhGTzCJp6d%E3ylautK_#S^NZAwwUP@4-Wy{5aV^oUxb%& zCUVdTBQYumqK?=(JjN%R0TJs|nAS3oSvC+|3gM{bT)kX6IQo?6|F_@1`{Om}#fO=mGx4RO;8x9GEc#JCI~2cC>jx?Mj4G0C?JI&r(9qtFKe6 zEy5ZkMezaTIxtFA%yR+ULATeX+&%Han3dLN)~|Q~#|{$CB`^vI5<9nJ11{+B8c{Ov z{>VY}yZXYts@ml>+F2U>wZY?hjcTah@cO;#AzUl7>oP6Cb_1l~5C}fOA&(_UjoJ}} z=UzRh+UBFrPr9dGP6k!7l(7TogH#|@*~^k9Oo>@Adzc0o?oSc=T$qT0zCR{(KphTw z_2IpChKOPm+(}uCOBkP)`**>nVcsv#_~)PBc8wp% z9W=W3u)Cu}#=I57u=g~aZY36-|EnuS%mzLpZeU~TxbhN%g7PvibvJgzeqab2t=zs*{KhLlmBK$kCMatM>2 zhtviuArpgCX3B3Rz`4V(p+h=qY(}3h;arsu=gc9XEd&*FtT3T)VxN7l*8lB;uU_S1 zx`Wp$87r9SOw#GtPq|nlVTX>*Vvz(}eun*vFw5-UwC5TI`h&#|@Z>7gt&7q6VAJSq zVGsxENl>-%R`3k*W;*t%xEP6mPN%EgdcUi!0I$);yi{A}*ip<#55*D_Ml{uubc8=P zPz@lj5mN-J=jfD!#YerG+F81WfpsfznwZ$5NE6{V)$@?cf}+N&v3z`7bBdOKisQejEY)l&g)FIn*e)ozjNupph!pE$ zs`W5hcgor`8!WP>iNt1LDq@xUvR3UY#6!_E1g#uC^Ule9D;9CqZ%K78mYYTTUfFy5 z^UpuO-n$q{*3Xk9{}osK)7gmT&&3!?k0+ljy=lV3a zc5New;0-tiD$R)FGHb|#RGkU=V=6tT8X}a=ruOXdaYD!Rc+BTWrscj!`Q4D&ow6$ISFuWf9msc{XOtm>IvR zS-kLm=#Ms@^DO z^1^I7r0t2=<&*%}dE9f5(9s9v39e!J({WrbT=eVso>kARF!zYyjqvJNW{AWe3IIW> z1D}VRWmC}AX$kcU6>519;~+1HAJ%yP@uSDjU%h&|j}0P^9&;7vBXEUZja3K%XdMcA zFny&WD8j#Fb>MZ0)J^*zE8`U&UvRYUOzfH8jQ~6ba8R9Hx_Sz0fNsydpdUCudb@~<21A6 zNyH9*l+WpuKATZJF$w;x{1eLd*TR2YdA(f2r0`+Po0^>p&U>Ke2PyA%+E{;nDxR$% z0Pv1ZjLnAm1JD6+yzR!-iRw2|CPJvhPV|!o~-D-az+vARW?$n#w-VK7cOd_Ubqx^9yZA zSRHoQc_l<8LShf7zhOt<{koRk5i+X+s21d9o<;#bcIxQN=$i53{Nl4t9=&||_&Hef z&z}<%L>2%q@cd+~3lKvb@I$E3q955Ks&}FRbCBXLDq$y}!*z4l=XBKtenhatE*zG8 zwceSVEZ2>5#Fv=>1*{nIvBwwR=He;qP?X$L#Ks?B((x3V3-OWs-`gAQy?y)ko-gRZ zCay&vnugJZ`I&H_(+SS_uxW1*O@Uz0M|Dr^0-(y@G{irz$t4@Bfvn>8U259Edf`TtnwL8`8J^G`96S@VAiffe$To;^!|Os zg!A{86kvk_Fe6k>y&9}(`iEE{hDiQZnWdDr^&sv9v@grFEITLe8XE=Vm+KJ@0Z^UI zh&CF^t_lg6T{DiQujt&7!x25;>@0#!y`NZL&*@Q};>f6CiJ?%@u7-uuj$GW>fAH|# zx8J;b_we(vgT&iT0K_O@p%Vwh>ztT{`Dju)4`hY`neWH60l@9w%FC7VHy6u!O$cU} zQtW)7{59P;B+u71!Wmhv1dsp^kKj)cildv1SCn(jVLjvoTCB(9`Y}d%JU7SCjOeEv z9T4ymh>U6S--ua^)QEU;{!mx<$a|_8l5WrTa)JgQTQSQ;*;3WH_7l}UF#9rQJqYrI zl$%9yKw^M${=J|DiF?aBN6*co9YO$)$O=+Nus@bt`im)@@5iuSrN|s!l5j+B8g#^% zHN!V;*j{A#xl9)%GmwM%dVLyua;7^0)|l!Ia5p0}NyO>kxUB{t=^22i=NvdN5uyZh z@!4=4H?I4ZDtjaFeho3OZfE?!?_tcGnu6|ky^bcp#?;car6oN4>~(bhS>7*UdDg+0 zL&F%A#_nP90Q-M}FL?TF+=x$a5DdKLAxX8t7T}Kv)*u8OShGJ+v#-$`Xuvn?7f=F9 z(a*deeMt|ce7P&UU!I9mN09!)sZeuVl?f6y>_aX(OTm_L>%NT4H*CcA4E#kS#;DIlb7mdW>a&hIEo|# zWs7sf4CJwpeN(uFq*F)tH}dsz6wsgGMlj^G4T|hIZbBR39AtH@%s-|IZ5zSMfno+; zzgl48(E~~TAHI9|>8r6(#`jwT<4dhz$K;^SrnuRfUVWP~2-^5F(!q zJ7lDhKWAp3*g#Ms5;B?!`+yb289A#3*KA58*9dhMF?yrRh~zt)5H7(}7o9#Y#x*gy zS@f%RA#52YkJrbRnIG&8dFtNh(9?YM81sQIzj*obS6_em2xY*hFJC^7g({IE2M>_Q z6ZwPvtKpNU3;X491B2!HxZR&gORRIjCCncs0xea3r0lNU6gO5u)gK$HC>AP@Cl_|# z0xnloUornIiv1!7c##PQ!q@3aBuTDodMp+X!0b9mz!#m))YKGqbzvl6dP@IY>xkpp zdp+t*O|#o~ZE7hvb4|QNs4*VzN(LY{zpE0w+G4CFyn!jL0X&UKx{t};D=dH-aDQ-o z#&8Y|jtfIkR$ae0vWmQ9o=VcpE-P(so6+GHw-XGzyX6e4NWP_jSpFgca{Hw2{>fV5a37aDdbm5yEgD0 z+ZE=qEdh))B?^pi|B5ERl}hE{`A6jceDQdfW9`gH&E&l<=+p|{EWC&@y(YRJvv9we z`;l`$>F}Rv6eJo3f7gLsiT;ieKI42H8w_SIg+Em(BErH8xLGv&u>sNX`>ndF-yMi; zP&Cji+rYrQ-|WZLZl?}8Tv0hA6lw5>Q*#Y1_A6!~3F-a1euMiq3_%y25~b1S)cf<3z^t5mB{E@bgugr`t~PL{HNkW=PH6R$JIx?Lz|ob2wN8~}wtdcUn7 zH;#9Yj&bTbm8^d7acn=r3<$Km;&VEq7hp+Gz&OG|eky+8z>+!@c3wxz8k?=bw!vOK zA_drNk6-QvG*5uhvk{o3I2i6bB^mzl@g8Km^j~H0dXQxk9OVW433h4fFwgY$z%+Ab z8vE)Lt^3*r{=)-I%}gzAOfOvv=JVV6^NZJKqrL3(hHqo(n&br6W_;N27uG$z@>H?0 zFHF!)PIB8DPzAZ+p|MN!|FNNre$D!8_*t8X0AiCJVhdRQ&S@pS8eG(pUw#1d)5xC> zOVJ;L@nZxe#cku_xK`Bx!2Yju8i5LhzNHO35oA|$G=Eh6Z1_ct$iF80n&SUA24r6F zUP-~0)N2%2IBXpu+``6RK-Y%=Q7lGu$ajuoSx{k$>%?W86@K@LBiz}&r3OsY>||iMo^`$ZBl%|t!9Q^gh;|8>b?2CWb8BJ32o5WW8P0`MHeMAt zOA$*&{R~tMEIU636+o5`DZv`5fn0tE90Q{R6`dHsc5JUUJdyca@*ZObhIFiEs!sl@ z6r;XNjzcNk$RdkuI{vg+ zIU{mCTxX(=Xdd6YxA*$r&JM)9NB27SP#Xjcq7cx+p%~YNnb}dm2=-jR9*AqUAC~+* zd>s^coTK$mb?T!m#{a}-Zrg96_0Ofr7Le+hto*l2NZ@ROnTa%wbj zY6K-JsaXl#9uWM&NCM`k`#YkmDaZJb#=)t)m0@l3g=ghe8n7Sv|IM4z8^NV(S_s%( zhcv;6j{HPa2a#xhDD;c|zn|u>wr|I)Q#>HZ5JykZf3A4V=nVOPDrv`*TM5GGP$&L>Z2jGj8)ug0 zi;|3BFc^%CU@(&(P|_xn%48!?FfB(8OSXCAhAgX{hT$4}ybVhQ0b|KqfXjlbwD;Hr zsJaE?l#DZ1gMC#k?7`XHgTb!x<+-1K?yTRl-blF*4sGQplgZ4Cwchpddi#&<{mc^D)`*PxOdLctj zyhD5-;5f*GuD7f8^lPjB;B#J1Y$uBU7xeqwYKcArW7bjU@e;E@9%y2 zChd?YckYq4R1SXnWwo6s^|)^b$Ms+);x8RbVWLv+(4> zZ+|Pg>i3U=@~|%e+9i9~HhOaL^Z&KB_~E~i7Z2*AGz0pZ-EcmDGY7pT=SevSP)wwt zl4}<`l`^n(f_-EkIDrA6L7cneIF<6G>s|@O%|ZX-YN{4NIzrlwI75gx8TbW)gd-4w z_+M+x;HSOLVgKz2<)Z}CUuo&@pjq0OKAK2V1@%R3@4 z7gJRSJ7<0p-thh(QQl;~Z*b4P`sCg_@4orwJNNc6gdDs0n;dSQ=64!_H>tnQ3sH7b z<_Y#*Qg{LoB(_MOPwt+cogGc5c8~2AUJp}#M*eX$7%nW{hZfgEdp#JB2ln2ucXWCb z4R*;DD5H<=J%4To{eoSAWq|bvIl;f&y7lPElfV4sJEj4Dv70J@OKSiw1?N5VivPwl z2`(KRq85H2&G3Fz%xAMB-I(`l;19_6l%^j*Fnb5Oe%hEUPu9gcacdIyE4H=+e-e2;R34dUvGW-__r`tfBV&o_k#xxbeF#7YSOa4xM=kT1IXa=+E1{R1Z$0X zZ{7$F1nnUH=C%7~04!a4eDeJd1gIB_RwE7Mb^`%c80U*>1jm&MS#U_ic`i=0A3xAM z6g^B0D74Iz!JmtXX}{n7-~*rfbz4?sSPUF~aB$6`YL)(^>LseZ7Nbl3_I*>cp^%#* z)1W`hrUh?Rn$}r$&Y5i)!D_pW#IiC7n<{UaKIrqEp*?FWnG z__$ANXo<7xF8)J&n{gY>fqwp zt711(;GcKw#qCzaA0^L=XKe#<`7A4%%tIdgku=>GNw(9vFOC>u6s}iByB|b-Losg$ z`m0t8*#**7~B2tU_LK?lVgwgUeSXIlonF&R62e8 z2hU#o_`m%2zmNdFJ#Zi?o>!hfe2{qUX7no#K#<_|_CJgO0t4WNyN-peyO-ZuyK4i; zV$27RLDR>z5>CW#?W^zA3T1-?rk#T<-ukPM7MR;j)dK+nKI|XLuty|<&{O-obA@Qp zNkHO=A#s3N(aqL-)hPh&FvWj3%~DP*9V5T83?6!YSJkBPCsYyJurf_qexsg1K7|79 z()Zh+*I%!ENH?{LI!~vmkKwiA=WwtuPOR8Vv>Dh;%Z>25vi7kwp|4aIs50LA)g~^j z$tR{QI5QH!*5cP9nO2?P%(30GSl%~^RcmhZLd8d7Rr|n#Z~y$BfOOLytSyeptov&xQ(9Q zoeZ{cg%{TCTMaN=#PNQR?fdbgN8f(> z?W2z$J^9O%Z#%yJzY$J;^nfJ|rrM1~+q3C(kEWPa{bYdKZ?(aHKG?4gTA1MF1)jw; z_MnlZLJZyqw2zf8!@(e~f%#hGyYgu$VMRZ2$tSA(9~xE zPNn+2D5h0b(Un!A7gW?^7VaMR>|3S0SPJel>bEn8srxUDGz8MtQ1I{I-RQdET_1S) z;jeLjs0K8hzwCx}y|Y{_!i1>5=)ZOt9|bLlpjXmA_~z*!rC9O!>$eYteeXI_$h+j$ zEW8~$22*1P0=^a*d;8|U-ss*qY~?s^I`1!t^W1QP(xLC;bzleNf=?;rs%v#ge}ybC zx^^3^0H^Bw%MBBD8thTfSy`6SC~yre)|eZOGWu>L(Ec)HwRp)EL*M=2rfUVhf8(%~ z{&JL*vgC=6LYeN$Z?D@ABn~NTQn6Q#${3aQr_p$C0|1YXV56cgPs%1&g>Qds%jm-F zs2L!TOReB#@H1K_sa*hBuIZa&d8eDi*L(_mDufL!W9e1B1?Jv@koYeWE6qs=iK=u9 zb`E|k1&StK=T@n&bA6l&pOZbjPFcxEMI6QZt-ZZvS{KJn6LE+$u(Quv9ly5|-eS@U zixdtFRR8H!15kgywS0`e=%p>w^bVuHO+o7 z7#|&--TBSCfA@FqzWdIHAKpDZJsP7tVD0}tmIlEtTUrjde8_g>(@($s?pt323JUb_ z@R=R^G70h&z@{Wd%g+w}^P{mQe%AB9@tZeJ2m9s4K>!3@9*m}m7JO@zdzn<$mI~xa zRP$};j`Ak~lti&q&s76tngE2=|1#|IUX|^Df5bhM0qiSAOUux%nmo-~gdl6Zs>tn6 zEcqZN|73vbqR)gtOrmnCQ=*a(BWmA1I&iiDYn^iCg$BUDQ847`w^IOazI~*>y>Mc0 z-*s{|PIGP>69J}Zk|TgTAApFy`CEqYKm7Q6#XopdN5qK)JL(4rP&XLVE8BGc^)Rf~ zf(>^7+o&MWragddgb|Qyz)g1o4vc)20DYgkyV76)8qm)z8B2CYt8U*8(LVJ5eubbF zA}G?n1!R?3@r*N=6O;I^7m`fY4Oa2Zn=Rp~^vS zet*)`%~msdPXJ-Nr-{7X-(E?O!s^Jx{bo}Y|2buP+O++8%&Dv@Ahp`x&TEU(lmyVw z*4ugBa6dc98$ZH}G|#sx=k3yuVgBtSJG$*8hr4H)z&d)$fgXxLL8Hgs88Fh@*{2!h z*`3(u&pF<$@E`j7b7TQdF#w*7A@ju??LGVCd;kSk*x||K^!WJf&gos=v3E~9i}l3^ z?_d7#dBfswU0gi;_%Gjn`{JMf`Jcc2wsXknUvSRLgOg|c9iPc~Nb`PhKIg~hc54TI z)cw!*s}pw&efI3x#WLB;cZdO_*gcwLq;e%6&IeX5xz?9!9cSgMN{j!pE$_2eReRtE zw2v16Q(Xa>6hW%6fz`Mk{bUw=1&e$tyezM>LSmxw5C}{^IV;GU!Od1yn^s;7y6Z+L zBfCrHC8ji7lmL*asc?X31qXa!Eksa|8y>X_KR?;Slnh6au84~6qu2FRG79P>(ElF% z_8MBvQv6pv;FAEka&?0yc~}Q7wb%Cvks5t+^=Reo9EW?U4G(%T zi7-5t6s!QxlWD{Cz;}^TM$K$1{Z9&7&m+ptG`g%L=Kj`sg z!`*N)nNcsGAY`{04?CC*#;13zT7UWRlRy8bKmXiBNCVF2>+IRX?_PY$ts}K*X#hKy z1vian&z8k*5yt(R_P_C_cKvU@acW1ung&Dw4liD{6F!IdAw(F>RG>l|L=oCM9 zEVP@&Hg<+keu&0bSyr{%e4hcnq8$$gY4ir720E5lEi^JW-HyT{r_M7%Wr>t`si?V(QQvs#Xz@}F>>YbR&-p&{pF2d z!*O{S=>3;LiU8rQ4=yi5^t&R*;fWu_HxF;U{r*j!wMvQ)uo<}Ujozv|a>)(+`;&Tf3v@E6` z#cU2X%cS0Jn|+=3awd3bkloBh(%-bmdWQ_UNoof-_tkPzt!ho*jL!u9&b6b?O5OU=5uQewvlxM*3A$3 zC>y{$*0=A~+bG?R=V>&E(@-ZA2{NMlm|`G~?S!=oA+o1_wQr(mHbjMEu&|86W1-1q z3E<5)&!0Vc@#jDP^(q?A#_SI;1@*u#)>>z6)MxAPIs@=qwurx_ z)A3ac1jWx+uY4eY6dq-j_M>v5D?n<8CM}FNK60sdQZb*okQFo~= zf)0KiZik2e`q_`a|NZwb9)0%BjRTxE+v1^n;H*R~5}ksqj{UIJ(`Wu4p4O8@FTzVPIyhCI??^QzV<-vZxwu)PdxQ@?KiV( z#{nbN|C_S0rY*?}{)nN`Pr|O=u82IzwY+cgyi9!$$Zno=PzoIeK<|@ipZ7S=_o)W< zg>)#dDydN>Rz27UrXc)PLbBu+EGH)iG+U@^u|Tgv@lxkZK`Ib_@x3wFizpXd-N&#`?Qv=YulKUGD88v9_U{%3+=k$1T_t^fO9!<{g zo*mykJDXhn@Wn?T-IdVh*gA`W6$6v!KRo*DPxk*Fd4R#o%lBW>TMUL&0@e_mpH43? zKmO+Pac6dY8_X8ufLk3l{LcED$>fm%xmg_kNbHp^BHj4>f{z}+yi z9*GU?o&TXUEu@?W9xfyV1+IqUFxt8RG4AQ6DdboQK z6njIwg!=#R=7CFsIJN?7L2 zG@lN8bKa^O%NlXEq28Nr0PODf4DLarihT{250k&B=W9MY*!4Qv)oac3dz|@J6$rh%M+sP6{;=nDLn`C$ zuoB#FqrtriQbCh@_jq}FdiyQA-9ENb{Mc?*&!4^bo_*&0+yZohoTzpDXSeS>zxw%q z{_O|LAcN;uq=T!-w}0ceH%@ock#qZ8^KZ1Wre6)p&K(9~R~yue+3aWl*wFrY{IAz-0^sb|D-8pu zEegdwzt!v;&F$=OOTTV`o!5D@?XgbZQmJpPq33UJ8KYs5X9av9*jeVVLZB3m z&#sn6Nr8PlU)v<9qI%Dvo{M3-c8Oj4Ed4oKIYNKii>&^q0kD^YD4vrTbZ`3zje)hn zJ?i~AQ#||M^8{FaboqK|NB}V^m^;dh;^jcZe=8X#6XM66IS1O^M{1&f)F28jQoqs7o2y2h3YECC$A ztLrbb93yv!REcCXW{5<+_XTe@;6{rAh z%k#>ofa@>}tkk=|3OM~t?fnvh&Z}2bi+J}(CHCb6m@sVM0F zmE&|a0t0~X1_%K7$D-aoV%HAz`kX$%%5+7;suAS8CX#dS-wF6;>RNvr0QW-fb1meX zDO1D70X%`;#|R+^f56I<0Epv$LF!rYpJl@6r3L%?tp&NoHh9~uazT%Q99u*V{hjK1 zfpV?7pYIZ;0MWv_WtIrrZ8Y2o7ZWNAl>$YX4yHfCqFm$806cjKN#+(D6?OF zaPYwga@ybkhTSk4xyzy5?8)xf)H6%8rogI(vt#@EQ&fKkM80K#*=#uXsP_uMdV?d& z0E45B9sIBCIG-OKaqM&c-@bDv6#jQk*V79Xa-f2d^IM0L9{fQ(l)j*|3hn*D!XBQL zqojz10$$f`?_XsC|7Ah|>oqg@b^z$@eW{v2ox;?|{jJe?Hci$$wzK2`{0`PT_z|g~ z2U^9l5-8LLu&xz6t0-8#xj%E|V=%`RR#>phk-~T0;lgggtJlpsqH|r< zt_SVxpOV@S#KV4o#*DfMTe^kR0gH4x3#S=$9b$+RyvvekF@Ml*5)NgV@vE)53k(Hg z@V4C7{{PbPf35%=MsWY?N@N&;A9yC|9t4;{|6+BrgsP}=L+32|wb0;1zhe&G-{>^8 zqM3NAR>=}X&c;2h@xx9mZEb|7TKO-^+as@iLJBz028j`+%bK4!XVANwgKmop?gops zcBv}iRYwP~kr+v;Z$mpWEtlH;F{ZQZo=*b_pZ!`O%%>$qyV!VEq!-NZau#%2%m;2R zZG&Y5!Fmp=E%rRRLQ~@u}}E z8w_a(>e_;0u-j!|1v&nEyJ-5;&p-ck^@$bbd!BmFhy835au>to8QV9Ap8xdZ&;RLv z`}0qq+?(9JYbF2rdUBfv<_by8d_H{mq-ddE5ouPQ1YUE-?*gTc_K`M@M27@9gVI5UjGqpc2^d6ZDVJ@7x%@DXz zDA#4>2mmrb)eoFE7nw)7`66uW9OU+T52(xb;QGwF=mtqhmm#~9K3GTU3Tns4mh|xyj}?23}ZeE_qT7jU5FzD{WBjZ$S#9a zkyJ(H=+kvFec$3A3V?R+T6AKFHG*B+zu2s>rAsy(c~k9NH`S(}SSbwQ#yaRk>yadA z83j*uG3dHM?Ob%Iz9%z`h26c&y^z}^b{JR_TBnLkCAsk~`;tQu5!to$RT)W|sp_+! z0nrW4j^;hYGA#k?UuHgeZhzx6t&6KeS;NexgiCB+*O57K=ZcU zMYO!Y=2tRtnvPWojXkz~fZ@V6N=2%5n!e!uX*!r&t8b^d=6=-k8tvPA!yU3b`IZeo zg~4$D^Ur_%+u#1@4_k183;+f_@VUDbOknP;PCmcdYc60J zU`91SAutnhsyIFSGuhHsjeAaDg_byV*omY~)SvsB7(?|Cp zOc}0kPTmc)n#=2EfRaEfftxqI8T;3E@z31v(pzxX-#Bnt5@dib++pxU@`hFh{3@Zz zPs@ViO0)K~kIYUxVXZ>cD%fl1@ zB`Gt=B9V+lcH+3ginuF%eP03km~wUEo&aR1jcA}NiV(!PsyydO(F6eHorc+Aa(Y;MG`zI{Vwka|Jf2`c$eBav1F077+{1jgb%OebqDF2J;)fLt}PWcsc zk9{6cHqHFfXgl|r->?TkfLQ>b9mf9lAv>6?!0%auVD*6gNBsBheepkj`uYF!KY#dW z7|ji-no6-^2fw6K|IjQpa1&v=N~`_Jda$R7R~A|RN^yh(?0<&z#xEnAZhoH)5?lp7YoLuTSmeC6~ivHg)V{tshd`|<-kFR zScU0@w*Fl4k`d7QU}*t9t=RsrG%av&`^Hm37l{1@Z&xY;B#bAreSiSA<;B5uMehWk z7T+l>QP5$wB)h)0`0PLKi_>1{kLP6BS1T_&l@S`kU(fvA%V?vf3v z7&5H@W}3Mi&ZfWNu(Wd-l3uy7zVrikUcLy<>?hYd2h}q6i#!ZD$rV#pO05UVPWB{` z^+Nbl)IiA*N{nH2B8y#qZ&kIDI2VXtt@MT1;7_;_;maC)fZ*)=a@6|lYO%9?OO0LE zd2Fd*3xcQ4osn;ki~FXU{~W0eamOev@T0X$grY!2LZO#53{$;r%|fuPuzbQ!Alhr; z&$>&31U(G9Z!fS z;2^b*VZtu{__Q-U^+39|-u^cKZC~7hinw$wv{J0jmLK3#x&mS;r3kqjL}b-~7Y77X7ngsdCf~{tKZ3 zvUNO1K|IKa68lllyMJpg?6Jm53+q5I){8Lq*Ma^@{GV5^TLNw8!&VRowm!lRzYzD? z>U744mUHv0%q1e2w=kn3D2qr{%<6?)13<=&P1@gw>w>Z#@T%z;@ zE3j_g-R&>ja{R>6g%>U4H~@gd>O8!@5eNV`FK@hk>+NgRZL9pcK{67AwAU?XY`S4Svf_W( zynW_0<&<^cMuHDQb#P|z2c24BSF*<=gp;zSq|dc!-wTT~PgMtO^P%ORFRhj@dRy%QC=dYm^wLb9vgktNl2;KSb4RTJGiJ$p-;V$NXT^!SZ8-#L5l z%P+sYr;(s~cOeE_q|ZP2(2spz00YwMzDklcW6XsT0JdVo>%M!3SV{>*USaIPEHYgQnEoG;aP9bj!Hime zpsqVm^c@E5*RV5wIJJCZX;=7C&U&HL6|%K2mEmgS$|C&}saZ=frEkt<#M zDiEF)O&`Dv3nLMNpkZ}yf2hI_{NloISUs1n4<4FZ2!38G)q6#kLI&QrPz7$2%T~*at*sNCL3`@cvY;;0DuY z&s+gXuund^7wzq13Hv!l7Y4N>}w1z0;UxCleQA`}EJ0Mgb3)vHhsNHnOb!f0R` zJ|d8x7JTd**SRGCA%E>}9`wbm)ey$e&Mm-Arp@Jz7VhT_fj~qz0@qHT@=Zkx%y+7? zoAuprdrtdFHatOCveRTq1hHSUE8L=@O!iJ!cR37_>!!+p-1rzx{+LzMRHdEDlD-_d zcPOJ1t0>sOZ%l!NJJd@+W!Fs=*>OE+4OC=Pys+_|2WO z+mHtCT0B3WSXD5Y40h0g=hk!V(O_3T+t1if+NnT0=i{44A8s&_U`M<9ZjVs3NEwA>td~MlVtK1fr9Pp z|4>3$CjeTA?m-5_OdNvV*i@5$Xyr>M1=-c)jG%rH6O?Wbg5ce21|P08UG2Wzj3y9! zes!~SyIj%7u0~$?3pyvI87}ITdVRhN1pzo$8rjtd!W+qj`PW6)ses=1cLNVdl-V0D zs&T_r0fOsbiv{)?q3f!=D(tBVE+tZOBUqDmuh{~A@DLf$60$UOB zwqXBrV0?C~MWgQ_&A(9*u-C$$PKU`sl9z=xM{hmv}kM=-C3)8J;?acUk!&oFf3-sOA>HuWCdUOWB|075|gF#E*8K?*# zGxX-sm+!sz-rv9XoQ%AG{_+RRg8ur`OS}8Tf_7s4{pr0A?L$^c`u=LkJ}T5RUAdm7o%gE^k-e%UeYzCxxld=@tibs$G1w*CFZyEBs-|2`sWG(!J-Bdx zQ|@qscTNH{LROusY}{=8z$>PM*6ch`Ctu?LMxmAh97T#61#dKF0IG_#E9pBv5x_#V z;q;bqjPL8MEG&KZZ^veycvx?N-~k2PTeXgcc8Zp!9mL!HK5yjw$HF6e{t>r`M38Gt zEcOXokm)iJR?PWnQ@Y5KMk7ymalr1e1^DDcE3@zIt+JP9N008F8K9*B1Uxh%6hc0> z*dAEutL|6i`x{CC0QLpg^#ow=*^?eTfA5_)5%u^IaUYA`=g*&i`Ft|ir>+LNPwhKA zpPY6kv%&L|M}PkFfBN6-_P4)%v!8UB{hht@cmI#S`@44;7M>FMGx7_XVFMO+L(Icj z2p&z2a2whc%fn~E5AgDElH1oXJw zlcilpqm_8;E2lNcZ_*-;Hdohqb5GSN-+UiOQuIh+CLhS3TT?%Yp`Xy=2mD{49j;aYrTT4P z_pg7#6QKdR*Vw5OwFUwiRQ?gEbV~l0r6jVTqK8i@RZ0iw!2Z#+$9L$G^4RB;=xpQ~ z7_*iNEN1E`O(XXAaWB#&aP|2#y8GVU4?mpTqslMR(W58de*5XS;r8kM`}aS+|GVG) z?y20gKK=MYn!R!A%3Wx8Y@_{0e{Ws+nFXjO`z-u>><1#R_Is9eB8BU=$BDn4Bhza& z`mh(N2cJhnB>;52s01ic29Z9$AD^Dt8LpP!TKMsRWFh)*ZwQ^i&cTE^Ez?Al2Bs%J zz5o7C@Bi@m=gnX+>5MzaZ@l^Mcy32Wq8Lks#QG4O=9rIN?O$G5nvu`=po5M#+y2YL zXBXIqw$j0^d)Q*J81DQHM1-jvW^;`M+qr+yPX09aXV158yZvAMl_Gxan?OP{o4yK7 z057eUvwv(02<5dt66%4JDEH%@xeLREq89o$S%U+}i#SCIB&9Dz4^RLRfZ_-Su=Aq< zkl!e30&dv43GCybFxr=i(8h&%9pme@YQOE)N8Q(hz>AOxP}}lV{=sVWdR8c5>;9-j z)f)8E$WEM5mksw4v28?$9Br`dzm9qMbiPS*dhMYb`*phqE$8>r17H4r68{Y;Kw1D@ zPZbOdK2Y6{M)(a2QCtq$wQZIcVCtmtsfB_3|N7O6Q}@DgL;42a{LI4nDt>M!$+$Dw zO>bn+o~){a8`<=)KiCtub28tj^}UZixw<%f`_^Zlkq|!q_~GFt11R`EEQ3Nfr_b6_@hw$f}dob)NJ1mm7J8^;Rd2H<|h_avNPW+$i ze2-IY{;2rpUJ(qo>;k-F@Umd1r2#WL&TotVh-V#OhV}}{e*y4gKnFWki1OQ@7M~A9 z0h}D)dGpOT&WelAuU_sYV4O4-K=Ce$US59VaDbQijoB^guvmuPckya%iEg*6-7?5? zz-t%5`CuTX(D7v0oAb@r(17ja-xVJAe)`wz>w=!QRN%kzu=m@vysrEm29T}YE->?@ z0INH>i0C1-_M`4vQ~J!gYfgh<-)6TC9%ED$zSPaL4bp%mQJ+*k0(9!JAV}y6c+giO z09Z7O0<1BU`-*j2T)N0QBXrj>aVp49*O%d(?XrKc7N8u-Z_qAc z>GF+)tR1=G9R!@$wei7W%i4E(eY@cEH$LzlpnK?+fbI2ut6^HnsK2(L%aP^~>rPU;6v~v<|6f_~|KSgRw2wV_`tfQiN$}iS zfL^?xHv7TR@!6TR*UtyT6fVHLp3fQV^|s;$SS7H954^Rf(;LQ;{u%nfAD%ulKYt@k z0Fdj7Tqa-`+kF_aS@e%z*9`Y2_pk}X@MAcSQ4Awg?Rclt*x48A9Uj-tcG~`<=RZCC z`KKQ)R&HZt?a=<^%O8IJ`KM1T0X(-SxwGQL62P;EhX>17?A%hv`OMqE=g4p0T+E;f zb!Nl4;J!OBf34hy<9=EQz;)%nR%KuKMgTaTXnxmSfxnfiJy3P8R(@TlAOP2HC19N$ z8e=h%$b~(<0uF#xK=KuJbWqtsjI0F6BnQZU!Wvaoq0J7$@yY}aId&r1#_IZu>ROXu zk{*oob>!0>AB05U=>3sEfFpOV(Q$=>tp)T&5ErmIR`7beVv|i$r^%=koprg?`R5q- zs;q1(TC^JW(pf%k75qFDmw z(2b2$(rzQ3AY96o$<>zhZqJ+a$gA3gc& zUwQMmnJ%WS#2TCE1#e6U}2Z+&AOb(Fk1eE+lWKKCB-XOzeceHXr%{Rwx6i-2sJ;XljN8AHB_x;U3@Y?=8{?qqQ z9Zl*{AIT>Fx-<_~P3m!7su2UcR$HEQ*^%>544*nZx?Pp)dL5>2**gm>I z#G|JKA#^~bZweR-kbF1I2g8#gUEd>m~6YD_`6Nv^MsQNZz34x4HPfuTd z{=)|+y8nRDKjVZw+1^sh^8t#%yOiyEFmB8K zU6}K?T;I0nR~N4#U$M@vX9Dew zBGPi?g%1k7P#{y-q0a-+93dH61auMJKq3fFlWgiDVarMrLbh6h@2S~z$zJ%uTztZu zj6{AdxbsC82tD;X>k? zc)Ipn=%+wQ!K~hBOcHz=Pz4!D`gb#hq5xQ|EY>Z34-`;d$f66%4q(~>JkVfnwYT&5 zx4w9%um<KYjh;8QTCd0FAd^ zt2g&v-|F|gNQV%FF+??r+OTRA{k-y<#Y&Fu?l{dp|M6IZg2CkFPyh38fBW0dSNmSHqt{8B$a(<#!6trZeOX5?;+@X<>G}C&0Om-* zzu8RiUn@prEJ$42Q;4Dt~`KF(q3T2C+)svygp_Y?E3`zi<{pn5@zfh2J#kU++E zNKmMO{mq;QAP{h+v4XwAv7Ig;bDeRw1D1UBbTf4=kp9TBNmgl2Sh;XlYfG}JgdXf_ zGG%X0B{5o^xVP=d9=Ry2S(R{ECH$)I2P%xJpage=OsG=)-z7S-?^NwtA1A{LQk!Uq5^F?GrBw$P!O}cmL6oZ+4SA zXJ^N=m)XPbo@K*(gY}K?zW(|zBgJ-52Rd|n1>DiPtJSYi!-6Ip9*kNUB*z5?`o)M% zso20GCRgGCOp`KaWK@VeCuVOVIiK9Sg4AU{IH`vgit5FCq4u^zmIy>#dk9XLO{hIi zrEUTBpXtTh!93V+KWZNE7=K~CeDU!rjhn5cJ0dPpAI?DwMi78fpD-K`_WO^%{P4ZA z_wL>cVxPPD7M&ncQ7_(E1=5?7D%1}QILs*wd>W{3U^pHGH4pwDDtjUBd3^f(rPc31 zjDrynI9#lGkqbF>B@b&$PEW^20}I|k?8t)oE*j35Pf{c7a0N$|fLNc6J4hC->HK~0 zzwiN{Ue0HW#SzLNGoXXJUx)v?-nA1z-Cilwv)9!?n+|>r2DttNM|af@`m%LSLhzZN zwfB%BDEXOM?CN!~vy4DfKz0yWQ9JKEZP90^6lYdQuN>c-K+K0ho^(*lf}cg71y7A_T<}dpK$71 z!T*^R1mD;}|Hxwh+b5qX5!mh4XOBMnMtR_~C!c-x=*73slFe0FbierK{;htxXl{J= z+3y~_Exn)^73gzoN}^=NPPxTfgiH1d;p&ys&5l`)I`I!DE~nLc`| zn*Xb>zWVXOgFk&`{~mnx_)m8G{^|YuUq5|v@zU1;uHr>nK?jU3V~Y+D;T}iZxM|*d z^W8V^+@08`?0BR}}<-0nG2xX1X<|HsB}Xo+g!r+&{17O4bUi7*CFxF5FL!g1$l z>^EocT6~7UKkkU@FL6P*zgFK9Lun`OYF$7avowC~taXgj`d;mv?1;a$I{-M`Um?E>WxxM>LKXZqagxX9fjH|smg+F z+X%g*iF&dTi;+MWqIMRmJx8+wMv2ee5MS+fZ*u-_-t~t|LE2K^H#T5FD~D{ zWeK1~CID>wK$%DOBrRXqxzh)dD(R0-E-t(S^ZH2cT}wawS11#WWDF>UPvL+YyGGI+ zU%k{(Jf$TC3$tAC|JWCBKz36yz@WEf<({Wh%Ug>(itRyUmy^IEr4{zuS`wb=rg-_} zai{?vJox&-_g_D?lmF@c7cXAizkmPf)B8^!tz->?Ku`<~aKBCSb-Le0Qv8-skOV@}&>c<&Eq@^15x-W1-cje#!k$c1&*1l?~y_CCFRU2T5C5uG&utIsXWCK`P z>tBw<0_1xLH{Ttpou$vU`#O?XGD9cb%Ns3Rj@=pYe)D5L(l0~w-UtVHs~C0=U_e2F z_Zt5bkobpA%H1GV&~?kggG&TCRaSia-NVo8i}h}FlAbIV@&Ebz0>KKmOr|F;YP{${WE^drj*pMCazI`2lq-9@qb^wYOj(J%k<*=M)@ za{ptW{|8Ty4}2}4E#N`WPHWG9SO6oG$KVnOqqyv^5JpFd1zDS|>vu|}qjOE`Rf$tFg7VEm5o^}BB z4;JqIsYRpxTmbX-as`csK4i4;kDgoczaXgT7lh(hQ83gguL>Xx|96i1oew1KW#K;< z`e2~0Ylr;S>vjEa!@p=DeXqhWZ~bd65GnAg(pTIE0q<)?KZpLRiT zQQOBPIEpMtO(?I!Xc1d+`LnVQtm|MuD7q$y0Ba&tAXuZAm4$1(PF)&MmO>l%990@v zdfZ!Y>b|sH=~6FS3sO+P7f$HYTv+wusOCPqyKMzwG-b${F{$&Sru~YW2^0=!DP1IjV`8>4=3w~pK*Kp(w{MsJF6;UhY;ZuA_SjeKMVx~>O;$8z8m&58k4N3rHvgdKg%^rF#H=VkPM$I z(&-I&dBoR_{k=KrK;w>+dFHK~E+^>=Vda*LkJpd)69D+IO{|&^~9`JxmKVfnffZ=Bes@>4J$K zAi$8ugLdEp6HP=n7TW+g5X(TgpRNiz5=}s8{gG1ym$Im{liP`cOF@O*eYnyx?z)?A>R?MvBx`Es-Re(Lrs;eZ^)kKVP_oCY>3Wh~s zVCx=8{NLAtil=UP!*_qq1@NjMod2(9fiAtpgE4&wy5m85Nb7*Z%UicT`}pI_Zu4@# z?%sU+Mt8dI_g^h9l3z+IRkzL6$;taazkl-n;p+YO-@p0I+n;@NaPh;5HS-@_-u&!V zHASo9bn@P0{onulUvAy%?x!!G-TL?&%M7>Pzj5Orlp&YCrC>RVnhFd+{YU^Cuf-4K zIHq-|BLP+yD^*JJ^tIn~t4$zb#}ZZ`9OWiUMCi?|e5cnFpx0V?JM(MD;joGix*LO@ z&;fYvZBwFd3qLQmmoNm`ELQfllnr(gr^Wu|zuYGUSPlqvzvY3aPe1*Z`2Uu<$7QnD zTwxeyY-wG`dqRJo?jvmpdV4f|j&$CN2RPIi@A;P>efZIrXLnf;FcJ_Zm<;gIN0C)2 z5v96@7BN7AXn?qv)%$ll?i->sFz+#fWbtS}1DT>nKBMb#q7gqJRKIydl!pv3^hLk| zF|e7Gf$akeC4){7^W7~x2?Sl>BecJEBL5MJn!;U=0P0`C{(SF$t>fWs|En;7%U(@c z&^h1V1-d*vI6H`ezZL!h0c!@J&S6*QHRK;%*=E&d^^Z6nCW?d0K#nCRY;oCWS}B;f^G= zMCn&*6)ZK_p#+Y61iv0Li~gp&;d4`bZlyuI!DeUFmC{X@2|~cf9=a&twTS0|OL-jz zt8tF^6D%9s?%&xzigQ&0Au-5Ju*LZ(&QGLvM&!XfxX^NqaAN@{w87=gTemK6+`7E@ z;AC}ic=Oh0Z{K`2vTn$Z$*a}Fk6#wa+|D{_`p|9biC#*l-g)zE65SI45QX4j+}nTh<%ISBTs|Uk zPq98E>ecKE)-#!mqcGcGBlhZKv`z))d+y!6RqKKlKDVI8Y^aAsFE$`l{?USZUs-?- zfcLzfAOQ>)PU&4R`kUbfaKzYevGeVqzgqL(x{lK@-pksF|GMqey zJ&Lvi5Z}lj677$5ye^ztP+0tfbwng-t?}MorXw-CpQ6|^E7)(D3uZ-)T$nvp9OuFf z@a0MXtOThR{dy^wJfI1R?uZ5St1$SNDG5x#Db7Gdyb!W0#&;H?)}C5a%cWdPzYQq_ z1yffV2jjm~scLltOtQlJe!Calf0xhDWS{`FaaQYUG_^;vTDX#$_(`|xT;DMK?>aHY z`2Zf%>IvF<-)nxQ3)Ded4Ute;!>{u(fPcWZYfFrP1YWv0p}!Fj7iA8qcb**H`s};! zEc!qG{NS_qtM11)Mh9Zhrfud+>JCeDL-z``{}6@6VoR zH=n&)-Fo}*@aFrHrGkd&PcJ-e#kwUQHvwlgd7&AEn}w?AX@!-$=o1}CId!^n^o6BN zs2i3xW%X>;0K`As`|#jyu9Z8m>axl{nCddj%bO;So48jG$J;r~S_?=?h}xmJbt67I zdk5no!OqY=l^kfY-IE|JJ)ro%Paav=KhP-fd%|DI;OnQ~f6rSK1oxl4v>@NEnfa&1 zHfG)qA80TiDiK7Q3iecs-_tCRE#SQ~h`RslyKhE z`u!ou1v~sf^jOb7827Bwr`Xx2d2;do``Io+-w#mX!u14(+90r4fb&D_helv9_o$zf z0B{N>E2tH)1Yof?XCE+_$qQ_v@NW*1xQ^4a~}4g!Fer zK=GOyM0BDUNRgh_q7ec5u_K+=7W`UL?kEZ9{k^qEJGDkGaVHeO+L{4tc|;7TuNgvi zH~Z78Pbd}T#dPZGF&WGt810#k5h0K*^hRFVJ2H@|o|XUEhUxRjy$Kl;Iu5n95E@BU z6bnJOCt6m4O*|5#7GvGz$QS?BvY^UGIHS_WuS5ixO~HomtyZHyYO#8*;j7A6njq}T zJW38ja32QGUA(kZFVG;|phobHAwcgW(ECAX4t(tHavM%*KA|S*+S5A``atI`dVIHT z-SCt7vv0q<@&1XW7CX}4{@~)^*3EwZvsH{Tp~KltuOw}0#XZvV#3%dQ9e!3{zG z9lJ=`VtMAUbyLvk4^AQYf**QWLOXKMUmXO(g+$k(VN936{rKH(GVY&3pzvWS> zbTkc|{U(*!LnAM31UjY9gI>2gcG~e_SBw<{lmHf=k&AHmqN^`H*{ez>uu${fGHn~C zFn=wbSOT#7S_n4zK1q}5$6r5Y6!7);4<7u<;{H#h0w#l&2wuE+`ryHnq$cF!xwWa0 z~4`O_!&_6UM$dO!kT z=0BJW#*@jBzMj%x7_n_=X97M}uzL#qi`?(5g%2^~8@gDx^#DU82ZovjSVds*ZH>t| z;DW}(4!eK41NZ-)IU17{eJ2yfFMIX+uvg0T8Z~QbFHFEjl-ZH5`rN8SZWJzCJgtgcMZyVP<8q_Y zpE6*yoUZKk#d4+k-jat5O+XE8Ujj9q_~$Z@C%Cwb0m!+Z%DLp7?-`@34YrGveRMg?_E-Sy>@}*axqB z;YmtFox_uU7}`m5#*xwN?nMp8g!{rFl1h2bj%xLdedK=-gcMzjSena-lRDN9s)BwY zvi;OZb>zU`QqG-A`&eF~&XYIH{*izO1q`4+cl8oG_Ho;T3#=gkQ^B0Ex*dsj@Y}_| zwFaQ>?XRvt2gJj%PGZF!`$BH+Hl-EzR69{lM~Uw!@b&rAj$ zs}!(0;OV0)&Z2GPBgWp)JN<2}m7rywUF6gFLDXDbO^N@HKDzrJEWdZ(JROVLv!8zR z^G{b_{O~`1`ss%+t{R^QXfQa|+JMG?xQpAH)HrDcdObH1WPv!`#;!kRVLg;WHz~rA z{B#wAg+}`F1JEo$<{tK1p{P?#oI{)B82T$eJSJq!O=p0s+^S7Fhq;-L0a_?c}c&`qTlL zK59=Z1so_vU?qT5yFSYSIRBO2_m_DvAOrv)yRfQC%0Tj_DtAes4Hwze6+__aU=VU4 zn((p=>_N9K(E5o3zKfZ70)zWTGSzU}kKa#>uwBWqp zVMl&X=QZ*FNRs4uERCp7Fdz#Oh!@?iJvx{xFcJ2!uBiYnN6Az6E@(J3nWXu1K)<;( z{NpAVd05p~W87y2cPumRCa-g7xOKq?dur0Pc(>1Kb4Q~;81z8aLe1+9ojnI}Aa#L| zR6s&0e(ZM-U)%=I-RfInGNz>e_#qsvvY=TemMcclFT&VZ~EEC`E~ zQ9~R>NbwTN_ddFN_sfsIv>fo8J7j=;y8P+qzy5&q$A`~9`eg4ap@A6;^3MWVVIS8q z;08K;MvASZ=+HGsBljgy_83^4?^NYjBB0(M4i;_%c!WtvOYa5X%O~c;@uFiHLBT%* z14!`C32lww>pG16{j67B_*IDi3iW3RfEu67$dy)D%%s1z@4PGMI_UFv zK}gX4XuGwbJR%dhM~Pi{>!$24@LXRg27v6RB(LeCy9tD`azqih6tsT`?W~0=&^_|EWU}o4&6cyE4}62D>iVZwC5odw_cREGR&b^mRGFLpRrH z$#h*;Cve$Pbj&YYg7cs+MjfJkOPB~eSBalMuipfSLVsih((2}4zWeTNNv9-s0%vSK zE>;NgExlowYTSWa_f(GSs}q#Fx~v+O?&aZgP4vBC8Fz!KGU@C>MTz8tWTHw^97*U~ z-B41(lk~BSJ#u12R8lr2Y`npn0LnmbbBS_VI6xaNnoc4q=7J0Y zm<&FBdjH8uB0$<61R!F5vJaGZsLOkjN$kcfm4V&(UNo7UAD?%q4ff6Si!Xlo=ljsK8+1})nXxX@I4Z1Mu8+S>bb;X``85PJ{?tT*I# zxa*A*S0g|Vj7s?=bkqmkAXuPvfm?0_F44h-0JJB8z2u_fPG`PAtJn@~&S;FimjpM5 z`$h%;l`Ko>u@gK1>09IoR$2ns1PmUH(FWDDoa4= zgt85Y7NYEEn&sK%Blqf1WADrRFf*{2-+Q3I55`=4;J}_%JwQxAI)T~1(iuXekPH_N z7#IfP6JrE{c@~3(@AwvaAuk^d0SBG)nN_fuie$i%I|zz_%V^)Amv{O23TvATvH=I4AX0XnUd zP&y@*SW?Y1X#v(a0?Q}76fy|JuLK6F!1$LrXjv(`HS>ElUAF3fUIn;jC-~lAQCzt) z2&2DW*_M{X{M1+eh)SUnm4pI=QUfsoN721|ckeP$ z+iyRzS0s)&Z8$)M)5rRv7Y}!@{v~t)cJM!VZ2v6$A7~wD3E+GC|Id$BPR`|Fho}Kl z^lEH3scB5jk+dNMQ2(`9yUATR;{63(p9Y07F;>orT(h-R?EW zFVq1$AM^j43Lu~X{q{@EULf`@p|28#f)wm10Cu%xMK$$%eGBP@quqCV6bS3Q(0396 zK>7g+_jq5_WRW}sUNZw}paXN0Whf$s*pt%gR|9R?(-bD*&t zXxbYezrB&1aiA#rNZpG3T&!sM>SC_3tb%81CDJ03Hfkh0W-DZOr~05!6408^0p6^p zS1WsuU1n~-YL@f}9-*~O(IlqBlpJxv$NcHRp6EiTphNNEr1eI0>*uNpPT&A0^Gv=3 z_A05o*(eiCe{%Z;=Vex)`3defjhj>d%3qlIahr_+^Vvhr zo`de;3}?H*hHn5^SVe#^K{M=l^|dokB4YxaCEw&ZIUw5Yq*yi#93Qwqb7>X~W(!yN z8VoxjF0VU%8GDQUBPdRN-3h3ruXVmFfY)BUD=!50Fi_?eI}~JvysEbfppAQ{9`CgA zqtw{}<;zk&UbyU+D}$vh2y5DXYwqhgi^4gDKAym?8Cna5oD2UiexU?F-Bna7KrRBp znqRYkjFUN_{#+1LXrIcb_ms$B{UnaNw2RjP!7TDE9_(MqvY+(|rsXGxlNO64N!owC19Q@bHxQx$6Qr+vf4#a-* zgrH7=6SmUhTsWJCO}{|O{ctC=3#esQt153Syb?duK|tz=pSthPW>r-=Ymx-}P3%8% zoD?c4=~F#eflfq9n4f+|rH@tcDkVTmiz@~(>V?99bZr-3jx02{kAN|0ksZ4|_ZOzyIFfxc}tY%YD;}>sZoXsPM&_Ld3niXGi#O$Ny1^;~0@&Etxd0 z%HCu=01hzez~Du?`}A}Myl=nVBW)P>2>#~)3uyZ1LK6b=9}dTDvoLmtFTI%H0lF9u z7W4%ZtMAzW@^t9+=j<00|3ha5feL^R*wGQMf#3DWp9oe%UkB)K)}{c*4SR*)6Te*x zCFbMn_ zng$>Y4n@l#iOE5EA^|c-1>n%eJb(t^LcxC`dtYg5bX~BJsoPSM1Gq|7s`H-^c2wRY z$0IHiAa#v;{7RB#3Ss<`O4=JFl1k^>C!*Kjl2K8yxFe^Ea;;L`PRRb#@PII73j*M) zAC`F=I)7#_|8iIfQ)xM2ISO=&l^X+$^mk$Z&?JteLPZ(-Z+rLx1@LL2scYH96eLZa zO@aR5(m?_!QXsK*_Sfmwey)Ab>g^ghUJ+R~2<5mPms|U9xMl9YjfUg#J)O4ENAG?3 z;orac=9_09-iw}VOFtNFdt-oj+r7o_KmtA5u5GtbGrhR+ zY&N%#bkwT<8qX7||D6E(Eepu^f~{U4Z&{V&G$Qc?d~OhA1$d#QJ~8fv#2jA%9+HUx zM|!P;9TJuBy|D@k`(T`+=WU^Gj~E=5tx^Dok!FHZv5WH(2ZFN5 z8l6lDQ6dX_mDs3}ZRA2&R2w{ZihLdPL6#afh!R{0mZJf?zeKI4-|hl65|C&n)`4{V zC*4+`^rHKkQ*RA^f0X2vg!HDXWiZmRT7XP}4Z&!Xk)5UzK`56RX=;bTbImz)bGZN# z`b-57PytSpVyWj@0aJyr5+T7=5OBSi*o^!i2uN|Lm^qLVU`LJ)FRZU0DdaDft-6xB zuaWmsYtFc^ZqVY6ljoBPjCe zLhP>~A70__)kI#$k5TVZC|f#^Nno5xa$vrV&QB+^1!CSu7S;>T17v`4_(JvSbAjFW&Nv`mJ_;z>-%_x zU)Q^Td&4Dk9?Ahv25kAGc@0--t&V>u=Fd77)ju%(RN^xTtThhg?OK3=T$_m4*#W}( z=I#q112TWjEETMuo#@_q`;KqwPJ_wPHvhGmU>6GL#l1;e9h;{6h zpX3%&ts=g(^8J9^w9|l*wuBe`pdKc#Qh!fSA`3Aq@Jq>NSprbqV&7IH1UDwygcRn3 zyp@ztkk?K&C7imcjJ5FDPD`qUTt3Zm%w_VQumPdwx2VsM<47z6tgY`#G&0?2)G3Gq zEP|EKJW7pQ64|8s7((#gknH%kp+th?pJ*k~?<3@x$oH${_eg@V5+Jv_15_7b-=NK% z(+uRfw7|C*ch~&e%5i7Atq}keDBt>P>&VB$Xt4#3Yh}M``oTSq7^}94|2OYU?oNil z?I`yp%n^y#D?m<=(s!kw{l~<*0^Ul2ufO*`;6HKv|L%861CLf12G7Y#5i2|(YhVMK zeSs%s{~|z2F)6)j_I`Sg75`)Tc1(EA_VnSt;yOGyJ?;2r(C&aJSkM8%(}(Bl3_<^K zCf)`Fc)$JJ!!aOX$%z2~upgdHAVdIC%1pxodkcM$xlZ@mH+5Oo76TsXn)vNrV}f+uG^(yC=P}i>;0Rw5PQ)uU(`Mp)E2S5iyiIj)E*6y-!y0eqF*!|dKJQApvD$t z?Xsit99i#zgSZp_r2AXSE}~kx5U(PDyue}00bf=gywxi<_3jh4@|B>{0l8okQou+q zl*{($FE;@!ybS$-REu3=%vDA5I!iDg5I)rc9JxUnS#PnW1(GC7D@b@KOtqi{ZUh22 zS#1tZ`e9&yGE&8UqA<5Uq#Ks%2M0lF+@&=~L45eYKQHgRQJ)b9_?^BabuHFbzVaG$B*&<6uiR*mz~u{^HBm@+c&jkq|xg>&H zcBn?8$Qt=z0YTJ?ZQ)cpNSbU|0$?JsR5eDDOza+LeQh94hAOL4+2QhExVfRRLaSv} zO0l=|9%KMbmz4lR%K03xg?1bZ2+~*DiRS9&?7 za+zR&SKV@!JcfO7DCf$qCE8C9k9>?DOcFTfq-1F)A zSZX}9&CxJea&gexSt`q{$RehPCcs{*zWc@4SB7J21IE!leR*;D;^}n{Ksj*#>!*P% zsCmHshcB(2NZ0!>zF4I+t6T+3VXHS+Q;vR10*e1AW+E{T6kpXPjQV1B8Ar!GPaJ*oTkdIXB3rjGzvweJgcIsw8A>`WxpsE2!-y>0h?B_n}cy#}= zYXv{n8uP0ja4`VIb1Hs7{fl$~=ng`?AN!_2#-Vf97O|jqks;~<_Y~61FLWA$7Sw!h zs0!aQgAVw#_V}Lz1aMc-I(QBP_O}=>YD^oi^*>PjBHUbZY#C<%uI7h5kQ#l_f(Q>m z_#6`K6d8@E;tU2x9N1!d)49EHC^s z$Y^-$u$1?37k5sj3)F}02az2J7({e+I8@~W+AwMtQ6VQwVX%$TzQx>B=W40>zx}Xz zfs3a66`e7Q;X{47kz(4jEMYzvS`I@hFOMU@WYC+b4xmYp48Ux$7|&oCPh<~jarcX# z`ZNH)ZUjgJ;b31oKgjY#tw}!TdguVIhXHN;+rM7$4V>?oMdB>#W;p)U{clistebI>^3TQdW+DwNP9IKAyb0WUt*_3PG<7-~yR_QR!S)d?au^nq(Bz zCu#+>0`wmH0vw;r{_J#Lv2G*$s0VcFBS)~jCfUQU%crYllU!-mM(V<8D@irqt0a>t za?Vs5Dvi{SWJ*o-I+!L@V579Rv&>V;!)qxqKZ$x#(Juuf9trm!7?OR*6x%bi zw4sry+#5&11%&i#dAkHcVIkhw)8o11Ovq*Di8e1vpfm)9_Q&~F7;*6f_sDbO4U{s+ zAC8&w;jq)Acjo+`_r|WaJw`8N946};#S65zX8~`~aXwIi4U*sPxQ~*LU)=Yg|D6K? zzJ4020EY=$3V680R@{N4?m-E7VJm& z@lJ3;r3aP^2mt4H_LC8JaQ;vdE#(tr6+=<%YKE0{i4Bn4h0*#dIH-zgmM5DfNIi?{ zX*`!F~B&W#D*L{tqp6zlG%cugG7Qit4(V}=(#}7L{6AJ=(mpHV(q(H)@mte zfP0vDb8i*?MWxJu?S{xdND?iBRWa*J`>95fDpiS1@+qaVQHs1^vFjamsh*!7CC4eV zX#72fZ(6@ksqm8~Z#FVq(qsf2aN?T}9|Z7s=-USs!ka#*W0r_`dXA?42M7%bHFnByYzq(pAzZb&1DH&Wy6Ec$$n-Iqy*dM*|e5xJrVwXtrrIWE3E$UO|5t z1@^Xk)IoaVtqeE3;Esk1FAO;LC7SDcyG#e{#l$Ot7(21?m~)cyn}z*YHlWjoPo92# z|3#Ys{KeNk6+l1e+b6fWQ@cpD0VSC&SKj@PRwtGLx))Qx2vQ9JrWpAWG8l{YBr&+{ zSV9)l5R)02L9MS621S}R~UjRS^lyET3 z&G_`4ckZ0sd8c!Y>$~>+(Cklx0EYm3TkmawpZUG8>+1#FUkLuSDg&MQR+7{~ZS*xF zfUjFa?4b>al{-?g3qb^=yLFWXC?MRQ6tq9f6lnyb0H}}dcUl3Gq>vgZ-5AIv$))^* zXoR=q!v3K3i}+|V(w^>m334O9I}9&F?B;nB{iu zO6qD`I9hH3#|D>QjI?+V&h(3F1r|xW%$3v!@zT($Q#&;=l&r|-ry(UxtsJ%YsrK3bSdp(PqXS z5~_r$Aip1RE$wTY#;PdP{x|+7AU9b9k{4i;rTSZBC48U6E(i&LO8W~gh+Rnd{@Ue( zc^*<=b=^asFWS!Cvd7ZFu(u!L;75-gf+=cULfX2T;h8nrkq_l52w>mcb9CN#a7w?9 zdygfCID7a+DxpuWXMlctasP!i2v1I^@U1tgU|>zxR!lTgITA0u2v{{+!23N{6^ypl zF<8<;MFedxt^bFU)6=8*a16v3(l72n&I6=kS$HlRNN>z>|E*deRSljXNAQ0k|1Wm? z-Z*f9XCeWU0u~EvYpeix6J~P9;`SG!UrfMtI}5N^zi1-xOS}8q?RM?lKZuCAm~byl z3$+=X&u6?C4%@uo2>@dME=7ObNk9S_sDP-~sCT*K5L5qHdPi&N|B~a=eC_crLy*js zKVkif_A9U&<)_<;W=<$6r)uOS0hUM>a2>OXsW`sWF;oFGSPCUXOUk%ULw@lG+)S+U zp%+h36!eCvgX_~-I;h!7Wi{e-mK%hg2_*rL`l>$)YlK7(I9++eE5X~oTsfEhjg*1q z6x{dGfY`gfHnYRgMRUj%7S&x5h!3hh;Cdembo&8Nz!s1LbE6n5@iJDmKXUYwboCUY znwpjycfIFFEE*CWi$D84X7)QzRCLo1*e?rKn_2+$I5zfc$Ql2anRgBRbfvWWR9BGo z5JA;kU$97C@@w^HuyLKgBtt8W8Oc=0K#H}g5Qr`WSfR#Y&j~OPV+TaOU;}ymWxw;i z@W6TAYZ^RvdsYQ->i1$R-M4A8bteD-y=f|k(4IH^)X%{DWE2#6L=!L_h0j9CAZm3t zkrNoIQ|}Kf2$7fW8(Y5b;pEJ$dxL#s8k+fL!&+7nO8>)5xuO?7=Ho&=V@DrZtv`Ilxv zLM)E$?6X2(O;Bj;-`Lm4wJfuC*)GWCk+viP?D}ONX2k-0 zdghBxi?LF8c_Ad9um80gaG_vL2LlXPlH^JW2ciuj@h3wwnuLwe%utCpK9yfeV#9j? zT=g`o6G#fAO(HFPSTC)R`^iT2n1D10g)`xXyQ;U6UHkHT*Mnfj$o;Q0Z7S)tYUXdq@6&9KmeRMN|8gh zBxk8a$Z`u31i{b(%##%9Olu^!*lr9L4FAOMP2*^r5=O%zJb9PbaFsogZDpW4u>PO_ z83RW8$gtJj2U0&6Yz5@^LLrJAsS`QVmITs#@W{%5r(y>_byVQ}Cr=)Ib8@xce=^iH@>=HC2lWFg0LgsrfHb2hpv9wm2oCg|l>ug!4P5*y@PH%> z;)Y_P{W1%b+TTp*f!R>Y5A`}n;tqHju$avTlQW{<;$I0M4FBt2e$Dv_t3783t{oD{ zJg*LAzhr{qx*ZC(6@hp~e9R4t52U7+ z^I1J}U%H^1pbOUTP6d%U!4p>&q)Gy#ixYWfAuArv`U9B5-N38oL1+zDVba)LPM11n zMMJ2hYQTn_tk4jDI_#b2w?QQ%lJn5_D%E6O3<6nxAtFXd{gazlf` z4TTnCUHi=!>QN-~0|_v;ni)%wrYQD#-XI8&TK~VrA`=6E;kK6x0kHK+Kok$zqkE-H z`7aW!b9aY=3aYnOOw#Rl!g3wbZiEKf-MyJ_|G?maF9BcOZkY;YZqs+ zUpoWE8vvIFSS=kpfK@01eL+!GC=>wv%QSO>ac(D%U3_j2jV}=hRLcyhwG?$K+G>Ga zJy<}F*+K(@#Ij7Om06<2;z*uEVjvRICnx=C*}a$ws>60bZbR(w$qj_a((!(k0>0}~ zLfvI|ngFh4;buiYU|_VrUE|2|0gtxqLu$A{t8BBm3iw^gU#lzOP}O{IC!MW4P1Z~{ z>}RW{c|^8rr>`K042hs{a@@QKMGwc z1uOwzmn~S}upkWj+8xLla)3-SPRRPpN+^23wfNLpy&TA+5)JgzS317?|1Kf`FEJURx zU>|pT0*`5X-eFD<*hAI=z&#YQf9DAxWQ;Xzet*Y~b>AWE;vU6=K)>})Bp?8I7Kt^L zEZ%^45s4bcu--XA0_RykN;9(3fI!Z`-uF6KZ{r+Zg?Z@?!Ab^&&Ug9|KyGpvEE%9J zK@KMAye{%uJ&1Mwi8rPx<^!aGQY|WRQ#K9XhC(mkc59F{ev(=o5fkhrekwoYq z82lAX)c|Uy+yx&4xF!N;csWF zMa_1A91#8ivJMkUa5BD!t*(8hGa36lEuLF*5vTi}5CM(~78#c@;{N5s?>?1X(8ret zmv0|fQu^qlPri8h!%si`a77`rnl{tx+m-&e+xGOlb2R9+vj#l9SjgSs))`-O{yWFV zr?Y|Vf`-H0TpIv44Hmsunr71Vx;7FW3I+eWRCB= z`R>1a_susuzc>g`gTVFfS0ukhp!4poH+@J4#Btd3`7z%v`-J`9&9CbKd9no0vH2WM zkeo-P10HLd82b*PwL{C{aj?cOx!{!lcy*bS>e_q?i6U1!PLqCX_CqC zPflTVo`VL0@q)&nU9pvub=|+KjZ}@Nt0_JOiI7`%+GNSPLH-x2oCWukoutjfI~uRJ1RTFR288Dbp%v*{BhGA(p>U@OaWguN&$P4Lb>ln zTKPxw9!jBGa6UbqlSl26d<4Tth<^+ERoDU22#^4x zPUqYr^q#5=kibstlg*cfddL9t`Q+?;CeJVl4l*RrLU0~Q&p_;^;2AvhwE)Th_VXNd zCPyt3aG>?y(QFZ%ffWCX*@O8#wt3K`uA#u;WhZgfNAb{7uvs^v&Orp*p*o5Tn^4G5RG*rXZ zGg(`;4f%)9=j()ZJWc^%)H3y~BIp&u6EGu0CxB%(69UQri35k~pQ(RR245fGy-8m9 zx~qVHZXKuH%Mu8_@syS|HNrG==TS7ktS6z0F1SMl6h(yKX-+vo?rD5UhM2$z3DFTV zZwZ8opfBTb&_d2u(l8)Y0xk=1FcOBo3!c`VVE^K<8x(h?#EE@DpSAl1l_e_$4g30n zEVMFYQwaMOwZIdm0{c*G+sQ7WjVVXJwab^x?7tZF z@!xZ)_;`5`p*OH|%Y&YDK0E<>!VvzGn*j%j%Ge&lVw4Hiv9IYpYc93OpF0-Mc6lLB5p}?7z*`K&T2*BY_ont+= znObEr8rhXe!z=@GIRj30&=LC8XgZQ%r-uDdVuF638{;vwmp$odDZwDBcvT1&+6uDw zJCU$9Q}&T~e(Ll$I1O%aA5c|8NU`c?Mxdx1t>FAS)Td;>ULpi$1&r0wwaOUKHPC8^ zb!%p<2ug6xvYx7DR?wtH*2t0*FKw)N0R?a}Hws40{yDt8y&?~kIQcC_@Mr!vjn^(R zC_bbw1uNGQ5cI;gipm^GV+9JHUwNh#fgRzNZz~3WxPkVB3`GL0w&Sr*)BVs6|FlUX z*Z}EP)>=Z#0tnc<9st|FG5m*G+%{|EM>MRN?6ZSF# z+@m8PVM1nr7N(`-0^#sL60m*}T?s$qH$I4($}wpkwAn znE?{rib^p9fZ%q7{=s}VZsXbga25X9jbFG&L9PFVN92KhQSP$9jeL|-gi zOmKG%@(obm`Oqu)Ifa2w9p=~RW4&uLgSrA92cgd}KX5#d7KNaI^;S>o0P%nXy`ccG zNINkmm|G_1t-vA#eFV(kc3QBFUSa_pmCL0C!;qd0I7H-D*OTFaNg*i1hYhi zG{*-B1LzFj@AcgJ9vlcM0aI{E@*kEW9q43hmTSZ)aHRuCn^^)#)NN?^1(aAw9DDRp zg6YkwFP1!lU7OK`aDiy(E`4k#aNlw&Sx;Bb>N_&PN__i3<;q2(4lP%fO+W%O!?cG5 zb-zlo4H707DyrRpge;f{SHBF)jj27y&8A5asgUfxa#p#qKB_EJ`1gPsbB38kIxTeO z+wq?Y3SdkdPZGb{6(kQy;H34xZ9~Xf%SSB0BY`gfjvPfFEcZ3_q(3~d4 zeiz01wQ~4}37~#`&Ay1v&W_)D%bEZk*Px4zyZ|5roDQHeTCAwaSYZ429Bo3!131#R z)ddQwKRxDv!^N0kzyu!9c<8dj)HO5L3xpGFFBW{GBS!)Il0zCoN&p`90Dj~I-nReq zwgQ+h+B03TtBSxsGnBC3d=VM~P{4sAaJ?yrugeH~i-Zxiia`<)4#J}~G^k!&hzKCK zzZichYN2spDevsKw?M}bd7bW1{i=};6CTgE0Lb$Sy;~%@;N=TuR|-`>1HHf))Kb6! zZKRVrW0PQ)vOOaX{%p?KNa!;=%<&NWYUeqkhR&Er&9Zs-9_tu}z97W#(@=Ojvt7 z_N2a3I!KUXezUQ61tTn3#Z^=gV1deYo@ot`75=-eEO95m_3c6I7r%i<;YutU>5)(i zsDrZJL1qge0H+#XFi_kuv1Q*^TJ2SDV{0{vMme|QF_8sF1F~^Dk62;=m;>S=%V1z@ z6~2XxGDF1lFB=N8UmXfK7mX z{Zx1t%>}r~mX!*Ja_JL#b;>0&xgd;cHhaZXz9hO?VJXQ|I*q(Q8;eCZ41hSr1_E9J zj?f@uw{{F+B$e4=RZIn{=Om;X78A)TfJDFoao(8QAy4E2Se*Nndiu@M-mpU7dijlN zdkqJd7jn?)Tb@{o0)UQ4Yb?O{V2Kt$UqR2~Eu}q(cAu2I`ZNlCm+!X(pb9?SCtR(C zJ*8qTVlyzCEu%mJt}1hFJ`xCzX(Ms43>*+O4o!i2|0HkRr32$0`&mKhR8?+qo53=| zy93cti_SC;yL`lx5CF-xIXPe|N2a9)m4Yr-P07xtP(!dO^!G`oIaz(XrNiRaYx%X3 z|1U@Yyf+{QH`)eT0>BdpIY1=&tV!QD7=qCQaLDfmJ?GPCYc^y62>xf_b94s?;Kmo2 z0G)Gd)sM9f3{9XL==8uTII8-0ROSF`fU37=T}$a{|gLJ zMQ^>;IX-*q_Ssnn%h2)Zk$+?8Mrf0{->3Km0dydvE{(L0qWg%-^8&`OEbxFT{Un?se$<%0CqL*V$k`2vdTE z>8fWaTH<8@m%>_R;tnSk9K9+v!-*O2>mnbd~!buV69)N#lc?uTOMbGDnWIi95z z>iQu>A01$^whI(nUS|p9%m=h9EPaw#xiH64o}mH;Fxf5xi=T7b5&+ej^rzw?t=D$2 zBn7ezsbuKWx?$>I#Xyd>Us0lll{grf#6brTYZb|!+_K6B?n~wLd*WXvb48Ni<$>E9p%WivT@YyE~~Cf2B80gg1+?hzy{cx2US;d1Ejbt0pMdGfRSK+ z;^>Yf%G0l0qEmc8yB@P#c3bXjPD^1w7)Pib(_27G9Nu}1<4z;IA244kZPMRXN?;`} z>{LhE3IYuVMIb~`K}qA}ORM@>HdGt73x#+QN(%h?$~tF^!H+wY&?=LNpt;IEwQ`=K zWPn-V*e`XvYf}1tX%&$H`0iVkC}?L|ot7fJRRaecivK^Jb7Ady6GETHe}cvU7uLx= zl++(u8VmvlQD6tL515QC?mOdCC^Zx90lbFR1GH^GhxOgS?zvT;D(plI7IalM1Dt#( zcR}~v`|!O%7G#v1?m3ZtC@$}N1LYTrdaX0VHA>Nq44;MG6%fLee$$on{*$Cr8wKjG)z z0x0U|Gj|qyUEv3yzeo7c$AKuL>fVv3gPp?%!k!=ih14w20ubUw5F)aY9&Oo(79Fgh06$m zDkLgO0Se$5_tIjI$5U_4X&Q#wwka@DaQi9eD$j-B}y7X1^Q`^%~PhfmNj*tmQkc79eabT=;lv+OG| zlrr1ch@LwY7|?E|2son^y#Hh+7*LZ*sf7AIVM|#t!P8=IZ>az&a4@%=_NC;^fdl4D zGVMI436%1hIcHF90AcL^Bn{+HP@17o?M=+Bt*&qbO zDbV2NA<(!C!6p-=1@>uUNwxX#on!mDx88j7joY`+XGPKD(OOMEd&pB>j>AGP9PK_a zoN!blTiWZR(t0MG@QjXOu70YEB5Z|ft#0fMaW z+;{$N669TfxTzL+eVd0pV2cBoue}z~`LDU)z*~fP7Y#A;^f*_$u)ZEJ?4%dQ@KOHZ zYl~7LfzS|&TlGH%gjVGjkO^tOi?ufd0Cwu4Mi!v-_Ma3&EUtkGTgbhVbeG0avMvmY zHr%;c+u2nM7|8Mlh&!+$VM>R2&;HOl`sWg7M0axc?8$u>ehStQc6+hpZ!DW)J0$@k zd- ztuv|u<);KAbg`jvutd_$PzJDklBluBr<>%;nvtn0icGbq)zzsJX%iyV0&BH&Qt^iPSrcy}01?{-&Ze+&-~gJ;aI%3%SCZ`t z&MJ~1hm|0JpadE>MUWBz0T>ij_Lz&UonEKP3C8H)t6_ow2XzfCqNa>>6yfg7==C zqxtl_)9K8`QGi6nE--od@}<@NFN4WoL*Q#oV(&$N|Bii8=dCy1eB-U-J7>cpc3sc0 zYx(Lsy)HT?9{7{9^P^V!Z?UjgcQ(Q}8hc;w{lRQZSAhBNOhy0}^^;KGqY&t>0T%b; zBaID+`wo7fa0HJ!;^eN0_cez+#Jtu+w2gf51F(o?CZNvW&i81i(tuK7(54GV4rsd{ zUTs$T;(NzA%(G`UhpI>1JzXZ z_5!J_eEsp)KiciF{rmmz?ZsD*zk2XO-n8rhz2X4<-`KOWW19{GTKh)32cmDqNjHM8 zXfa=Il0F#Z688?QNcv*zGx75ozg{{lDBuICrTd3y_fIvALcef5i1b@#KvV~#V%chJ ziAUCH_8pT(s(MoW+9LL47h>Vc|Lo5M^qp!S$jBW?m56~%SXd6ida%z+iz}f>Xtt>! zff^f5T|GpTL?Y83y2%qj7E^i%Q7i3xWzY&5$b&!w1sNj7?RgcI#DtPUyBob@KM7Bg zgc#?t2yzDlF%?SJCt=)|=i7qyU@FahJi1ez&m#83NQ1NZS0!M9Yxf>X+i$FOT8|Oj zaNYwm1pjv!q`+KK)51EkCa zUi|sbA4~sB+$I^0k};F*@i7bk<2USf`}piQN?(h46Z-y5CT&pr_<=7t)d#$3m&&g* zk_Okg%AWq*I%>3khO>bA<#g{%Jlv1Qv+Gu17E*vEfb&z80v(H8r|1Rw{!rF?^&b)d zR>F%G%;)zQhVo#iafG`9ErK3Mh<}YLXdT8SUjygQlSYm35@IIGOH7f{(u9pl$S0*p265rBOts`+1^+sh3_jHVtNep zJ6PIWHn_LlU--j)zl@S|xr&@qZ#HR$8O$t;OX3wBuHVS~dDBV*j z!;XO&YE`ib&K-fAXeuebWuu&ASvwx8_)-Rrt;Fu3|7RtDdUYH?5Cq8UMJ&Rf{J`B* zTaFIe7>Bs(NC#{{7vT7S)6>pCV7;wS0~4VM>~68xA@(C*Jbd!x zZ+}A+7wmKrCi}xc7U+N+0-Uf2 zvViFRk7Wa-%|Ek0=*wdOfcD1FYlEW!UH)XI1t1B4v=LSSDmTtgJ2V|?0or2TX8?nR zAM^9wbqC-^V!QBB77Qu~)b)b~*oEQ1tR(~vS^7~iu+^Abn$eQL+Ox3yXS0{O&ZuBW zsjG&e5W3@BcEJ$!`O*zcr4u?|OHnW_a4G`3DvnUvLII^4s04s+SH=vX)JSkDN!v<{ zej1M4VaBw)9N1KERFfb|=`gvp>NuC;N1ADY8n?S3EI zedcvxNjKH06Uon@&C-H|0?}M^S>K{+XYUv&~ibfgZa*F1|1=&9$?{L?=wW% zm(H2!&j*CB%>Sf*faIU3z^Q_#(j>I737*QCyO>n9JsBI{pqyq?6ZjA?pq2gt)wAW>4jf>Mj^4!ebzK@E(w#?+#{C9-%J?AR)&8a|b4d zec^mIQ+ANnxhMjH{c+H*WfKH%KpV(d@{TPbq2O}`f8TX)p89$~2Yii-h=2ux>i`Cz z|JJy$j_WKJjL$x$^;R~eKo_%DVGd%y;DJRYuuDA`{Vxlk{tPyfxJ3nq`IY6HxbF^HQu9=AlSW>_S62CzfBwBVPWxtPR8vHZ> zf=t_|+4L$1Z%lPsf#bKbRJ(Kr0i+HFp;q;v4w%~63al4ikMbk;}QCx%^po{yUHZDIk4EYuAioWL-gK+I(wlb6a$_y zRDp4e>g%{VmzH`I;+=rYaZZqO0Rw@KngV^rJoVNC!NH+@M=J%OGpe=AgaLKpul=xB zO+kgk9w6TK?ww&oaz^mK_15jTtO+<@M0-h&jPF`YK51{Klc4I=&ImfM4NL|L_X7=m z%jCLskpzgv^tI0rr#!(vn9S_`BiRlgk;hKYDgWCW6~Mwrd5ds#&jW5(d4be$J>YXl zkk0`G{8ja!v@sa!tQRaeWQ>_F42OXuG!LM_00{7h#0YGTL&y$VPr8n!4#e7kGCwBP zkqyidEC)p>+;X{g#(-*ms|Zv9tkwLxZ7}D0yep9YV#K#rO~5A*C~;nKN`+4o3*9(? zfGCO!2@%&mhM{k z2py-q6=gW;Im88b*|^)#NQ%W$*dhUFB|Z{O7`?nrSJGh@7v<85{bc~rk68Xyih1<6 zAfHQn!`kl6g=hdptc5yyo1paCF+V}9em%huP;!PC{t%^V({ofew@tGJ8AN_S7Qj+j zY47{EJ4Y{6!bs4d(L=EN0T(ROEY)4}GTsZnYu_9#4~&PBJ>sLKBuC0dPDIoI87l78 zrb)I)E5ayne1;BsdjK>BX(}p4tQqYYy}>qOQ4CTu-Oh)3-W%e{1z!g|;7}SMbt;w3 zZEx5%)108ghN-ERz;RjlUJ>QJU=Kzl19aV0K3O$ckaC9x8viEWq^T)Yv(Q_f_q8;$DN~Vu~M(! zPfyQ9NCx>ITFh}DyGpI(G`4I6)<3G-P4FN6jSc5?DABh3fIq&@H7FK(&cYG?| zujl{s*A;!(|Et#LgD?)}K?iK%8~?>DM1DIY6ghZ4@9ouL5b5*UiiOk@fR6p1uLgE= zB!buS6O}Ts763U*0XvuLlqEs%2Pjk&tVJILK&UgG=04rqhf_@0cX1|0`jjKAW{-@} zXc1Rh9AYn!N&y8TX|#Z@1l%XrAkfo+)fV6B{QoVtAAkQJe*E$2qZ2TI>Jq zAknIw&S^d%B9qO~uv`je=p z+kVPWSG$1HvI5Zq251G?7lpGX(_S-YA6THq*GRp5ik?>JPlTx_*sTdlt-cl=(zSnm ztd>ZJS3eK^zx@~IY1ksNBrGJ#o3a;_jevDSjUD;!*k#ARs5+KUfJuT8VyTd9AWBhd z1Si#^foDJx#6ESinp)o3z$eyXaRVrEt-(ZMjg9?5*fQEv>g_e6p#I3~Ii#O$Z;tv0 z5}*+fV99Iu2x+VNhhhp_3b+?mfn%%MLo)CN+rsRguMP&S^5>-G>D_alu1CMB00pI$ z07!TGu-aM+yI#hpr#+O^GZ9c)p`pTL9Yz2;|E&V> zW8Kj^^Xu`Sl7J+H^?6Ml&`v|!EjJnk2c@{@d-FA|)G9QXk$?$Pv)cJdMGNi2}U z-wlNWt+hTQ2a*V?vPBS5dGS%?R^+XAnIy89g zk|L0URJ^Ntkcv+c$Fs?#m%GCR{$)57Gt|qTz4+?KKPUtI@wb2c@$rveef?yaAx9ba zvZe-ySnrU>saZNU9PxIAXQ{Y_l>Nf$2RBmSc|o9~FNd;~#QkMnxLGDl_H>%t$q$ZT z358tbpXF#OMLp_8^SY>TVI%z~ME~Ocev|b>k^aJfTC_^BICmX;KcQb^p?RFN0Aj6MGn~t{EaH?J`J>Z?a&Zb@RsCZy#JtSvPX@66)v@%E>pCs~*jMK3!Q{ z0}n_`>uvH((3-pnnBJVJphZ=p97wSM&bp}5WKSiYSai#cTW~%$@YT(m2W8CzlmG*) z2te{WMXQU6f0pCd1w=pdu^)g0NPmfy`D!#$!3FJ^Chbtbo(K}da5EE!AV4MBfjaEr zmCXQY_j=T~W$q>f>s-0wTxuL1{CO4~>CKgxV}HOD<{x|Ht@=X>do za>Xm_0HzG#smPa7`L<$0%O?wcj$m2s0O7IbJWh8>#)P9Rh1G$3E4dvV^H`pOSgBFm zf@Ha#q92~q1;k*0(k)}S4OjP6#N2#qk8feV6;Y^3867lxN%-Yf1+AG7Hc}Rq2a%M; z?E&bEQbrJT0`mQyN(HoNC~Yj0AX)+ozurlU-3@_gymAnM>9#3q>pL>E=1>Dj?HaR+ zKbBIqWVC<>4>8-7_=ko6cI(Q(QU_qXKT|Q?8?3Fk<`54dso4=4aYLcR^OR}P802JG$GPZ%S(b|il*ffcg1t#73bgvC znOz}8wnIl;+rM2V|3%5lCvqenM0S+tOec_mk2r18UR`&;_WA!GEeZVi)%Pzxy>+;h zS8+<)fJP;WofEG{AtzVUK?5$mAkN@2i1bSdP&S}$6|zw(z&Q*@JoWl+7p6^sF9E`) zAI+c*wt77cfl-RievVLVOtoMdSp**ipVS>@9WExu0#qpciNC=rq(M=Gz*m@Y6+ zDI4@eP{Ux1N55dFcYlma|4K=bwX(@%|PB7z#jR zxhQQU{S_p^(k)W73y{H4m=;jQz| zc_-k0e9?bh{RfzD%L16qG~FjjEHo)-!2sY49TPCq62Ped8U((k3~3yo{or`G*tKAR zU5_@6m_`J_GRRI?p}1bG(cF)FcHGO5#JvaB?jcz0>b1Zhyi!}Bo*`$f7uRkPP~^f2 zq6U=7W>AAG=mHj}^O!=xF7AEW^Xkm%4~`Pc-MWa`q;XDtN(xe4_`fg@csAq zzdeK%YbOs9U`g(f^qAqgjx2z{r7GUcjrQzMZdEO7ehWO2{N*^jluSoejvxhY|@;PneT^tX`Y?8QG($ zw0h9nn1Y5R=o5(oFqh8&2iY+W7buhdrbQQk8_Oavhg>$iNot4ug3RGJk#XjiHLeKu z-PwK=glN1iEq!Qd2N@7>5X%B)zzbE z|JRm0?D)6hEwaWxUaPX+S}h($=@y<$JdF0WU37UakvvZFo?B>bqkA&`U^A%de%Lku z&it9kC+M{yfPg^4|4jl?NK-hPbh?3d{oacaB~5etF4hueI)iiwFu_*iPkmYFb)pwZ zGQ4{i03L$?;{QyjK+6E9$dXEv*b4wV>#fu{yK}qa1%419SuDnRf&Cb5Eug(;#|!@1BCvkiL7)bEujxP@@aPokeI4;C8;%yJ zjUJ7?U)W(ju1~b2VGRTqF%tLm9 z3-4mb?x5CXQC!Eh#eK2PSpTyVKqz2cAy~uaPrvb^&~hr%*uKD4WKDg#?xm$22$7X= zC@o>XG~cb`{csS=knrXMVE@1W;~)O;_^Zc{AOHB(pT6xEx+?Rc=6GY~)!^b_mpIY^ zpkg@7K-E$RScOfO&gjlN_z{#ondN1|@sCQ%dcxcK0d#;iSvk6xCb;FGrC0SCCdh;v zB|$8VmT>Mj64S^S9YQiq-0Mg!y&bS1kD81DoJ;!)gjW@1>criotkEk3V4rD{MguNm zK3okg71GnBQx`lYUQ23!&_U>kvxT!WxXG#p6>-3>`1uKOBm_lH*-%7y_3h*Bp3^ztpO&O#v)8B~4_k3)VCvu057W20m-7ZbS3O{wSCkS^Td_FLAkQ zdNmC}nxg2_OMAOi^x$C~3oS>-Z;`oAMU~r%?K>Rzh^BjyqkRX#_a}I2hUuSLekFjT zFgSEX9&L4J;v_+34N!rx{ead)+PxN>FgeQf^e)XFhv7zpz;xdLVUmDSbMNf<^!W7r z{P^tl?PCf6fdcOwV@a$gBk~;Jx#rkD;n6(|scAg$3~=Oi%F*~Wd4NLT)H;83fra|h zgpZO!75~Iv*h8O^$*2oXuh)RH@UK1d0|9V;-5uzJzlH&z`~eJ5TBAw;g8H@?0u}Z_ z6wtwd^jm>ctb6nV*Z@_dT~G-D|LggbN?l>erD7h<9~k>f_bM4-Q;zuu z3$Uupu>2n3BsyBzY2RO6&@1$(=ma62+uZ~vu|R{Y78)zWtw<=#HFLD(a>cfpePk-f z0;Uj|orEP_y=8_9G=GY82GaUTRk2emrbuh3Q_zYa(GH_3scIS-ALX%~THCZI6rj~H z*>)@KUDjO~2VxS6Wy!z7V5%5~mML)LJ>9W&QfyVNJ}(jRiJ}rh0DlrqAgjW%RLX#% zX-Z4HtWIEFY%OcEKe2N)iYqMwsUp%^EFgPBbVEyCg=T;`kU(q0<6gSO4ge+1VLbGk z2>#n%B((pL{ks?Ddybz~*&Vj$Ui9|{VIaUrt}|(;>u3Z|oUG^QVBv%L)PmE)E?H@f zQGn*MRw|-Y7=S$(AiDrHiR5Q&qR8rc%K*oxcc}q115gS$JDpi8Dsl3O`;kkXymNeZ z+V&iQCp?x)$WaScRDrUx$k_mjR*jKLXdk7xBtUW-h zDOw8~%$(2Kf3^T~Fus%p?n>sSg?1gNDTrklVC{f_eFaEy1_KlbAaWe4@*#YAjTIbfQQ|k_`NVW?n?lokO`COgF#ft~l`u}JLzwm!wJ$Uf_lV?|{7Sgrl zBKuyunpyzFuu~CA(T=z_akHZ4=pW6Bum(`Rt=y%ZUz0ckbMz=f4nsblhTFxBw~OBKS~wi@?V z%_uzr=qMH4*g`x){7c|teo`z!umra7pU>l5CVtlX$FVd(q^#+J0Fgv|AqYl;mKQ82 z{lItpzj6ao4j77{9|!>x!mOiRfDpo+({aZEfo)z;Cjm7e(wt!E5^{*cX(KZgBTy|2 ziU4vq-tRf2f#RK6CO8WT;7!W_cGQz6hG7hd9LdBE`{T3Ib}%7!fO`I;=gM9#^go%6 zCz=9?`ZKeya?OvK5BfVtRQwa40}}72Y6Z^2Y=GCgs|UQF{IhGo--6Pc1N|tJ2w@}` zW`4U^*+IfXmV_3&nNY+-ZT{EHMu)AKV5=;siNMS|4QYbSgI_2&7X7iK|CJARP(62U zE|xpkzrY^Q?njkCK?|Qy3_ihtiOQJJ%g`zY z2g?!6OioNzE%ulON)u!$Dj&|V%Mn_kQYRA@05mY|LU)M$Aa9J2kC7C6_ zZ=wbP>CH+*g-V8gAoGgS#XO}BW=RXFj_p=p{Ee&up#CLxWHO)6g%q+&^$L4)M*bG? zTg1GqzTOjwYVCanXQ61-K(V*)kzGCdfBoY>{n!8YPyfe%`@jF&fBmO_{P#b;c>6}b zq^dxnhx;a)lY)Y^ikR+*{UAxX=ZHc*&4L}Kz=VzEF?(pWS$PfffIW@QdJQbo5|TCO zqS~Sy_W#gQ6_#dzjQAJ=;E3Tz$^q6GNJP^rfJu9_>-Yx>(7C70yN}K~mKBaP-DmMX zK_Yz6DuRsLiJO-M<1i_-lQv8TkO+@)_8g3Ng_BV+ix3FNe!Xl`n|n}MHQn3_H~PY>eIxz6f|ulH2*3+ns)xr*|k^glGJaDiV!S^L0A zko?ickGX(G0FDCecz2*h;kn&9zR&YHKui3ahdKZ2_TQQR+6B&Dw+d(hEN0%}d&=-R zqV)i~A#K56_AAjKy%h~-vw^;ncT<@DRvg4Nr~poUPIwTzV*H9a2>B~k+!bo{Ii%Ja zxw5o|%53FA=E8x6%ZUndXfZ1j(pZUw^}@c0BYQX9ZFh@^>|@7yiqRq_a4rM3d-m;9 z=Kp^b{LgNGy8rFRE53+J8pA@(6A=!i9|+u`xM2je-PRat00@H(>)L}dDU}*gvT>IW zO9m)>a`Yzx_f3rSVsnD=o8 zUPpj`Y3Jk^CB#_#5%tUfHae7YAiQ{;PZ8-^Lf;h#!ipELQFi8T_fkBT73WrFJWUhP z!vf2(%eh$n<-x!Ir~meU{I~z(pZ@Xx{h$By|Mh?U=l}elAAkC=Td^=qov=^$VRc8( zYI-ZNQZ}S%lc|&^nJ3gp$Pi{0srxjO9C&F(^LC$=T3^5+g7F6{o3e_Gut8~b7Y}>e zI!fhffKWl64i{3?x2uS8ncHV#1?D1v^7Cm=?Ow3b3e|r5mUOiw0pk7sM2kQl0*u3M zj=H}i>%fj+JBt6|t~ckb=?#Q0agxju0?F zLF!1=J;<$aC-iQ;7|*lRNNr4`(V*p(M2KK)^eEOKF-yi=721xM#r!V^T zyx1@7)KB(%1HiPW`|<;Z!k z;Cm~ndWhh&nJ#54)Zg?s68lKXrjM`BR5k`)-iuuaAa}LN%37@SB&G?~y4F2zv>=ydDQBd0CR4uo9;v@50%4fO6wrm$5LvQUT|@hp3WEjW08D`F8m#$u zv2Nzc+wHhFo=E~!N_orxwwM4)Xsfl;c85Rb24Cap#_hmg`W)kq(6{$gk&mVSGYOv@ zyG^in5ngTZt%ugZ$>eD>xBJZE%cBT0If`LD%_ypd6`57rSVaS zSVXBG6h7i1ltdtWLmd=7?+RkhgScEK@qEZ2AZ?a^`8pi`LID543V^2{uizFAhc$|{ z>6{DOPJHW4YA7kdWMb)$sga2p zGCdpnN;}*4TL~-gMC8Xl+>ToCxeX%0tm1Iy%QZ!zp%VGmij5+Q(*E23{P^Gh;nN#s z#69gI-fkN!!7UW(daju}M!g{Ti%h$OwPgS*VarYy+LlMw;EFX`q%|nYDcZx`Zg=K)pgcfK&u7>y^A9gr{DkSS2u+3AHHR-^A5MrD_&XKjHBA-QdwR83@v^cjyXjn(`3dBL8aE zzqqdQ75qmt*BIB(pGm+x7?4Rc!LVwE+12AE5yK`1q>_Pj5~|8r=7+im;O;j@=!=<-=oU z794g-CP1Lmg7QkP*+>DYz?Tm((7t6We}{U<#)tZVfJf>TRzQ0!JTrVVNB}Z0wuc#c zIRSKVwOnqN*&0iYG}~&Mo?GtF=5Y-mheCf@RCeB0s{KI%ijjh=mwmTFA!JRe{em)8 z>H-q;tJzBJn}q?uyP`=U_W7|-?n8ro*$m@l_wt`VJf-rIPZ`i-_N%g}oJ2IZ8wTWvr;dhk$VdTuT&MSZb00tPfzQP-t|? z^fiqUZL@YGlf~E8$__hy=Y#D)I3T;K>akk=ZP4^?xqcmqe;q{;5<5%>$3pgTx;iVj zYQG)0KEhVEx`56U;VwsGZp$TEfqLy0URb65y%6iNC%!9URMdjR!4YlS$OW* zBY=U#3!Wogu$#M9Ku^x0!(KoH`$o^5$<@8+>p5RyX9tF_`!?It9%YxDQ0p2OQ;YbLHmE6gS%u|*BvJ$1?yW2=JB|X zB!Fa`!i!_J+&rpo-=)GaD^ctL2e!}K$*tcm_sF?+xyG++g1_Wsd0)SavtVtj? z&3nk$)=4#SPs4V0`P1~&n(7^Ae!NeIZ{2=lyt4u!O94pCPw$)`-#*`B?U46&DlxWI zZ7o5Q<#8T?$3l&moK&d?%Js`ip(m);z<33d#_cwa^2(oOvqh@`T8IxPV?8PI{4lDc z-X1j&Yv&LG-n#GSY=KLtoU{hl^7{}4dxL98O@?*9E|J0(j?*H*SLw zz$6Sy02%}h3K<1ym_$u2&ZQu8l!eB zaG_nTa#@6KfZWe;u-&2hLxW(Kj$qw(fj;E?fBg7QUp@X0|KZ2q{`la**Y}^CBwOoh zqgtAUkOb&l#TyX4E8RAFGq&s0K8+>^7j@~%Bt*LWdP)y#xph^+mdL^uIIjr(8#gW< zjWXB7>f2XknS19Z?)WLo-6Y92iPXF@eHcFfhM!jE=oX;X6y$EtCvu=h3+NXwFA_0w zqy+}`PP~1K!ICoH4*o6ebeZbFW#fnRE&)rTtF_DOfy0#&f9H6zk7^MBi)}u5VgXN=I1Lcww6V`@Bi_J$iLM0U z(mZE(8%YApIsq||kQV+1t(|nE{Xj1Ol=IFrKSp2MciT zUQhydP4Mw>+@A1){CLZ+HsIXPSf3B7?C(gUYKKkgBPd-5d zz`{S?3(>3=paCB-X^(b+0lWox$E5-R25JPrfZ*)5-6S65i$ja`j_dgxg@a#91Fr)t zFX)YlH~2^1_#=^)gCca#cpF0Qr9 zTcPbEVC~1E4 z!Q(&Jjf3B^z@NUpfB(gkYP#?3BLuS2d0p%aw89%kjT%N5t~$fVn4Zhcv~)`VaX^m0 z4FLP4OI#uhF8eq-{V7mDwuA`giqnS#pj8SW{VLBGmFWK?NBD{|I9er=6aa%Kpd8_! zSsPy?1!(Cb&(^uca+>vG_w%t}*+UElRa3>&>BjvjtsYIK8llj+=ZR$(SP=N(`~Qs+3Z!1%KYf4k$!iG)a}Kev)X zOdym&WS3>trgZ}r>RWIDn(Ox@`Q@_!O9MUOYA1uiJ(t!F!#~}@dHD+S#lR6jE)zDf z9N?Z|r}{j+T(#dVl~7u~0Fdly4tLgV;TM!D$5^tlODKpW)@9e_W*>wGJs=( zye7DPj6d1g9oT|zsT9!tPtQD-&pWSs_zo2SQwwkLCv6583ku8+aEDKaJBxJeU#S-6 zyb}h1(*E$qeYSYL2@Dy)K0k1I4=w(wu}~ThW@AkMcmPrYnPH4anbv?-Kf7`e58?`< z0{+yR4$3ExZa!A+->WqN^p9Il5T*fA0M?+O@aS<1=J#XFmf0T|iS0UOra(KgCYB3;La_y1&_{~!NI5AfCF$3JrZzt7%^-T!=V z1;uv2B0wI&V9;YHfLiU*Q?Hu_QQ&Opc9#{!rTt)#LfW$$K5r7GUa7idQS4J9Bf z#IilJGA6&|GG|Hj`-U=J(`V%t;{1%Wd67oQqZIA*W`3~*pEbOJTtny=S^WaGWUsc^cpu$UXnW2C<0;jWxA z#YXRMLg8UKD2AoTm~bDdgSoB&)CB}o@LO-a`I|T2{Eg*--~7gN9Vs0scD!|I2Ic5cc3{ccHJ;FlAwVmkmILH7|r_-L> z`L1YBtsj^NbFy8{I3Tvz?_pgG8IWWk$)DIBk|9AhSB+0wBEvq(jWSH+?sh(4A_S@2 zlEgfocg_cJex&)o>+PM*&G5*Q%f;K@e*NIb|KP&EkAM8?@%Q(iTuk%0S5QXuaG$ff zN4(?Q&9SX0w&<^mMnYl*`(Mg{0yAJ1c6jhf;-iGZr#}=|m-SLZ7ziBpL(2Jtk zaO^j+25vP_#XQ4PmvfqwMh>adJRVZ!=jm{`-)o_dHcHB>EoD62IXkn9d7qOTvdH?M zo?X8G@z~O<#PuxNtzZ}qXj_N$SzErk_9a0q+%fCkl$s#NdR6_6@3ohzR7p ze7n}j%s!y4?ONhumK((X)iaxO1r4@yKk){4@e&`VM8dbqo-0aC~&?nxRK7(W9vW=HnLX)PnXej(SnJ z?fsRwOZ?U&bOAyF2ogbWylK(@4{!d%KfL+IZ~oyQLIya$tuDYNgzzl&R^U{f|9NQX znFdT=_X6Gycp4Dc@BI9X)FC-g@Ba}4Ir6(}@ZT==W;F0eq2&)G02KqXS*Q?ZK|shN z{>W*-Gp`{e2sVTII}5@B_UpCUUtIkA7x9px z03XK;)LH-t0BM~RRz0u|0G&6NgJ`OyJeQKkKXT*g+SUDl5*5~x?}paw#KYk(e3}en zRROAQakP8(;_1_;Uq5(k1;AH-di?nN@1H(-cmX@0Xe3o)_t`RrE_^L6Vr!31q;6;i z;I07#hO7cQ4~0&&Mw2CYABkUYf_(3WO&l|O84Hx9YYAuQ1F$i~HP_xnQ-+vlI*_ea z9PCM6BRNQ7$f}g4y|f70)ODHwKL#&hL6hU7mTh|){O47^#lesEe(ORyMJd9sPcBgi z+^z*;WZ@Un<|C z@bYRqpO4{uUFR^9>lo4TEo$77b*V-;H{v6d9j)`0EG{J&H?0(>cv_-!JkFi7^* z67{}Z&K%3-6e@3$Lf*-xw+$D7_8hzKOaAn4L z;DG_&;-Etsu>ZFC&}z>&#__gH;csi^y^ZIV0N!|O2Uf4Kv>G7-Z!thw+v#mBTqC8G ztvcXsP2hv>wH&Z^0jwqm|6olAdBejK9We6d*^Dm7dg&v5R3u5>DSzE5O9E#p=*N&l}+^l=aUu`?DE^* z1&mwy4H7&VM00&J3D_THC81yo{jvzg2I>NJzKDzf&1OqK{s=`EIKnv(Qq49sy z{{M$?`*%tKng*Pc0gfdkY6;-%HdJC&0(PWNCZY?R26r*-0FOjCQZ6`Zqn#Oo=Yn>2 z;5)%n$UrU!FcXl+`uWA{>Az6Iv;$!gFbEStSO>FqM!+Dzz7$U(UnGu|JB|jLBFuu& zpi>Ei`yC1b41zyKRUCz(Se_s15o&)$7*nKMoTK}{ZW#ptJ_%`^`}798dGHir?1unh z&(VL|HM=|eZk7ko`cjENWq>c#V#w)_zGJ$+xc@!sANJoL&;Wh#r?0<$`thpS+KHMD zw-(_I(jP{7jwE?i1;J8gX^AR$h_xK^AcXB$h5-+*k1(7T z3<@^={>24xFXJz;ZizJydAY>Lha*sGqtq2m<845}%K!ju z(yC-qF`&lZZVS2hq+0}kS)d`{Q}E9fedXSh3p!D3k=IthZb=Cih+B297Necx+i#qc z36SCRkhW&RuE{svm|JDLXQ9VbWxv}-LoU|wa39IRC(@pUUHve!*ag)mnZd-}Ny{23 z!mS+v_9rRwhJ~PhstP^uDKY>%)C>N=v>?XcJi;*pL*c*T+E{I@$;Y#etx7DKwJJr*a1ij%ujaB_S_h7 z=A_?f9+-i$9GN+NAfSjO`~wAWHyiuVU-^U3U09?_3T!tV&toBhVQS22)q^~sMe|&4 zU^swi*+=4t>^PBm2sR9ekuvZpioI0@AOs+$0O#k1Td%qVWfH8tDnJrcW zMH;Ctu&bNr%Nig$_*(0~$B+Mj0?^|Je|qrX(~q-wh@HG|btFN-aIDZ{*%`Cl&S^&M zz=Z5l$Pf&TpNwO_(C&=RfE`-{QBTI*4#f?g<%zWWkjr%|fJP#MOO5y$|Cgf@_rIJk ztF%hkr z-n{6%@y2gD77>;QU7z26`vd#?3|PG8Di|_}=wVu_DN1BT$aX$NolF&!Wo;`?007u` z*`?@&3ns7y7d+frX;H(#>21e$N$q9&7xjj7yK^i$kk9prZUF*_eel2m{<|Fx0-%4m z5E5)4(O;DV9Ula^Os7u#7DYcEpHEup5TCR8eUE$-=mg;j69@u)fQo%q2!0+00m2{q zyTk1+>P^m|1ry9+5cp;r|8{%VU%nek0n)^A$OD}Jw?jj4tcS3d$a?`4ICbl%O6Uc$fDllRX-M#S-QI zS!>;Y=61lAUhw&FkFN3lFtwnU`e3o2y6~iL+fk5Aqatrx`);)r<5p8!oG0CZk~~KX z2{*kA^*ynA?T5fC2z#Y|!)j0xsVJ}naG+?nW!Ie1(gzrl-o(ckYkdiQYnm4N#NMe z;|RvG{WdIPJ%2_DP*d+2;K|SeJQ*^;w_htCJh>}NF{S_m65wz^NCQ*{n2tOQzywBHk%<{2aQG8j8oOk$Hi)1JbbX#$*q0k}j_ll8ItZL($pdhEv(2mbFMs%_moI<#(?9*` zpMH4#`sK@)AAfW2FX|2wpbRPpNJBIYEGpHBCV?WZYz{3O_!AvWoMR>Yc=SQWa@=Kq zvCu$HDjix6nn{EYvae`4h~&(q5jGQdcshaox11%C6_}Qc0r+%XOFDz?`Q;c*E>8bq ze89ih3Vu+61wU~b@9kvc6Yz8V(9(t!2;e|-Q2QGvc7ooZN*@s z%dL@0tH#QRe1wKpY2V(;v#Df9qo@E4C^~CKqMSx$>;^?_T3h=peNyQ3Erv`jaUZII z-Ei41+>Kzc@$xggZo-1kuno8h6T+YX$W~BoKPtCZ!ue9K0qV3;`d9_kkA4+|L@6w+ zH*bB-*n5Ls-r;Ez4u8$&^-p+!qk%3OW}$unT1*?s1+YJWlRzKEiBA<1ry6ZCPH6gn z@PV%Ar8Azl35>)jL8c^4-Nx8jiM%eL106nHzx=&-{(t&^{psJne*NR?_iwLfdzcKC zspT<}`>_=ki>yb+%&<^BO6zaun~p4NVYMBgc%O{jg}~~1XgL$4d}k=*PNt_w@J$07 zQ0`b##8@t&36%Fy{bSiz1%vK0JC(!PWR}dNp0Stu*M!K)rsG*VKG@w)tO8ilS5FBU zOHXl})ON@u()vdMOanDUH36Z~*DhKnpwJjm3_H*njOo`Q?g>|;4X`8iV5fexr>noS z$VeZsT2b5OaL-4y0N;=B=5fD{ZeMh;dD4qJ-;VN@C7DRIxs^;^)ruKlhd2fSr8wTM zAJ$ojBe-Uj6RL`|U2eMzq5c2H+g#1u_TJ$zJ)9UWIw!NZw!r5m1Bog{yX`N!X2om*Cd73FqVz zHfQ;{;$S(e*wo^NQ8JYl#@+UV3KXI@3e(_4+$sIKUYqbEE&z@bb zpV1CnxZ4nv0lO8naC`0vK%)YiF>Kt@r+a`Y->> zKmGns|Mc1}@88|lYIl!9fF2m}fK6tNN-;p_^P3yYLWgy{5 z=Newnty8z!)JMqQ0t(#6>wdsrXC%V{8X; zlFYIht{x}WQBaeNr%c!E2;Z|4Q?Myh+|Ok6MLBQ9{&*bOF9045WFCa|Kmx2O$UDe- zna?`O>IGn zN}!$AY|M67S1DNsSg}LI#i|G}0Vu;%|6L~M!J?EJQDMLK>=5ULWWZeeTP6iL3Xb+P zbJ7YAYmIz_uY=UUI(7N*|FGpvqo4vDr|z;7<$?7RMf15`zw)(yAM$DIry9h7=e`IK zI(~-%l@XkXt99M2gt$@U8p;Pz3?r&kXRE>d698aueo_S>RKk7-J$wZ#0xr)v{v89j z`{cPXGqIFbod^C}^kj zVo(BYj=n&S0SxcLL{HGa4+mUdoYElGdjY6|48mqnv(kCnhg6aWRiHYSJ%D&az6vm8 zil7b{T#6GE2Nl%zrxvUq;@dq1tSP{zkxxrEXgnU43m60X&IRNmm4mW}#$X{*VVFeP z1Bl~?FT(N<3xGfT{`W86zkmPn1AqeROin9^>~K<>0eHw1N++l)vkHvtA_(|n(Q;9w{518beBxHU5rH!X#*{7cICsX1_-r)h*q=#J_d9U@QE&= z`2YGC{|~rgAbNd`3DW@?3>HBv{mKGwn0d%5F#4==kPe7c1nCWY6SxhY<3Ita z+le;L3_?k8lWd0WplCJbLj>F_fSxJ7-66tEP%fn=w`N(Mi0`67(&jYou3RItbKm(lg10E=W zpm|Wz@GsDqVCTKSR?;Cp6qpC4u_l$fmzn`D0#a8gQ7-WPVPP5q0(>Xvt`z`xp4`0) z02m@&mH#pD9}LOhf#U*q=?64KKwn4`f^*RYi2dDaF>*FBO@RmmmkbS_J#!3kXBpsf zbkqZ19QT41(N@F+Xnz>}Evm)94fdR{kR!k>uL55*x!1D0i2^-XVh8 z&NNHZK}I-7dpHNhha{|%e3J}@f_1Vaa)`%%4dH&RiBwAt6zh>N-EIOKkr zwypR}u!|(Ch}^|W_!3JPk6$qOv$J0upqF;}@#EVM<7_MsL%NYZM|T3M5vzn*yTKEv zw1BDG5(i?k8IYiSie#wC$_~m^(+U1J*!b~twuPO~v%o*+*>9$XemrJTo>qA*x;%UQ(}Lwk&`Enz-AwBJ z*+dL~0xpZX66&Ao8S4UwDfRwEgsi=ar~?xBxxG;BfNJdCEqggu%ro2X;+_Ps{{H(r z70Wsnn{4B?yvW;Xedo!yO_P9kMOM%8LT|lF=-D#F!4FOr$V|ktF9JlO4Y1mmcO_MU zrS6i3##%jlmRD9#BsjMox&Zm{ChAcF?(#rRN@?#vdZU;)mNO7D!9n;dSnJFM`Hg}G z;EnFb&7@z5z$g%Z=EE9|@6gc*EOes=ZG`^T`vxuHl5J!|OxbJSyot1~lxno-#Dkc6 zgh2obWpP@`EAURJ0PY4lKu7=&J}(3wJUG7f%ac2wU7lE-=G0-@f%WAGV&2YsglAR@ zTpYUvWq@;T9qAaa1M7Df1O#R<a$gOiX3tfX&4RC7kV1xeki6xd8OP6Q*=5^>~p1mysBo|$fK(!{BS z))iRxvBtKsU$^}QR=yHa_ks!8A3XFQ{_uk~|D6Ny^3Q+%@%4-AC5G3j%nE901Tj;T zig1m8AcJo9exR<6Sw2LesBBA@0xiV#S*i{2yjCdtd;o`w{n4ch4|kf?Ph_K8t9(3#GS}7hSX;OOh1!(sU`o?|qzs z)J>W3^W{@fYC1`13m4dD?CU_C<2?0|0qMdXJ=@16?EuHD3hvH2&N`8ml6?VE>!5|3 zfcDt0ZHc~JsuEEUjM)o4Egw8<(W|)$SDbROv50=dEEKBD+rw-GP&3J;$msRyF zR~Il1>^o#ehJ2(R+baN-)iV@3E=ZCAiNJFfd|Ka;DHhWUvc-zlke3!Jau#+Pzcv+Ywr7W zyIw!@sll@wsDJ`Nw~SVy!yhg~I^fa_6aWu&!hF~O^c4V&|AsGL)KPPk0`al`Y8BI$ z>%+~4J>lH(zjiPimRmm+*Zm%k8$8~FF&BAuzh>!JIH^EoZr;7 zJ&IX7(V;$_CTb+c`;3;t-Yh5V_t-rqfO!RO1Jkyfm5LRs3FzsgsI}Ydz)tkkk^=d? zqRV_MfW6>7(z;958ak19_8A^v1#p`c43QQb;%w5|wZr}Z9(RCgV?o`hqt^;R6ZyGs&$Oxn*>@Gm zC8!ezKnNX&`#jidU-`06E~yZrJr}hHL}1te^f{okLG9{ukbfZb&lc|ZUc0b3koux= z8Z?Gew92&-DYD8iDvU#aBDSGS0M4J00PcP-nc%xx3{nShtopy!h~Nka-n?gp@X2v{ z@NK{mc$jBM1PqJEfjqRkDFb*j61mS|yTu^#v2yP~fqg4y3e`j9FIUk*wZ z97!?4f8*re76-9l5Y`Api^G_)*BEd>{_7(bC~OH*8$_4^c@4lpOxJBG1QtM#{MzvK z4fEDg;9(RC=jVIdDPU!WyFSu=dimdL^)P|~`0362AAb0E?_a-s`Dc*7_y6ww`yXGt zJ?XXiXMZghGZWi%V}}eC4Z2f63$^AtWp+p=t$06^tehgNOUJyI$p`cOu^ezXv4OY3 z)NgY7@Ps=oWz7eh$I9VZm?KQqk!KcT@Rf*O54V@f9uto|pJu)t zpuRPor)`#6ZCNSgk~os+CxJ=F+l-iC+3U!D(f0MJo&Ta*jP87Ykt3$gw7!#W=iZ)P zUjXh&wxMA8hv+ZbJhSXjYO%lQtuQGhKpyu6tmDjr-Odp}KiCf>D?cdSBr>zMcWly^;jIU-BD=a_=1YCS(Au0(vu`1Q6x`-tXK0M7gqp z!vE>l-#+yW;iG}ic?0&fPXdR~D}P4?*MoKl`vE&Tfss1=p|&4DelPt4=pS4FRznp) zrvuH0FH9u}CXhebaJ)9jWU_$;)Vl_l+VV6q#U(ZoD6R;qs)62+SW`0Dj?$1wtj1+-+NG^r7whE*jC$#Q&LW`m+~*R?{CY zKfZns|L>mW{=Li&YEpr)f-=mW6J|`JBO>|)-zaOR;(GR(%jv^4mav_*0dVmc{1r%D zO$rP7vBa?^?DEaVvWi7MwAYkSLc&Y}gdkd0nc==Am{1;iBLCfY+pj4nG;Tfny_NEt zPO8Lphn_Cd8I&AWjo6oH#E;G=hQ^s(xH1?*g=&3}fXIr2cx*L8gpe;g0yOwp+gU%M zo`4noDCZN=or~{Bni3Y>;n1~h+M>8=zt?IqPDVR6Rp;;v8N}7C{pQwp?yMu}^-!Rv zsbzt^bp|;T$Aqn*q-ClXlXe&@*!?DmwEMuHX-ga*a@H98lGGbT&J}K3G(rnh*(ZrC zfTEToR(9a_lW5SR9>B@|z8##36By+~)jcQx&xf&}JN|?^-WdR4z%OXeIdifSS!f3j zcJMCj-rM27^NzrI0im{cqG080WGL*^g7To%GD9r-As}^B$(wM#zOaM*TRYD0dKYjP zJ|GePbsz=@D!@}Rz*FG>c5_G&I`ltB3LhM?gU}T?Yv9f^N`cX6FbGgEJN~zDkbjus ztvm-T!f3B?OW^_WQy-ApjgBF}|B+3=sz#fu(d^&pzX*^tpn>Ts5w6&8IAHMytlUy? zvkBupRs0pv|X`FJGei^Ah6!k1zk(s{fB~?k!0$OdKQ{4;)r&62;&U)e%T7Oefnt8_BFK zlz43x&=Zv2QMW>%i+?KXC#AizC-{UQ(_h%@UNoU;TP|IbkGdZdUJL$mrd8mvvj?!> zLkMs>o0O3&bW^{Va{uv)V#3@E54}tvKp_XpaV96tX<^+2JXE=Ql%2Hr4}5h2xy}y# z7OiRM^GkP)F>f!{TfoEeBA#67F_YpVc8_Pru~Cz40*fwEhF`R3ivpB7qJ ztEx_D9F70_&c)gy?_gOf!y<}N<31H#hpl*4c>(L>;mx;`yDpp{D@v^0Lbb`huIMb8 zj3@R}A?wQ6%-Tez5D0}AV1Zfy25HYy8WhF?^!rh82*e5`aksBgiZ3_;nmVqZYVpU{ z08|gq(D-qzPpJ>6@`V}(G&h|5HQ5^m*7RKsH2* z0|Whse~GjJKnPP=1W9Jl8px!!4lY)&56tdgqMqMExv8xKA5_K29t|7By|q$OALq|( z59iDSH&2b+giMk|J_wAE6^swCe}DMBo%}HWU;n7~|F>VZ5p{$V0%A>-nEq+Im}Owb z3_070wK$4apY#(_Ix++;Dwi8q1TH-ZNV{Ln!F1xnyU>H}2I#%|NXI!#!JEBQL#&m* ze+H@FqQB4`##o)sW~GSp>=u=KbTi2Tc$3Yb!e)JESu`tac5_7Rk@jq6)%c$AdKc{v z`Blz&KgO<1!oi)sjum{%ZrWT@pu~qVW3CPbS2MW|CadKIpvLs#`@197+S*7%z}6Wt zY?)@aU;NV#FMhXx(KZrI*Rn)cUEH}dvPaIkG(H^coTJv){0^powZj_LN=loaqLX7= ztF2_Tb>aQ9Aa2;ADfVV4=V*@L))cKve9-aiMC4;-)2Kvts=ugEDeCh|iKtR2H zq9Wnx033SUhxmVR1P*=2AS@e%%gCJpj~l-}@>_(Mcl1BVepF#FVDD8opb4U^Aim#C z`A`RIu~i*r1FO6qm_9xUOwj&t*C0WFWPl_Xg<7is$Zt`dG@ABlV~~C!ikv0HHUThz z#lQUl?b}$>4$cC*xb8*ksObEA%LoB!$Z0&p8SY=b^r8Pt&HjEA2JrsP39_U1vaA~` zM=2sAh^andv;@#=*qBL}5Z+(gqQu*_4hUF78wA1jd%|3fdz|CU@+VU8#By87V4^{w z6#X$VwvXlY(H0ZQduvuVb*CY|$#^Dr9lDB6vjCiO*)X!#5rQUBU&WAo?8O!1SYiig z%OYTx9rguG*CI=uG-JU9=(k7R$C`suJlbx#O#Do;$ekd*iS{vyJRO`M&Ro^x`TEY? zbtmO7`@LF32_ypw%j0DB=*6G@?SK2~C+jmu6-J9?nJ?HKiZ;ZJ$|78Q@7+3;5d3 z^{@>PDu5^Y_w@7Gz}J4tJA3fko52745=IE$-h7pSh0!8R_ZFef_rO-Mzc_*b%`g{O zEW%V^6+nYJ7zj31x z_sa!p#PECbrqj-^RTlX?J<4EJipsSN{GIu!-ggjhd#G=1ZNs#{Z)BM7UfV+|cL&2}ivT|DI@K_|#LyQ%wb)-mnCIQ3BkQ0$(551P?|Yy9)>X zq!u)k1iI!N-+=TDiC{I@2c$tSklWHw`14^3DBstMCQo%6(tsN=^*E6&sq}}$(SEvY zi3E5dkmn&8gnznMQzv0$m=fcC?)Pd^pFWhr?zJc=OJYj^yy2B03TjO7^C&;PdhuSG zUl#g*V)_5&>-RsteEsuZWD}6UDrjUG6obw1wLJ(@iUU0}L1lniw&2hcvZ>Yn<5N4b zCR+2CM>vuoZ0%cp07x*gqn>#?q~A;PA$vArk0eFIg zUx9=d0v!|lGWVIKoc6w*_psgzYXI`3sEb2la1qM3|#sa_DJ8I zok-};Uh!+G2Sjq|5J`u$C5!={UspI4wx`|?O$Mo^K=p?ifwlPiB=0A_{!gb2K%0vK zB&Q;KsBw@iLykBW3(yIlmHqMP^4b5Lr+KgjQPU!U&(zQTyc)Rub2kK$B=E?K{~-Z{ zS^$h6SrAPEwn2Bdx}gUS=pe`aLJ(9HBt-*9Kt%>-8CJ1mkfX(1P7X=uyNs+0C+dP#yzNg$_s0VyW=*bf+20Yx4D4;%gFn|N!KGml^M~ni{ z@5Q~|NEt+IKX2@X6Z+@u2i(qw!ry^D`;qmcVGn8`Q2N51!uxhHF8gYQgUkwBeMYi_D)3<(65$q*=mbUwrm-)|(QjV9{QVDq_U2#nzxONvUmwac0{l;Sd<>53+{MhT!|ZXC)t@OMdbD@I`^3_j za5^}AOPqf2D@Uhi+Oi%bBU`QGJ8*9W|HQry>Hvy81&cmdW0YbH%nJLkw0tmGN?UAN z!UE9p4{D%oF^w{@dt|RQk-_eyYqgKq;d(gEqCJEDB%0DAaIhctdwc#J^2Kd$x9(%c zbk=Eekmj<}lGfk-lQK%CUCF*p&S%pU?#_6cQ*I=AVX<3vxizL26!Qz{0Q`PY4vh0c zTHKL~fLlTutuJ>w3@tA%AqGTjDT9N_Y$9an=LDv>-XU}vD zBx&Efwx1#oz8CFT!b#on|~K(xea zrvjlW-fxru`lym-Q0%LLI>0gWlZ!~J^x%)BpjZ6cIlkF27l2LJ=#jLn7}?DNYSWMds3hJgq4$M&U~d+FR-%vypd#(^4`C3fcTSt7(g zWV%lz@RLnvWgriVMoh8LN%KcqUu9xtd4$jyo*nykV-GZ$;@m~6K88}0rdp&!S$EGI5&n-A;-%|qvtbw0TTiAZ?Bo?zSRt+h@DFj_dY6cGyC`*uyS#x4WUu(F zw7SdUFvBiD?3*hl`%%=!nx3wR+8)+_JOPYCH!q-1ZoTutUsF{=@ZoT;piyuS3b)RWD?_2TSXJAR z?*Y_Eu+Tz6Ew?ncnlo11&acuSY^G(wR3?PycfxYu*PrzP-p3Ds-Y*EkA`LkEhf?6l z)Bgh!_#a)R1sFzte#$Qb0ua1_LspR>-J*_inE84_cA4E0N{JC5zrp=Nsr?Jb^|59lneQN zH*zCNT_DD_M?NtA`!lOQFMs%v_MZaaF{EdGH>aBD;?928E!|W^31IF0K5p#~ z*Rf9+u{%`2z>hLj=k{eNE!*oBotjFuKFdLN|3n4>I(XUm;dk5G;}$H)+4pASx5)Es zW=|T45KPz@QfdrHp^~?V$fCF%6*UDw!cIfN=u_q{BKiqv#2b{(TY{iC(z*($Y{FFs zAomsU{?*VN%q6Wa<6kxV+ueZq2T{QxZ2L=`Zt-*u97#zm_KJq7X z_$`0;cMn1d@MM4mMHb-pkq`I>c;6}heytv+{{aL{>n7~}+^_{2$%Du}*o6SyTEGro z?G|Gbng`+^v^?K8ggv~XAk~~-2Y4@YVUF9}C;c405dCWqe+dcN`=s4yMA)du2ZBr* z-Gy&X1r_Pfyr|em%Il!?25G4L?)v>p2mUMzQ2pEG4{xvUZBldvh5I2ZZ#4u8=SVg% zc4P`5fS_*h9@Rcj+0q4QgAMkNTK@6G%6VK%CK*IL3jf)yTQkP@c3%r{sDNW(@Y1$q z&bOaT_L337BA~SDml?t|lP)ObzyuJHS{eNDh1TlxV8z20b9+!J$KPEZK zKCp%3xZrdYdxo{UZk#5aGi!5;I96zo{@(B1=e%eME>ZWi>f1(0AS+8CW(CxC2SM7Z z%c>lE3w(GWv}Gu-(Nb6@w)f~>f&|JM2v>o+i0lZ2uM`zmbL%GNv2|QY8nr!H$7+LE zV+%_hX*8W`o7sZ9XG=Lh1p0LiGefn)p7jJsStNl@62W+nwU`qre|K^Ts3Z~`kbo<= zK4^bH4zzGgUCv+feN`CkE7%>{^Rr(?z+yCv1%2gD1mGQF+*0JH5LmARBCrTT0mtB9 z2M^DMlm*;ONW$TqC7VpbavT9%#okI#CyXHp*f?2`MFIP!=gI+JKMC-kSN{)=YXSC| z0o3zzJ@}|Go}Hr9gOU#K(q# zR*Fdo4)Nk?ETEtl3AwQtnq0qs|D!Vie)#>5uU~w$|K7j*;}>N?TG7wz zJ7ysO#u~h%198jW3URVR^jaB}NXlLc7kELa!cQ2>&6bStC!3i#zGG&G2#WX^yS1}_ zGMSdMrR5$>FVMb|N`~CJi{wDD4|&fL4&H^}c=|3@9g{-3&Wel;6$#}=US@Wf@As3o z8?(h%SlQebi3EMMzLZ5%`;8Q!|HWZn%M(cUeVj%642!)ut=Z;LgN?^lstjPU-zK1d z)#%Q}#kz`GtK#|H$aoJUkS^G7s2hjiTEQ?+IQtRAD~c;wcPkyxsM)VOO;iG)Jp_)y zZ>9rs>1FJiFx@MQs$y&q6;x zxEswEVI^Sh&|rgIPTiN51Io;8E%B+Ilz3d?QYmK!gw#483)*uIt3uuicmnV$;AwdC zY9y=a5N0yOeR1m05x zoL{znYs_G+{KS@sk)@L`8?r!7V^OC`+kf}wJq3U=z{?+AzJKxJ#m6_V{^nuXL&k(t zZi&UNHI`$|rMB!CvgutwenK)}8X zGT?aP7Q*&Qx0mz8N_ow)tZA6B6EIsYt=4CPP)tgz3MLSuBX9wDdea`YycW@rNm632 zWbtGX9(B`=Rr?5&+H*^&CM>UX({_w-FJ>S)+Itby-w1g<1OlvsmdkI?fwM<7K*omb z5>c*O4GISsI|laWqD-^*$a(9#%Zm#L^>%F1`_C&?VX=$xHMLgyn8b|M7i%j9&}|&E zmJp{h$G{@v00TC$5(HMJJu7Rfa457xKd!9?kv@o(d2A$gm867`8z++A5+LlKXh-0wuLdaIzkT|2IOv}|I1UVkso#UI|Lz#~pVt9@_h2;8 zdsczVyAsg1Xa;}q)lG2_kV3Z-;4XF05hZq`urzce{58aXIMfMyM8Z@V#A>wTAQGnS zUW<9Fi)tYWEBll6vJbOI)__pJqSk@sINCdH(ANwbt$Pm2?m5PUQf9883ppvf*?jo; z9=(6T|Mu_o%h&JUJ^IycG(9LR6Bj4v)nhNBsHaEO=n73sT9QaI^&9!tB%JC31|}PL zIW2quJB{tnhj7gXq^~Ry@v|>TaNW~(0HwYRc~J}pKd_td&!sxRC$N|lD*5U4SvHEA zNjc6YT@I6=*e6mNO%e=$8T8|@lTEdJgkiyiU1Xd*oyJ2@9?;LS2aGW(=wLMLV-&Z> z=!ROW*=5tHbLEgMXY*bnq3Y0Us}Lv)sz@$Z7K~ESOp&e{NkBL4V$Iv>Q*}^fZ2KVD z#mc>P3N#13P;ILs@>XO$d(kI83$(8*P;jw)gfAdce<{`if+HaWMX6_Ct2G#9fJdm& zukBZt05(cwhhiYDEt{kc_hT+G2!m7nS527srGw7j+;PB>&*pbp;tBPDD}&B`1eYg{($>zoRx@?UXZ&5xjf69c$cE)iA2 z9`cyIR;&Ohq$n>}iA*GPpA4j7ZJ4JfVedO$+YhTjS`r96UwiNSKuu)Yb) zzpMrd8c_Am;ZZ}r0eYkUC+Lz5pR<3F4+)h8f~$htiAy}Jfp$WfW&cD00&rbeTahxV zsB&n-U%kNi?}tAP%0J+LFFrn+@b&iGRWmkdH)&-c$)gtkp~PNkaLlC%Lg8O>6x-p< z{HAX={n#SE7I_ngqoBl0oTQIKe3 z5KZ6NWSr&d80`0f(};clNx|;XggyW=4@+^$*iJ#Qk0td3%)iIkTMmCZ)%Dy`#6jUM zaqE;dlq^ZpOop87_RuFtO%quF%K)gSvk6#enoihh{GP^#^kNOa53stkDC~9CRxAnA zw=$2U7}%blS8m1PiGy9-$QSpxQ)l0&O{22!W2iuVW?$>2QCHO}rX-^iNdR86eNlk5 zR*)htT&HOw{)lt@Wn~--=d)x@0&s&+J7;nEu9U$<$%mce+dCW=MB{E(4RXK4>D;Nm z!V29%D@cM19@rB5cF*{b6Xv`1aV2mYGQnuD=VGg&Q)lnkjbTve zoA!WyFnAgMxw4N|f!Wvfz5U)gfDI-2irOD!kk3bo_!QA)?ehX|7{>pv-&_2D{P^}XA?GwyCe(|yR?{L?0bB>s?zI8H{8^-6Ydh=^ ziw;l#WmC>Dhk-5xfCN&jRp;>i3BRnj({?f)qh`X8Z;W{mdw~EL^!uzu;b||*m4Wj3dQ^=b3bD$jG;rZs)KPh~m!1xPWz(UnWTE9;NHOGS$ zc~6U2xq(-elZ1)b`686`9=CD;B*5bw%Kt)#{nlB)RuHTQcW&3(-HmP%KzJB^&P4|3 z1M+?{nU!VXG}FFdhr?d?VwE!@oUk>N#D|E(%>f913Ujp_&;;%d1fi!-12m}9{prB< zv&&QM4ytbmaKP6B0{-rp0FG0@n@YebfJa@hqZMcxGy*Fp1`R^OK~}V(OIMK(oR+c^ zjAeMOt-O^B{z@CXSL`>v+}$#?fw1#zHyfAq6IIy#KXNbzrt)m*X@wB$p_P;cd~Kjw z=>>uyv1I`zGJe1Ibe$Piudx0__(MH_CxG`aUc7npDQ-EI(p<~M3;^uF&r||HS+rw? zel)7R+Q<`u4mp^-r)I)VKBabSO-l7F?H*Nq+--aM1oi9I6UZ-cpRpW-W>A5MIM5p! zqI4BY`-4L&>}1f38d!nuJ!|l;%-PXL(8|v^^crd5M}-Bq_WJMuiW%Kw>%lrNf`3KV zHiT@Usepxj1O*6@^gOfXzqF!2-nx>}=&UGdQQoOA+D7g>YMWd^DcM~+dr{Q2h^sC| z_fJKk$7(2+H;)x1Y*Q{zrkmCIdPQQWC_A7HDmk(@h{KWO^1|`H2&Vl z%RCtUg)RQMH|}EXfcAL?aJWF?f7NI()*OW|a;u;R|ngiJ(1J z@DiWnN8C@Q8!IJSV49@ISb}^28X)bY4MMB(7mkxuro)xS0;$kSOD5-X5)mxGK~D;R z21vX!3Sw5th>-PCb`K5Pp0D2EWjE&-PNKp^3`_l_Z#3`j465y!sbQvjk69T>J|HX?JKYux1Q61EZ|2pB- z*@JpR`$PAR*KIC=Iqd;5#*(EWlr#B{u`>{1|6D>&Zy8)B%&-%A2(rU67XKIQuEp=L z81J>xZ;ky}prJTKrBs1Q`pE$y-n^a2Uf6n!sl5p35|F^bAq%Q#RJBoSnE~nVvV?|@ z&xpJA=}p0o0M)%ko3{B9FDB&&#HdNzJy{t^BQM*R!Z*u;?kCfJ4hf*DtQvzVWXVD% z@6y}g1W)zSw!eX8NG1|_((fT3?~D2A&%ZgD)z&}qyR!;`DO|*Ozq7v!c4XEfG;@pC zjGg%p|i8A%ly3 zK*k~>p~i!aX`m=Pm(}0eIX_ARS`2J-JfNa2L9vnaZMGG*T{VR|7K^mlzF|e9| z%J+qRHL&(YfK`*Ph82Hb?>W+kVEwPaX0^AUra>4;s=X`6^j$9m5m1SPIHp%0g#)}= zfXTz-G5O^KJ%d!;KFQEN9ziiyW@Io)7H~udQjiM7X7lF#>mPov#oyOK`w#X1M>C7% zzM@#s46qXX2e6d0uSIE4LsHAdPNq$Xd73SMDU^bXP2I8_$YwxfX^i%tAj>T}6Nvbt zvP~EpS{X3OsCR$|*vc8(`D^9CMz9ai--K~g?7*8xo*4%Mpa;zQC+^1OkB@56rUeL7hT zGq7uAr$d#W9X8>`gHyF)yEgtlhtURYvgCSRp~ft zVI5iYcYW8ELig!K?2$G-%f2|ZiZDb#N17<~{Jq`J#SZv3PhlP?vMn2=S97|DmM$&M zVyOxAFugO1id-bcXyE?Se4sy)Is!irh<`YcoyI#?YM?_8i@)3LP_!T4d(QA1cJ8}9 z$6!}6FNOWJ5J9IAxk9k+y<;tC^nB~AKrc0hA|z}mr?Q@85^5dKhP=*}IETUnr(DLn zF)CQmAR)$blS~ODB6#N#^un?bJ|L(EJbfBifkOg#^5EN01t|Q#ee&R!?0``q0Ir4| zU*7@_<3P^{tDzwf$XCnZ&pxR97Sa zTJWO{%rLREzoUhIVbP8Krvy5usBQxLK^H8gEs6!^WPrlkcX`k$bCbR&h8B6$|?q4!#H+FRideqWX<+-qv&PU9q)2bL!|nY$=U zudFdopw3QczESa=5`I$p&wFX}<}1q>jN-T-o4TDGw@~~;9j~dghaW!v{?9L7y!-9z z*Z=Zw|GWSCzxteBb;}gf7wB zH7iGN?Dc1VCi~0t8f@x^Z<+i3-qAlzz!Ir{+J4#H+IfksC>o*N zL<>Ur`xWlPEgb+zKnVFVBec(z%d)U$5OgqWduD}`edhbQ-Bb!6NdfaEj`8G+(bH>l ziTuZhLOeJ&iEmvA@uu|xPy;{tq2btZk8ICDGFvfTw(22N zc-UbyA8k%gzj*y$+vR`!U;n57@BjEe{ICDr%b&*xbpyIbln!$%&h*#KESCZwKqqFJ zkf_rTX}E^pSBmq;E$Idul7w85PQd!olgNR*4Jyd0UhC@HFEro>Axg4 zb-T)cp9#n%WOv*Y-l+f(e776;!C_TMD{4WT$PxkKRvRl85tvN0A=^&CRM-}$g|rjL zPA7FQE^_8#N6{cIx>_ioY+YkuCm#TcgS&w&U^&2!{cj%}QNaI00{Be*s~POTR1VaG zNN7k4)#^y;@s8hRw}jj2i+umjoxehI(3ipWkMk9G*0K=yk_vz|3Cm@PiKFI${A58e zob=-PLSC+kfCH@(r>V-1UWIziUpfyNpjxG=c9Saji}~rh_XGIv2*WbLk3ibrY@T8-f&wvy8BHqHDvi#tD8UYd{w=nOT|CvKhXH^_7;M2T zn{M#EW^T7BMHJ4u$&CGA>+9@Q4(eZEIsg~6ofHvCzz&vL+QfEezIe`{^Hk`|_Xut)+nf_YZ&g?cbaTf|mhr zCqWJnx`^Lq=EDZI0_|+qUk$}J09R;QOd49y`?_^q;4}qV%ti0AptPTrG#_=;e6d7@ zASGDjYrEFIgfQ=lG#4qv{-g*HkpOa^W3s!9*vGqi<57|Ec`JnmUC4Dl7zWYhE9R0UXun3jcl9c znmfws_|91FIEOKF#Ox2T9?7&N>vFzv+#1c85u zc#%l*)K#m7avqFF$J^?00-fVKfZhY z<^Dc9$8dRdGKqy>rI3js7+Jk<%}7^46HcohR1b{>jRIH>N-5IIPrRgFO(l4ek8A$twxoEyY6|f3*ckuoaIe30#?uDVB z!jxq`9|h`j0DoHJseFXDou?!B1Kj!2Kp+9EgCq#Yy*C6QX=rzNF1FzOvkc($g37Z| z16OaQL}w=Be}R80YU1kcXp z7%rvpr*aSZMIIOpJiuY>@5TJE78K0E!@`{u0qbBE=(E6T)ht3ypqk!+LDui;VW_tn zHUe`$(R)2Y9@zS51}$_~$iI=6h{BF=Pls5L_gyD5g<{Rr!6{rEOAI%B`yU&>jE~#J(9FFRN*+!>Qvz zQiHjkh_Nb!Io9nFfnwQAVwKK6C-5;<-9H-D`gziYhrRC!v-K-k-{b1efD3Mq;aLr3 z{9JJVLZ>=K_IC7n9FKw!FsHw(gGmvH^dN*ljA1wi4ib*sr2BXY^fkQbDFfTq+RF*5 zlWd?zQEwmMAvLuZo#gB&M`bZ=KOo5{>A4nw_JUW@ z5deg$PrtrWN*L6Anpjl8Nc!OH`$8>%Cy32wzjA%g9G*Wz3RV0oDHhSxY23gjfG+^{ z&FS?EHT_ls{P=^f|GoI<7cbtN_BEcZGLdJrL{W;N4CJDu#@by!O3beuy`kElqLAst z{uGFSYt1z)d(^%X?IcmlkL`fw*(Nl$*0@iA#;1cT$K+dOx5^Tpk4j}g60D+bqQu=V_cEZG^vW>EjMQX<3Yrlmk##1YiP^OeR zur^}^a?uY1Snjg9(b^m#W&_LYEx zNCZqWt`-`iRT|ffQfU%;Lu@6+PoL9r% zP9;TPweLG|g6Iskfg2F`{T{a#|BGLSzc>H^2pqXm*v@4HLREo)7&jha03N}wdeOSj z+`2~u8D7bz-L&qE0~L*o|L95g;na6o zzg!`#oWj9hRfv8u6==ddv0ha!|0U2+cskJ*fdR(7GYr8EBK_9bdLXcTHnf=mp-Hft?MgVQs{1t1tJbge_T${#joOCV-FdiPm5WuC8=n5@HL zZpFO_gbCaI*(72`x6lT#B?#=tgeZ2fj3q~C*{7U1_mIP}%j&9$kk?ez-APu(loxXJ5Y@zftx0q^Fg~;2ljWpXmb-aapF`$W zDqkH_o<-d@|I1h7<=NT2Z+`ce{Xz0O6yK`ITiJ?nixu}W=+3npz;|R+hc0Yq#aOGZ zKW0M^Q}{*#B*G>I1$0&t%w61r>4g|HZvIn4_qI0b{{7{coRe#Hp-PrxxWMWJq9_1u z`t~yR(2o%Uo4^I+ml)W>rGbVjV7_yazoB;DdK(WdU@;5@IsOTK?FI$%5A%OAf_DYm zFbo*Ne|vnh@J&PQ2cclPfgTaVhvUW9)WPw>x&rtSNFL&MiC{w#Up+s!r}LA&49E7v zx=^V-gr^Xp-5dPZEx{z#eAJP5fD*6g0r>$7_Li;@->zeu=DoF8l{p}pgq1Km6N(L_e3pnMoR1{no>sqSgP(y| zWZ8x$TAX-Re=8|)bLL4iSd(ymGg+I*y=4o#duz$-Af+FQh!zuW74KEmI*&F=HTFw6 zuPp(ZH_VRKU7WFEB?`M3TJ4eu)eHnqTZ`$`!$N;-Q_}!|u zx3C&{F1RTgj$q(U`NHmd>>=jiUjVO@awB0X>15{vajr*@R@;GyTGdH*h=2O~c&lT6 z-gpgQ*>kRLBTy6wcxMf$6#&#E6A-NPb$}?Wy0HYnn(^p(_gQV4oEP+ize0sT~rmS8fxe<6bu5kbGighDNGBy>{=@Km%+-7HM+Pj2bk2d!Yep zA8L0Rk;JdvceKb+9LcTjugC=}>3;>hhZ!*RA5njS0AC*IKUV)QA&}%FmxtFkuW@HW ztqiovasmPYSsy3fui#Op;e`EN><`p(hWD}%qh&+z7Xd)6Z-MFek__qT*GuVFV<`%p zZ+~4-T|__Z#oICl6rV^P18YB8inr2SO)ay$i1uQgVnzU{!>Yn4eM&mis~Vgl2C`)8rd{y`>%!=!Padg~SrmUQe6hmHuz#hfJp=5pX= z?KX)c_peakOQ{O2FZT8ldlC{Nl<{DuH9k&s9kS^2NLB#tE-8!i?8vN2+s$8ZJJYZD?H6tK~g%UtDx1Qae&zdt|wTXDlqo4yo3s5ZvaWIAP zXWKvC8|Y@P`1a2t6ag*<0@z2~$4(Qh#U#`yU=Rzb!#}70QWjXHqS4+;Dgs^sh)pat zphhzWf>hsvIa1^5-)#G*tRssOuoC>YV#ioDXnOYc{g0aa|H$UAo&PV0{ul4AA0|3` z?BHN>f__M&uEAb1E1y8vi@c5#ZLcY<+3^evN#!lLfGO?4k}eii8@GEqh-Yg7Q6_ca z)jEzIlx`d5{+95PY%`l^Tvq@s6eT;oC4vxC?(LL}f;iA5vx14%LB4$1qy!zR8_tAfU235H=u4zTZ(D2yQpTOtsp)oP8tU=69$MxHkxvJzzXz#gm78CgXLjY)U} z&yJu22;Vh^*Gm^AeI-_%IzX?~2i!sW!~U{KW#vtzC4V83i@7+yk$f~%!;uP7#2eC} z<{8xitmLLq>X_k3U4Iowz&;5G)&L9V=1{iJLqZ5d;dx;H?0nasx?W}TdLRJood2tE z7e?r@t=oozp9#Uv@xNh}VCP?mY?AT;-(led<#}3fB#@kFTh4j`yWj+4@8F!KXyh>v zseU>@SFx;V-9S2;r?p-nfi z2(W+X_=iFuxPUX(*QW2P8DN1`0Q}V`q(LNrYG5c0yqfS;fR%8)KInze`wR0Qb_MD% zL=ZS=JwV#0xpsfh6(ZOu>Zd=xSK|-uFF1ee{D1qQ6`OD+ zsX({_YFFqk!TPKok>Q}$RG_H|dzBf1hT!~FQ2W*5j%2~1-Utxj_@gVx?KIq5t!<~i zHD}X_-MN*m-C=q%YqN2&Pm)|RVd#i1;SyjgSlU_YqQZOq`9(B@6CU+wcp!3N)5qCw}A+*^d4w6esCu$(vB<`;b%@<`c zB!Vnl(e21wxRwuF4{?iZJ3VA!QQ%)_{}LA*%veH}f2V_Q_9l+t&oj%kAJ3lbUT(u$ za!us*!Q~)1BKrll0zwj*iY1a_WYTAqeuc=Bbuwtm!vZGe#n;HKV_X5=)SY}mgMUwu z_mCB5jy^;llz%l5kSvfiK`Hignr~PBSct=WB>M(pfXf5H@DtBx=kWUWx$wUEg50nc z1gJ&ftq%5J!mW~E7wQKKy3h@5hqU0)INJGYpxrb6`@fG}lce_A4~)tB(@pDiacdPf z5}++hLcfkDAh@r0p2!#_z!r);LF;y`>>Wnx`_lN!2 z=?47GTG+#Z#J11u-nspdd(v1@O6H{4lh+FMLDEa#OB{_YW-oe>OOtL-U*r>B!TKB#ZQ>H}0iYC^<}<)L4}KqB_A3d@^MDq+HeY0~eX&Q}4PfqDzrg`G zNCAG{%ZK1z>kxOwugd^v)ORcn6#g)SM~(!pk2d1Ral$rejEOD0@D1bVS68X?5g<*g z3joK%b+)vZkyV?Jf}sVLYCt2U%se|B7|PaGv?kyN^a3jeg4=~1WMOA$>z6e$0?&{V zxbtkS)ko?R`^PlzGr_Mx3M{CEHQwzn24@h5|3wmP!s&ao5_Ia1^Z1(Z0iCvB{@V5b zxPtN*C4d#zR#2p?+?ND%Q3iUch4bL#i7H)HYyi6F7nuKCCQBp?{?<)u{YVP0m$A(% zT`ct{3ie=FqK>6?~}wy_Ksa|rg2eIsq?!S`pv zS1JL3D3o#=*tZ3n7Kf+DDvSRLqN+? zgs^meu=Ff4-wxE;U?OcW*-H{O@~ zLmb6bklA@|(PyVpfyA*iQOjn1xkls+eN*`z!;qd}aH=0LgXlnIfZAz!2qHL^Bo49j zm9>j)b#S5u1!kEY&>}i7;s9Ao)2`+&^gkb=Ob|qOwKH3#)p6(N*!bu2!mhM0>_hml znG>XYj;lY4*Xjo93;jJu>n_j(1^;_dFX6A7WT0^%;eUDQbUe=qyQ6(rm=A=6umD$4 zvk_L5c}}cC(wegCngp5bUThQXrk7e)L+TpyBj2TEcF@$Da(Vjc)9K{Xr%w+*EXSKN z?&Dr;VSB+_rE`An+X9vdwm<#!oQhyQx-tC?8-YzY_3Oa!3F85809HW^EXe!?i@>4K zKMDl+0MKjx{;2UuZiw9+Y!$N$Xy^J0DQ_u^J6EW2@E=-djKtPeP)vS@vA(PDgLcOGl=A!Gn9lQ9A(CG~-J;LFT` z&u&<1PP({vfI(eKP-=nXj9@EG4j^L`EI;fy^uiT;&4eZOhY@hhJq3imy$P`1abR$>?&nSX;Pw|715I%I8iaro0EXDEIDcDkAYm*~UL}+` z76f3~W2Mf+KFX`z92O#iKbFnWBS=?1oKA`V71BV>t9Rkd$M^fiP5i%_)WCiW59t*^ z$0aPdYV4QWqCJRNNiXFQ4$(w!#P)*$kOaz5R?NnS#9@C5H9K;1dzgxe6+8G))2$lV zU=eIRKUu>Xeb7M)0F4gVgD*k+T8kG8dw?-Ufw%jzikZo5jz%YQ+O99OP>xK#c8M8ESnz=!?WIuWBJ^45e4@;4Oi?Hri?e7K4r;>w>^>T1 z3>^0d>Hwq$yUt#~l~wDyS(N1rRRbD>-NnV4IOgc*;VibtWl92=;AW26Vd0huqBik_ z#A>u}yO%U-sr_fElapn;K$gI6fS*zl+0Ib3qD=FMNTk5XeTV(fXZ|SupIU-kYqPnp z_c0mQVH?K$X;=Z+ZbxBAIC5cc-|zLqd462$8IEq3{SoLVlHY*oxgFp-V1m2dbKSOH zJ50C^Xup<**Ul9P!U17FfYaY4$R)&~az)1{1u|fWIBRO8Q)z38U292cAtZZMp56ZH zt5;ur_4w6eBL3B@M{oY&(Len0XXSx6mIXfC{?h)t{o&z>l?V3sUB>BRD@Rj%QkOvw zAozRwN}q%QeCmHh1IP)3_ut?Rrrc4lnj=^6GjDKmKp)=ol-t1?Sji!z2!Vh!|9R>b zLasC*;!sL0A_;-UcmEk!q@3YQ{w)4;>Vbu-0wsVnirecD^DiX(^TJcWi??s@m6)@p zwevt0>?&sHOK9NlDRtH6wTgpA8_ALf2$qCw$TFJmsnjzKUwd(gAOqosVK3Si!L115 zK%Za(HZ3gn*%G!&q3t(#26FOSJ-%scjza4UYR>%vO~P06+74sIgmR81o1TV29M~A$?lX#r@ z15sQQh#Kv)B$!sP%OC-y&R)S)q@jdHf9x(1)Hw;@azqqc@n25W0ZjMB^UI4JO6FLb z!E4y>l>v&-0HB)8v(s~QYXZKDe=}Sf1hKYIkz-#|1|0MafZX?l^1w3s>{5ky&W{gc zK!*na0TzSemoMl-^$GI;)#cTo-zhfWpdL6M4Rif%sQPyv+uLDD$j5`q0K3sK$amp> z=T>F!ykXd~MeJYyIm5`!L~zn_av;Slqpw7+Sh@dHuwO--MHi#Dj<1pP9UU+ZdgIfF zw+Vks0AF3ddGqGcqd)%qM|=PCn@76w_U*@yKYaM|!)^QTgI)XZsh@A17&yAo{tW0} zSoRx+{LJEyqk(!z3dsE|Znyys-VJ==raYJF% z5&VSdAOo^mz{F6&uN{DDK}+W}0G{}WbA9sVThaY3?q9R^V^{z1{-YiL4>z?K{k|Vo zHBtcW83IPqDz(nP>Jym@B6qh^Qi{Oo00ZoFw>BOI5j=!;qm{r-pX0Crj#0CmkCvT_ zBT&aD$yl~E_H`|GMufYC#$;nPgJlPFgL_`>)+Q%$tma#OQGGVMKXvCgd-*FR5D7?% zC%!pU3R262tL;^fbD3^D=;c+WaRN0qV?;u}pa-XQUzSTYkFET$y4;;eJWY6z7v3$X zARq;}%&3)p(%P7?Bcy&Zm_OqvHc;2)j~a+<7J75TG_Fi(jKexrV`L*1x=$3S53@T>iiMmO+6m15B+re)ive+ z&;h*t_BXu-8EwRCs?M2mn_1YL~DNG{W$iCNvX$BBw#^_z7l2>@%hAw?~a4 z`}sX@uE+`_)d13LN~K1;MPSx6Y+$lTBUO`3{pp*vR8>*z!^d@`2Z zXh~%)c#U2Du?$HNAEXeU*^M0R<0)KY$Oy=fO_$|2UrJRet4| zzl#A@3pW7PYEU?bUAHSFfPwL+q!DPtUf1j7_w^nh@UOSWs1MEldKU)n&K~f=pp%By zyI~<{cfJl=!=wRn*=#g@uxDy@{$?{# z-(nE~Q?R#oKiP1kXKo~JS-Q*u9>%gF@Oh7_f4gZaHe{@NN8EqomcSE%|E*?VQ?Gi< z5OK%{Y=}j1fA*Z+atyHJdZJ01eYNbzQ4lVsrGRzrL7-zOIGssQ+AhNH#h$hR7aYs$5 z05)1%Mes=9BDd49-;5_qAp)g_04N|$)blH!c!J5qsZ6j}(2OI|u(FS1*$8y-5M^Q9 zbF-Xt!^I^rJOSkCO|5@C{_S!V=pt3H=d7wh4OBO{@n0a%g}uN4{m%!Mu2X$O)9%%~ zZj=VVdUVtUce;RR1G|ATpqfD7;Af(NfjhVey??0uT|Br7e8f}F*|LWDNM~ePO)(v?0|7@ZE*eZlyS@b_5N&E~%`1nIf z^E`<8td32;O9Bx0(gcKj@5g)`dqqx{9?*hj&<5aujbcC3-3IJP8&+D7%Ah-h)<;1B z8g8-|48N-JA!FDyNE2^3Quw}(05JXcdLPJ--fkWi0xkbNen-JC;Xm~Otp2@t_v+!N zL*@BXN>R*NLIUuKAkbkI=8$=h@lW66E?l!=tZxC`NA4FOH;S@F$q-^^ddNn&Bz_02Uw{DL|>4wv&9S100I| zWQ)eAB?0qDbZ#0!xu-BzQN#t7G9eFJ-(%o4!57`~lV8T-7l^FPtI^a&tIfShIt zU@t6yU%@7z3_g_%tRwdpV2~M8?{|qbOf0M7IKn0YSOCoANQ^-L6M7Y(YF|fxqtFj# zKN8<_AwX&V4jCcXe$Sov8)!eH(Qx~~^$%g~31H{@fnl>3 z;K;z%=LD)UltzX#L9|i4Sb>CqixNvUGQ7cf}MDhm< z;ZKgJSIv!nfIkb5@IV({`S_q7x{HOY0o4-q3JHf6Cs$E`YS1u*v>&k%n^?n-N(K$~ z8{%m(!4Y2(jDc|gv7g%a`^z;QzjOZ-{=fbB+mG+wy}A9N#n#j+Jx>F3z{2MEs`2HQ zN|i{|NTXHO#oO%|(UkHl=k^IufnQ3pP2p9&N{@Z!pn_bCa54p{OVm$CR@K-y z*(>`j%k3JtPql{qOSEnu?_CSgtB(mfs22ERFa_BvmwQb*RRty5d^ltX{Xt>i#1}aM zez25^PzPrtA(p%#!7n!gX_I&XFckd5zu~wa-Uc~P_x*B3-SBqqOF$v;B+@f-Kfogs zNCQCi073)7%|@X*U{JXB_Fzy4cdGAhdz=Y+ZWtX&12?8YP6hI9fg4X@PXhKd=Ezz{ zdikwZE(feAU$Mo8Aij6kOa_>cb4mNjL(cd|*N<-l2Yh%qP7dP_b^|B=<9oMXef8Cs zU%h%P9KaIAW6KbOCZM0zNBh6P{24etJ|_$`AOQqv6E=k<9vD2ujs!x-1~nwmaX+@K?@Neb2Ky%=~Pj4yz-(vvs`h|$U zmJdFDeEay}xRHqPA|wFvjnhd}iH*y#nz;+(WotX5OrGR<-hyegY?~(5-}i|3Z?vC= zC=l47x^ql}5aXL95Qbs*w`k`SnjmJIzQrsER=@b19Pd=#J`1-v90yx*70Dhd3K_+- z8hI7|3S{c}nI{3zw%JmVKO@wqg#OsGf(iplg76`XvE7ssaCHk5!nBU1Pq|nTE{*MM1(*!z|~b6yb3}pxH_hQ zQD_8$2Uy_v&YwSD2VdVR1Ul5%xyS!!M4kDu8rTkD>jAzEl)zBx?=Q98*5&xIN|+qZ!qeEqjsgl@p+daEI- zg|dgK06~gKxc3HLDj@jPxPfpwGzKAiG)KX};gUL#*f^NMDvl2oSJ1l~W{+i&{DUCgZCZ_8wiPG`p6ZlcXx;AzNzRv6gHj;(!fF>UeR0uydF zK0ITiAHEMKJKSKVXT-`_iNFnmbUsh~y-@UtPVS|l7J}Zc%Lp6@(TTMv1D+6 z=BXfjN<~8WtbLg<0xu2JKukk)s$&>yM2I0(={$i_MnwKzLpoaf!gD}LDzUeuHcJ3! zr}ys{qftAXjrW^2+0X3vS)hhIJT!2jh7vyacVF}{brA5!B%~D=F=4SmX3U^|8OObn zK;p?1dnS+qB%1h?6$yCBJ1h8R9Oyt`u5FKFZIF6d;I+Xbuove+DkBY*057#G_YWZI znHho#Iw^QD52oQv2jTIk0oaCJz~B+&ErTxW`62*+Yo`H`AVx|6!8Rxu3hBXoY_Q&G z29O@UJEY#DNbo98J*52UYndkZ^OKixkH0B@JC=X-2A7H0qE5tq;)_6XS*|O*R(&SYjhv*<;bsW}5%E z5&+zkYz1H%EdNrrd9q2?Lfck#uN`0*KbbrI6*z`zN>46eR5w-`F1&AiEWxc+_efQM z>U<8>KP|ov@To`p^ceXwzpnFsHoH$zVDFZ**)Q+lcM7e7rJos9f!7s2JQyoHC#pnB z2>g4LOv}LKFZa_u8AH!!Vxc{|_3G`LH}_7hsX@x#{@ATO+86f9xw><3EFCS56Z8x9Y$Q-twOc&WH|@Mj?6sB%$_LE^L8 zZM_fJ&WEaA;Gj>}hVh_!g6%pW0WL4<&3|}zTsc~AMRVPM=gNP>i#$!eX;uZRBQma> znp4+~nX8S~p&W#~8iY0ualHBT;Y-?n3-`y&2I&L7`tpNrv{GQX{L6AVIu7$SQCKxM z0)dN=61-gZNA`LnH0yx~tct+mUz`0f{WZmCQ2*)sz#w!%-@p|BL628DBiAX31_eKi*O^ zIXF5n^J0Jx?uR=$PB>GRafogLZ$~WvfIX{7d%ysS^(n$+h4&Oh@Fo`_u1y)`_|d8uLxr3;eX(w6jtuJrgo}4lm}$>rD-56cYqC3F$BCe$PfGP zzB@ZxLYlST2yg5Q$9$!gf(b85vFWQpcG9iGtA!Jb7NY^x_defg!6Bx%o&eM;Y^5PQ z6bWlD9d>~~qS+2Wn&G+LhL>nDXlb)6`OMi zddC0yq+R{+_T!5`sQQ07;Qw6Sy?MNhRq^?-cmc0cKb>2ME-HioSujeqxd`17I|ojJNkIF2nM6`+&=cH!u~ulNsh0&e2{(mp*T zg{j+>mZAJ8Fet~FNK69BeC)@QtPFsJcOuKZY7jf>?Qu@e&K}mhiS$E?PP>540y-qYCo_@#3fC0XP?4b| z^KNt4YklExNU}fjvnSnE8CC0JiB-An{R%@p?mw==aH?m#u05crjDNepw05UWf!pU|IpY zfGo7C<34uS9%r&S>Hu~B>BEOtk3YO(BuD}9l_dkpfiJ)O@c8ztzj^iQp=HR?Q2ht; z->?z58p2+oJyiXU3MlZ?@IIi0&7%2yAxPjv6B>cuRY6=gtY2veXV)tUd@ej^q;66y z1!0#PbzqhsS1XH>FA!T`*j{WbA=Y1hd@;~}!-XrqefQ|J7iP$zF4+)Y0;D#{MF5lj zpSZ!RfU;O7vKu!w^}R*7?_n{^%Mkb3Se)lc%XV(0Ar(IjGb;oHQE??RXlbjHU{&kI zEI>fW0HyZdM8>D`rXip~o&<F=joeQwf!hycg zdQmxX;_{v_->{ru@%!-M7q>rrxc&I?Z(b3SL?U_SFFNtjChEi(v$7H;zAd@^CyLG|Vc*(-6Nt&)}47CV#aQ&Efy^o1d>AUH^^!B47RW(e(*dHrj_K?9EP{jJ$~6Mh;4{QF zZXMPRW%A{?zkdbSaz{2RDHSYSF3I_bE1mnYZ{aE<5+3Fz4-X?;}HHYKE8YN zkngYvE8&Y_^E0jD#D0|w0|lSDW?Yj1Cqvw8h3us)2>N7CDxf;`W7M%A1Nw=Nie}^l zbxB%*UW>hjMkrJ^`=~n{Izb0|;cKT*L$HaD?<^`OI{(Gc=P2I8ngKFkT}C*0CPg67 z#LP**6E7Ny(&Ja`yJ>(0LXW?6;DDUqpO^|Hw0uAnpgWBn4*;xFSy2WYB0T5-O>q7G zShK~MXB9jD?^~^G!C?V#>(-e?=a*mJzGo3{HOzSY9f6}@KJ1wdQdASqLk@IR)Qwb+ z_f9${YQH-@dw6!jfz93RM$Ub^%VhtZQpxXr_wT>?&GkS2T6+1^eZAzu5;u^L}U?I;$%Z-!5>KGYRZVkq9j_u?klRo8(Fd zowgtL2WS$R9@TQ69vB77@S|(kT0_L$ zpdcclVy_6oKtRzPNG#&nsM;6YuUufUBa@Y2*%5?1`NW)A8SU{0h=1$^sQ-WX@YZ99 zfENE>z541)jR{8X19Y7J-HZjb#}^L%APUwr10E1m0|wTho?tz&_y$z5344NGOr&8@ z=!->-v-n9|Kyu_Dw)BDQH0rP1gI$NeTQ$%PkP1>hD_(Td)BaQazy5;-{cl6>ZqHWGT0LbAv)^FZbjs!A))%Dwvn67_ zF}gEMr-yT2@*wz%#L5D_Zk4!BhR*{pehcjH&Nx-w_Pe|h_hFHWrCxNr9$6|j=1 z24y^~DUX*P;OZXsS&%b{nGmS&I&^G8+dK6LK30@PO2=PSghG)DYl*-N%Y>mf2)yFz zb2jlj;79=vyqdeWh>H!!K6h{<)~l=0-wpEuPo>0&V-YH9HlrRr` zz>&8CIwofOj9mDT)%nrbJz=fY_-&FkpTYs%9^)V zUw!pzv&sN7ODo(^0ibP){1>)aVcY!Ir(9#jLunvmLl!6*w+ z1Jc2USdSrR?^8!*3K)o0-EJ@IRCZ!^IIx_-~JZ*|GQTc z`XX(86ZtX}t$f?cb)wS7mE_W-0lusw7Xrb2Xk(T6%!1uYzTXg=-VlCs@629+R_;;O zcM>qt84|_P7Phlql3__G1_EH{3oXfjStStd_In?WFVB`|zq_Yem7Jh&bA4_9{zl)< z;tV=bjP6qs1;I$`ESMxv42*rjPJjV)ftjZL1m#py`sMVqSs^96T|d3AnE{|c*a{d% zf_B@4oL~v$Iqc%S78NGAi*T(|2BE|(1ze8+*ezFecw z7p0wV5GxQZl`~ZRJor&gI0sR( zL*f}g(G39Wxq-ofcVrkJ`$loByJ($&-Xl(=HKfgA2ZWEyL_>2>4dGi=VInw^l$Hee z!$5x?;&rs0Z&|k&zqks`g#bs!4$p&Nr$B;X$2jN^4U5Gg0R*n#zzG7lQZ4Ov7{b2% zbE|-kuf(UP5wBOr$e9yt{U_`EA6>tJ4ah*?HbVgDfJy_F0Gh!JAXNUx3LxATf_ZU# zBT+z40FEH80)t4qeMc4Xl{<_(yT6g_kW_~Q-^UdQ*wJ2S+d&wsN)U}pEp}In7)#SJR4>Pj>z8(2JY+lx9Xk$Ejyg@iPMwQ(}#|F-LtD!&VE5jYfs|r^vs(5 z`vMV{_Z@RwI^Xax-=&Isz7g-Ug+>3JJ9jRsF18o?RoqpV7Z>L-K84z^%+u}q`SbbQ zPgpx9hokWMZ~^(Ucl^QgJWS>_rBAOctgmA4%|To`0dRJp6UlY$A+1mMv99+A=NI_z zUNR%%7e}s^6e<98#x2z(xo;SU-QwIe4V4spLU3Fua^T;`fCG6bT(-{ZAB7^{U#pmp zEW8*vjvYAdk>xvI%V8J}5dW{Y+vm@3)Q5blaJTcJp=gR#qxHarBC4XqmwC}JhFAn` zE)@Wrv}u^0Gi9d@_$!NnkFK2#Y>mGf0kMW(4bDKT!3u~fVCC!|CjluhRH3vFdOg%reRn(*g)pPEDBlJRxNd*4lb|4r6JH z?92WVyQ%Lls%pKP*Y?b|mzS3p=LGnOGWaSzCqLLL-Lin9H5`j!VduuaX5@0&daDre z%X5nmd$T4(JXij>@_nFi!9Xai3M;goKb$G@yMt7P{7;!2sM*v5M91}F|0;Jj3~NRM zi>UhivhXHgakDNI!hUyipzoaJj}Mpk_S@$o97_0yuM7KvM-jnultL^iSYlYOFYISN zKGHV81NvT00mS`krAB?e@MEzWd6f7mx*8UGYM1z=S?)jCdH*vR;MyvISHHUc>#siC z``|kPB!JNTA7gs)ufO=XIo_@`k*6NyBZYAvXnktQ#S&CFS8^<-E^%(ZR}Wip1sTnU z%iJ3rZ!U(b(qh>KxpM!F+U=u7uv%Vde#;O1Q1LI|&w~G7)B}9{mv`^};j7XmU(`TR zE38J)4As0?*q=IndxqK-tu|U~xsC`Q6w9|J3Q8g9{h-vY7C-7FU>~)<78C-Sa$jXY z+jC;~F3h7qS5`nAK$?D;-tey#2;K&Tj4?YmrO(P}5rX=l7rSa;S%$Hj#sJg*K?AC< zz{Bkf*M5yC^}L6~GY&n$>{MOM?R)neuI5}nrIbIpb?=^4H`awS(DOZB&hs!$yiXyp zTn4DX=la%Rgn7bjIZk=iS}c#|@!{aQp2cSU{rBJBy>sz=J&HD&-8IWsFm2Z77ZXX0FeUigXbf+?7I;Y4S8X$Zt9W>$}L9bLur7ww!8H@ z^}qw4-_YRg<^6it&e2hhY|x@#KJ-x_UBIti{q>Q(Lp#jg|CIzV8v1@l7K>pLFbZvf zT@eM)jR05v3IA?BuUj}nSOdC&Bk_?B|Ml{V=?h<_`kw>vLyV zd4^MVo5TH~hS>Tp<#xV#_2!);e*Z%G|1n^HZ{J+MI$f!3a=|q%3iliAtImFB=e&fK zJ4Jf!+A@+M_nux0fna&+@1r0N)_Ux#s83w9-Ki@sT}6QEV5djN<-^HAPFG>qg^l06 zWN`i5mcVG?Z1~AWB>7S9dzr#Y02WM}s+ zzHeD}JG+0X2{yrFA6gkHuH3NJMTkGYc=!k3NoNg;O9B@tEEdrhKrY<63a^@d zNd-tURKf1IDrHGra5oyX1lGgepy-268a%(WG@#1r0x_|z(p!YhxeDfmZxWLr97+>y z&bRslhhOcq!+8^=0z3n}a$3NvVIhdK^>gfR@!2e37is{nKfuJ`zD9j>40(U(7k&^)MS7( zzo82rg$9OQc&Vir$riDyRtQtXvH}yN~Qr34A%1MmKLhTr&xz>i_HYuYP{S z<<(yws|DB%-M5#%3`Z z3V@1l5yY1lak3hIRkAL%GhiNIznw#Yy8wo${jvwyJpPA2zT?P$_wmI$`v12&|3AKa zbNzVI_ZhV*R4qm)=`csFoD--`5mA2xUVWs|*;|>C56rhY}B&LlxV zqbS`?$9Xi=20-k}8^X_}peyA#7`53U7RM!$SSVhl7$fHKVN6_~E>8k!czH4`5C|UB zlAlP&v;Y?Ir(}Sc^#=D(XAprXA`X2zUwabx-d>q^?pP0SVNJkuEBozF=mo+S3#<$K z;Br@2>&sn=94)a461Miv*jr=fXX!RhcKZDBIKQaU(E=I;e^Dp=VuQsa(_hR3SAQ#` zySck<1O#ELq(FF$j%Tlx;#a^2jO#SS)7;;61(Xs%hjSn7HOCZCeU=IWE-(*PM03^y zsxa?&bbhO{z;XVaRt`qN0c0CoI(C62yc;-y+YuE`sCxwgtb?QXb^rqORqM-hiUI+y z_B-ZB9)O2M0Psfc0vIX)27)0}n@=Bp^-7{3e?=|-Slulz$M7)tN32RM}Z z10*nfr;#U$Q7AJUtKaQ{vRRN%u-bi9dDD=tjRRNP|Q0!Jajb1ACEX5-6SqLFGU%@!CWy>uL(R#0@mm21v&*lZxL!0+b!b zMjW4V*oo6d5JDLf3G+xhfA6}T+8D4KHzm>NgnYM!Q0@p`-+AXc|EK^^ScjH@GwnhR zuQd#J?$pUz=l9L!a@}iIFxSd&%w)kLbp&FzewK=bi|55i;i(Ni9B>ApJ>Uc*%@@K5 z?l~j!=l;Q5Y2(V-JA7SuvI=u{POE%POJN(A_T*}<{FR)%{+m>LdcS0x0N6MYnYOt$py;noO)=KRsQJarZ5zXJ@6; za%XZPi{|rg4B3CVoY)s4WXHP2{>iOdCm4L& z^)p4ecl=?GP(wjB3}z=P|CKJl1MLnVhEN%@c+1IuM{g_-%&m!k_UwY7zI;XyTdj{m zC|&vb{MWz!|16jB`uWm&{BX|efc667;`#X??oUto7OSE;Nc)#B*f z@$!q^=)$RZaXNo)=iwH1Uj7-?|0<38O}h0{@-sG`x9t940Qhr<>a1a4rr?cf@55yEP4*dz1>V#$y1gk*d^95{{( zcE=8aQs7emdm^~-JW9wC4wR1+UV%HH>Dgx~ znH%n`Ohv@gV^pCZ*d4P%=mZw7eGrg>P%wCRfg`IgN{qrrG~qA7lpvE3Y*jl@y z=>IB=h0qX6Jtq+I_}R;{!P(B;2b^|mDgYh*mH>GB@YE8(8Q1lQ_fMBc2GRZdC$qCNK2M@JB@x{BeImyw zD+}E31p!=mNk6Y;RtBzSXYsEO%N57`->xn6?9e{{+w%dy zUwhHcAne=|z?xhNayPng6il1|VgHtP?%us~=UnIURXVqNV^^iV930p|f#DmU-TrAk zs;V1`QT2H)7**BL2~hB(N8fhmc>Yl$OfAOz=9M)+iVO}+2ESSBgrt{yAeLOeTT7I`!Bu4^nA(H>eYz9wv zvRLmro#>~tj1%;d<8-a9I~I#OetBtGPVas5*xLK7+NBc$`|RYtCxDY%XRNiH==$l& z{nNl4q-C%~bMo-vtuJn?R5(!;Kzi_X`+d9)aUa^zi;FFW)#-fu?5Cf8+HTJ+gw8K$ z@1Y6&=hn<$?H`u&|KcHV|KF~k&(kZ5{$F2M4)|NI2RQqL37kLE!(Lh%`+LYd&+Q=s z-f(%b11PlXB!IP@#+D(j?DkQ5<+FU2eDuHOdvW~J@9xdlRDN;gGsl6~H+MJSn_RF3 za!0(+M|U7}TfvFXT>x~X8l)2Us20C3t%kHRn$K^DP1d%EIizJEA9*MP94UW(uJ4YG zftLbXw;tDT;xf1iNiaZ6p{)xH9A}4@>ziT0bfn zfCRiUxJdv*5*QtufMZFZF9zaVgu{Fx*!6RsR(L1R5cq$qMfLe*F)+J4dw>Y0f%VyZ zrZG$O`$?Zi&Jh=DAeFlaXFYDrUeW0vC%LeNAO{L zPZ;}s0*|)@Uy$@;O&7fYd46bMm{r?(i@XkE!P*s`s#ph}pU&62;)MQu+Ew;~2(@)I zb_vmTF(-2slkxpCD<<~yQB!!kZ?S%&%c&)TTW|uocv+y&Sy91%5AS{P<(FUHyXAMj zb>9s_tp;eT^%1SR6YBZ7I6Xi8<-@BFAO5lu)b>n6!ix(l0Disx_4Dr5^6#&@)6Mqz z`s#4?{MWy>PgrL7^{>zM5PEwNCi@f)S1CJr5q-=hnB#?Nob!MpT-?30Q$4=6__utb z{l2T}O3<6#uK9vxfP3ThBKPtjb+9iV7JkG|uG0CH)_5;TBm?t*{);MpBt*|sfG%ms zGH{;bB;g3DQ>=Yg-doHa5FGg<5fM@r3mACV2pE6~369PEblmQa1Oa+`RRzih&xdKJ zN4}5E&euce?|zv90!Sdh@Y(sU>XxUszW5-E5YRweM21Rz;kdnb>BBYxdBCH0rxJgt z<8LBs#g;{|k_C5%-2dYB&3QZX8ssSxj)!>K&_~2SvtM>GL8tDEDpGhUEp|ULI zRKtbNb$154iTZFwP|Wbr(F`J#2C44?Pz_Xhkf9ui-K5&|2v z;P*LsU2|utE!|1j&Lt5f+ff__^~=e0&)&;n2}W;Wyxgv<87KGNGQp^$0ATd?yt|6a z@gYvJAKLFZw#U=ucX8F8oN&P3SM+;S6ZT(xaSN89rGPI^ZiS2E1AKVzi+gqlZaTSd z4?2Io4g=~T{=c_x`h4Uq{CfSx;kr+jG6nslg$~dg54Lp$N=5`+V&LB7`^taH#j|=OloOOS<+eIc>v3fTom6 zF_iz?`|BO)A&swc&jRWjG+MZs@>wHckGO`FL?SybEzEEB7k-fwhSI@NgDd+gwwJ5} zVVk+!^!A``hvy8P`DX=yXMoW!K7upom!K-*>cUL1MFWZ8 zT-X1jJmCLZFVyX)x4!uDt1lma_0{JIfJoP_!6G07LjqX*KbQb62VLd=|40Cn|C!vCaj$40#1fPA7rq%*j z0)V=6^;!H^1D8L<`PJtT0w?$Lpc(39K&=PN-NVK=hT}*|h`6ewQ(a+h?8>)8V|*-j zK?k>kb*K~rsEjXf5P_`0.6.2') + + +def parse_args(): + parser = argparse.ArgumentParser(description='Train a detector') + parser.add_argument('config', help='train config file path') + parser.add_argument( + '--shape', + type=int, + nargs='+', + default=[512, 2048], + help='input image size') + parser.add_argument( + '--cfg-options', + nargs='+', + action=DictAction, + help='override some settings in the used config, the key-value pair ' + 'in xxx=yyy format will be merged into config file. If the value to ' + 'be overwritten is a list, it should be like key="[a,b]" or key=a,b ' + 'It also allows nested list/tuple values, e.g. key="[(a,b),(c,d)]" ' + 'Note that the quotation marks are necessary and that no white space ' + 'is allowed.') + parser.add_argument( + '--size-divisor', + type=int, + default=32, + help='Pad the input image, the minimum size that is divisible ' + 'by size_divisor, -1 means do not pad the image.') + args = parser.parse_args() + return args + +# dcnv3_flops(n=h*w, k=3*3, c=channels) +def dcnv3_flops(n, k, c): + return 5 * n * k * c + +def get_flops(model, input_shape): + flops, params = get_model_complexity_info(model, input_shape, as_strings=False) + + backbone = model.backbone + backbone_name = type(backbone).__name__ + _, H, W = input_shape + + temp = 0 + if 'Intern' in backbone_name: + depths = backbone.depths # [4, 4, 18, 4] + for idx, depth in enumerate(depths): + channels = backbone.channels * (2 ** idx) + h = H / (4 * (2 ** idx)) + w = W / (4 * (2 ** idx)) + temp += depth * dcnv3_flops(n=h*w, k=3*3, c=channels) + + flops = flops + temp + return flops_to_string(flops), params_to_string(params) + + +if __name__ == '__main__': + + args = parse_args() + + if len(args.shape) == 1: + h = w = args.shape[0] + elif len(args.shape) == 2: + h, w = args.shape + else: + raise ValueError('invalid input shape') + orig_shape = (3, h, w) + divisor = args.size_divisor + if divisor > 0: + h = int(np.ceil(h / divisor)) * divisor + w = int(np.ceil(w / divisor)) * divisor + + input_shape = (3, h, w) + + cfg = Config.fromfile(args.config) + if args.cfg_options is not None: + cfg.merge_from_dict(args.cfg_options) + + model = build_segmentor( + cfg.model, + train_cfg=cfg.get('train_cfg'), + test_cfg=cfg.get('test_cfg')) + + if torch.cuda.is_available(): + model.cuda() + model.eval() + if hasattr(model, 'forward_dummy'): + model.forward = model.forward_dummy + else: + raise NotImplementedError( + 'FLOPs counter is currently not currently supported with {}'. + format(model.__class__.__name__)) + + flops, params = get_flops(model, input_shape) + split_line = '=' * 30 + + if divisor > 0 and \ + input_shape != orig_shape: + print(f'{split_line}\nUse size divisor set input shape ' + f'from {orig_shape} to {input_shape}\n') + print(f'{split_line}\nInput shape: {input_shape}\n' + f'Flops: {flops}\nParams: {params}\n{split_line}') + print('!!!Please be cautious if you use the results in papers. ' + 'You may need to check if all ops are supported and verify that the ' + 'flops computation is correct.') diff --git a/segmentation/image_demo.py b/segmentation/image_demo.py new file mode 100644 index 0000000..649c453 --- /dev/null +++ b/segmentation/image_demo.py @@ -0,0 +1,70 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from argparse import ArgumentParser + +import mmcv + +import mmcv_custom # noqa: F401,F403 +import mmseg_custom # noqa: F401,F403 +from mmseg.apis import inference_segmentor, init_segmentor, show_result_pyplot +from mmseg.core.evaluation import get_palette +from mmcv.runner import load_checkpoint +from mmseg.core import get_classes +import cv2 +import os.path as osp +import os + + +def test_single_image(model, img_name, out_dir, color_palette, opacity): + result = inference_segmentor(model, img_name) + + # show the results + if hasattr(model, 'module'): + model = model.module + img = model.show_result(img_name, result, + palette=color_palette, + show=False, opacity=opacity) + + # save the results + mmcv.mkdir_or_exist(out_dir) + out_path = osp.join(out_dir, osp.basename(img_name)) + cv2.imwrite(out_path, img) + print(f"Result is save at {out_path}") + + +def main(): + parser = ArgumentParser() + parser.add_argument('img', help='Image file or a directory contains images') + parser.add_argument('config', help='Config file') + parser.add_argument('checkpoint', help='Checkpoint file') + parser.add_argument('--out', type=str, default="demo", help='out dir') + parser.add_argument( + '--device', default='cuda:0', help='Device used for inference') + parser.add_argument( + '--palette', + default='ade20k', + choices=['ade20k', 'cityscapes', 'cocostuff'], + help='Color palette used for segmentation map') + parser.add_argument( + '--opacity', + type=float, + default=0.5, + help='Opacity of painted segmentation map. In (0, 1] range.') + args = parser.parse_args() + + # build the model from a config file and a checkpoint file + model = init_segmentor(args.config, checkpoint=None, device=args.device) + checkpoint = load_checkpoint(model, args.checkpoint, map_location='cpu') + if 'CLASSES' in checkpoint.get('meta', {}): + model.CLASSES = checkpoint['meta']['CLASSES'] + else: + model.CLASSES = get_classes(args.palette) + + # check arg.img is directory of a single image. + if osp.isdir(args.img): + for img in os.listdir(args.img): + test_single_image(model, osp.join(args.img, img), args.out, get_palette(args.palette), args.opacity) + else: + test_single_image(model, args.img, args.out, get_palette(args.palette), args.opacity) + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/segmentation/mmcv_custom/__init__.py b/segmentation/mmcv_custom/__init__.py new file mode 100644 index 0000000..5d3efbe --- /dev/null +++ b/segmentation/mmcv_custom/__init__.py @@ -0,0 +1,11 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + +# -*- coding: utf-8 -*- +from .custom_layer_decay_optimizer_constructor import CustomLayerDecayOptimizerConstructor +from .layer_decay import LearningRateDecayOptimizerConstructor +from .layer_decay_vit import LayerDecayOptimizerConstructor_vit +__all__ = ['CustomLayerDecayOptimizerConstructor',] diff --git a/segmentation/mmcv_custom/custom_layer_decay_optimizer_constructor.py b/segmentation/mmcv_custom/custom_layer_decay_optimizer_constructor.py new file mode 100644 index 0000000..1cdf594 --- /dev/null +++ b/segmentation/mmcv_custom/custom_layer_decay_optimizer_constructor.py @@ -0,0 +1,157 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- +""" +Mostly copy-paste from BEiT library: +https://github.com/microsoft/unilm/blob/master/beit/semantic_segmentation/mmcv_custom/layer_decay_optimizer_constructor.py +""" + +import json + +from mmcv.runner import OPTIMIZER_BUILDERS, DefaultOptimizerConstructor +from mmcv.runner import get_dist_info +from mmseg.utils import get_root_logger + + +def get_num_layer_for_swin(var_name, num_max_layer, depths): + if var_name.startswith("backbone.patch_embed"): + return 0 + elif var_name.startswith('decode_head.mask_embed'): + return 0 + elif var_name.startswith('decode_head.cls_embed'): + return 0 + elif var_name.startswith('decode_head.level_embed'): + return 0 + elif var_name.startswith('decode_head.query_embed'): + return 0 + elif var_name.startswith('decode_head.query_feat'): + return 0 + if var_name.startswith("backbone.cb_modules.0.patch_embed"): + return 0 + elif "level_embeds" in var_name: + return 0 + elif var_name.startswith("backbone.layers") or var_name.startswith( + "backbone.levels"): + if var_name.split('.')[3] not in ['downsample', 'norm']: + stage_id = int(var_name.split('.')[2]) + layer_id = int(var_name.split('.')[4]) + # layers for Swin-Large: [2, 2, 18, 2] + if stage_id == 0: + return layer_id + 1 + elif stage_id == 1: + return layer_id + 1 + depths[0] + elif stage_id == 2: + return layer_id + 1 + depths[0] + depths[1] + else: + return layer_id + 1 + depths[0] + depths[1] + depths[2] + else: + stage_id = int(var_name.split('.')[2]) + if stage_id == 0: + return 1 + depths[0] + elif stage_id == 1: + return 1 + depths[0] + depths[1] + elif stage_id == 2: + return 1 + depths[0] + depths[1] + depths[2] + else: + return 1 + depths[0] + depths[1] + depths[2] + else: + return num_max_layer - 1 + + +@OPTIMIZER_BUILDERS.register_module() +class CustomLayerDecayOptimizerConstructor(DefaultOptimizerConstructor): + + def add_params(self, params, module, prefix='', is_dcn_module=None): + """Add all parameters of module to the params list. + The parameters of the given module will be added to the list of param + groups, with specific rules defined by paramwise_cfg. + Args: + params (list[dict]): A list of param groups, it will be modified + in place. + module (nn.Module): The module to be added. + prefix (str): The prefix of the module + is_dcn_module (int|float|None): If the current module is a + submodule of DCN, `is_dcn_module` will be passed to + control conv_offset layer's learning rate. Defaults to None. + """ + parameter_groups = {} + logger = get_root_logger() + logger.info(self.paramwise_cfg) + backbone_small_lr = self.paramwise_cfg.get('backbone_small_lr', False) + dino_head = self.paramwise_cfg.get('dino_head', False) + num_layers = self.paramwise_cfg.get('num_layers') + 2 + layer_decay_rate = self.paramwise_cfg.get('layer_decay_rate') + depths = self.paramwise_cfg.get('depths') + offset_lr_scale = self.paramwise_cfg.get('offset_lr_scale', 1.0) + + logger.info("Build CustomLayerDecayOptimizerConstructor %f - %d" % + (layer_decay_rate, num_layers)) + weight_decay = self.base_wd + + for name, param in module.named_parameters(): + if not param.requires_grad: + continue # frozen weights + if len(param.shape) == 1 or name.endswith(".bias") or \ + "relative_position" in name or \ + "norm" in name or\ + "sampling_offsets" in name: + group_name = "no_decay" + this_weight_decay = 0. + else: + group_name = "decay" + this_weight_decay = weight_decay + + layer_id = get_num_layer_for_swin(name, num_layers, depths) + if layer_id == num_layers - 1 and dino_head and \ + ("sampling_offsets" in name or "reference_points" in name): + group_name = "layer_%d_%s_0.1x" % (layer_id, group_name) + elif ("sampling_offsets" in name or "reference_points" in name) and "backbone" in name: + group_name = "layer_%d_%s_offset_lr_scale" % (layer_id, + group_name) + elif "offset_mask" in name and "offset_mask_dw" not in name: + group_name = "layer_%d_%s_offset_lr_scale" % (layer_id, + group_name) + else: + group_name = "layer_%d_%s" % (layer_id, group_name) + + if group_name not in parameter_groups: + scale = layer_decay_rate ** (num_layers - layer_id - 1) + if scale < 1 and backbone_small_lr == True: + scale = scale * 0.1 + if "0.1x" in group_name: + scale = scale * 0.1 + if "offset_lr_scale" in group_name: + scale = scale * offset_lr_scale + + parameter_groups[group_name] = { + "weight_decay": this_weight_decay, + "params": [], + "param_names": [], + "lr_scale": scale, + "group_name": group_name, + "lr": scale * self.base_lr, + } + + parameter_groups[group_name]["params"].append(param) + parameter_groups[group_name]["param_names"].append(name) + rank, _ = get_dist_info() + if rank == 0: + to_display = {} + for key in parameter_groups: + to_display[key] = { + "param_names": parameter_groups[key]["param_names"], + "lr_scale": parameter_groups[key]["lr_scale"], + "lr": parameter_groups[key]["lr"], + "weight_decay": parameter_groups[key]["weight_decay"], + } + logger.info("Param groups = %s" % json.dumps(to_display, indent=2)) + + # state_dict = module.state_dict() + # for group_name in parameter_groups: + # group = parameter_groups[group_name] + # for name in group["param_names"]: + # group["params"].append(state_dict[name]) + + params.extend(parameter_groups.values()) \ No newline at end of file diff --git a/segmentation/mmcv_custom/layer_decay.py b/segmentation/mmcv_custom/layer_decay.py new file mode 100644 index 0000000..1b36cc3 --- /dev/null +++ b/segmentation/mmcv_custom/layer_decay.py @@ -0,0 +1,123 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. + +# All rights reserved. + +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + + +import json +from mmcv.runner import OPTIMIZER_BUILDERS, DefaultOptimizerConstructor +from mmcv.runner import get_dist_info + + +def get_num_layer_layer_wise(var_name, num_max_layer=12): + + if var_name in ("backbone.cls_token", "backbone.mask_token", "backbone.pos_embed"): + return 0 + elif var_name.startswith("backbone.downsample_layers"): + stage_id = int(var_name.split('.')[2]) + if stage_id == 0: + layer_id = 0 + elif stage_id == 1: + layer_id = 2 + elif stage_id == 2: + layer_id = 3 + elif stage_id == 3: + layer_id = num_max_layer + return layer_id + elif var_name.startswith("backbone.stages"): + stage_id = int(var_name.split('.')[2]) + block_id = int(var_name.split('.')[3]) + if stage_id == 0: + layer_id = 1 + elif stage_id == 1: + layer_id = 2 + elif stage_id == 2: + layer_id = 3 + block_id // 3 + elif stage_id == 3: + layer_id = num_max_layer + return layer_id + else: + return num_max_layer + 1 + + +def get_num_layer_stage_wise(var_name, num_max_layer): + if var_name in ("backbone.cls_token", "backbone.mask_token", "backbone.pos_embed"): + return 0 + elif var_name.startswith("backbone.downsample_layers"): + return 0 + elif var_name.startswith("backbone.stages"): + stage_id = int(var_name.split('.')[2]) + return stage_id + 1 + else: + return num_max_layer - 1 + + +@OPTIMIZER_BUILDERS.register_module() +class LearningRateDecayOptimizerConstructor(DefaultOptimizerConstructor): + def add_params(self, params, module, prefix='', is_dcn_module=None): + """Add all parameters of module to the params list. + The parameters of the given module will be added to the list of param + groups, with specific rules defined by paramwise_cfg. + Args: + params (list[dict]): A list of param groups, it will be modified + in place. + module (nn.Module): The module to be added. + prefix (str): The prefix of the module + is_dcn_module (int|float|None): If the current module is a + submodule of DCN, `is_dcn_module` will be passed to + control conv_offset layer's learning rate. Defaults to None. + """ + parameter_groups = {} + print(self.paramwise_cfg) + num_layers = self.paramwise_cfg.get('num_layers') + 2 + decay_rate = self.paramwise_cfg.get('decay_rate') + decay_type = self.paramwise_cfg.get('decay_type', "layer_wise") + print("Build LearningRateDecayOptimizerConstructor %s %f - %d" % (decay_type, decay_rate, num_layers)) + weight_decay = self.base_wd + + for name, param in module.named_parameters(): + if not param.requires_grad: + continue # frozen weights + if len(param.shape) == 1 or name.endswith(".bias") or name in ('pos_embed', 'cls_token'): + group_name = "no_decay" + this_weight_decay = 0. + else: + group_name = "decay" + this_weight_decay = weight_decay + + if decay_type == "layer_wise": + layer_id = get_num_layer_layer_wise(name, self.paramwise_cfg.get('num_layers')) + elif decay_type == "stage_wise": + layer_id = get_num_layer_stage_wise(name, num_layers) + + group_name = "layer_%d_%s" % (layer_id, group_name) + + if group_name not in parameter_groups: + scale = decay_rate ** (num_layers - layer_id - 1) + + parameter_groups[group_name] = { + "weight_decay": this_weight_decay, + "params": [], + "param_names": [], + "lr_scale": scale, + "group_name": group_name, + "lr": scale * self.base_lr, + } + + parameter_groups[group_name]["params"].append(param) + parameter_groups[group_name]["param_names"].append(name) + rank, _ = get_dist_info() + if rank == 0: + to_display = {} + for key in parameter_groups: + to_display[key] = { + "param_names": parameter_groups[key]["param_names"], + "lr_scale": parameter_groups[key]["lr_scale"], + "lr": parameter_groups[key]["lr"], + "weight_decay": parameter_groups[key]["weight_decay"], + } + print("Param groups = %s" % json.dumps(to_display, indent=2)) + + params.extend(parameter_groups.values()) \ No newline at end of file diff --git a/segmentation/mmcv_custom/layer_decay_vit.py b/segmentation/mmcv_custom/layer_decay_vit.py new file mode 100644 index 0000000..bd0abcb --- /dev/null +++ b/segmentation/mmcv_custom/layer_decay_vit.py @@ -0,0 +1,105 @@ +import json +from mmcv.runner import OPTIMIZER_BUILDERS, DefaultOptimizerConstructor +from mmcv.runner import get_dist_info + + +def get_num_layer_for_vit(var_name, num_max_layer, layer_sep=None): + if var_name in ("backbone.cls_token", "backbone.mask_token", "backbone.pos_embed"): + return 0 + elif var_name.startswith("backbone.patch_embed"): + return 0 + elif var_name.startswith("backbone.blocks"): + layer_id = int(var_name.split('.')[2]) + return layer_id + 1 + elif var_name.startswith("backbone.layers"): + assert layer_sep is not None + split = var_name.split('.') + start_id = layer_sep[int(split[2])] + if split[3] == 'RC': + return start_id + return start_id + int(split[4]) + 1 + else: + return num_max_layer - 1 + + +@OPTIMIZER_BUILDERS.register_module() +class LayerDecayOptimizerConstructor_vit(DefaultOptimizerConstructor): + def add_params(self, params, module, prefix='', is_dcn_module=None): + """Add all parameters of module to the params list. + The parameters of the given module will be added to the list of param + groups, with specific rules defined by paramwise_cfg. + Args: + params (list[dict]): A list of param groups, it will be modified + in place. + module (nn.Module): The module to be added. + prefix (str): The prefix of the module + is_dcn_module (int|float|None): If the current module is a + submodule of DCN, `is_dcn_module` will be passed to + control conv_offset layer's learning rate. Defaults to None. + """ + # get param-wise options + + parameter_groups = {} + print(self.paramwise_cfg) + num_layers = self.paramwise_cfg.get('num_layers') + 2 + layer_sep = self.paramwise_cfg.get('layer_sep', None) + layer_decay_rate = self.paramwise_cfg.get('layer_decay_rate') + print("Build LayerDecayOptimizerConstructor %f - %d" % (layer_decay_rate, num_layers)) + weight_decay = self.base_wd + + custom_keys = self.paramwise_cfg.get('custom_keys', {}) + # first sort with alphabet order and then sort with reversed len of str + sorted_keys = sorted(custom_keys.keys()) + + for name, param in module.named_parameters(): + + if not param.requires_grad: + continue # frozen weights + + if len(param.shape) == 1 or name.endswith(".bias") or ('pos_embed' in name) or ('cls_token' in name) or ('rel_pos_' in name): + group_name = "no_decay" + this_weight_decay = 0. + else: + group_name = "decay" + this_weight_decay = weight_decay + + layer_id = get_num_layer_for_vit(name, num_layers, layer_sep) + group_name = "layer_%d_%s" % (layer_id, group_name) + + # if the parameter match one of the custom keys, ignore other rules + this_lr_multi = 1. + for key in sorted_keys: + if key in f'{name}': + lr_mult = custom_keys[key].get('lr_mult', 1.) + this_lr_multi = lr_mult + group_name = "%s_%s" % (group_name, key) + break + + if group_name not in parameter_groups: + scale = layer_decay_rate ** (num_layers - layer_id - 1) + + parameter_groups[group_name] = { + "weight_decay": this_weight_decay, + "params": [], + "param_names": [], + "lr_scale": scale, + "group_name": group_name, + "lr": scale * self.base_lr * this_lr_multi, + } + + parameter_groups[group_name]["params"].append(param) + parameter_groups[group_name]["param_names"].append(name) + + rank, _ = get_dist_info() + if rank == 0: + to_display = {} + for key in parameter_groups: + to_display[key] = { + "param_names": parameter_groups[key]["param_names"], + "lr_scale": parameter_groups[key]["lr_scale"], + "lr": parameter_groups[key]["lr"], + "weight_decay": parameter_groups[key]["weight_decay"], + } + print("Param groups = %s" % json.dumps(to_display, indent=2)) + + params.extend(parameter_groups.values()) \ No newline at end of file diff --git a/segmentation/mmseg_custom/__init__.py b/segmentation/mmseg_custom/__init__.py new file mode 100644 index 0000000..c2b1552 --- /dev/null +++ b/segmentation/mmseg_custom/__init__.py @@ -0,0 +1,9 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + +from .models import * # noqa: F401,F403 +from .datasets import * # noqa: F401,F403 +from .core import * # noqa: F401,F403 diff --git a/segmentation/mmseg_custom/core/__init__.py b/segmentation/mmseg_custom/core/__init__.py new file mode 100644 index 0000000..e6e4e90 --- /dev/null +++ b/segmentation/mmseg_custom/core/__init__.py @@ -0,0 +1,9 @@ +# Copyright (c) Shanghai AI Lab. All rights reserved. +from mmseg.core.evaluation import * # noqa: F401, F403 +from mmseg.core.seg import * # noqa: F401, F403 + +from .anchor import * # noqa: F401,F403 +from .box import * # noqa: F401,F403 +from .evaluation import * # noqa: F401,F403 +from .mask import * # noqa: F401,F403 +from .utils import * # noqa: F401, F403 diff --git a/segmentation/mmseg_custom/core/anchor/__init__.py b/segmentation/mmseg_custom/core/anchor/__init__.py new file mode 100644 index 0000000..3be0ee7 --- /dev/null +++ b/segmentation/mmseg_custom/core/anchor/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Shanghai AI Lab. All rights reserved. +from .point_generator import MlvlPointGenerator # noqa: F401,F403 diff --git a/segmentation/mmseg_custom/core/anchor/builder.py b/segmentation/mmseg_custom/core/anchor/builder.py new file mode 100644 index 0000000..ddb25ad --- /dev/null +++ b/segmentation/mmseg_custom/core/anchor/builder.py @@ -0,0 +1,19 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import warnings + +from mmcv.utils import Registry, build_from_cfg + +PRIOR_GENERATORS = Registry('Generator for anchors and points') + +ANCHOR_GENERATORS = PRIOR_GENERATORS + + +def build_prior_generator(cfg, default_args=None): + return build_from_cfg(cfg, PRIOR_GENERATORS, default_args) + + +def build_anchor_generator(cfg, default_args=None): + warnings.warn( + '``build_anchor_generator`` would be deprecated soon, please use ' + '``build_prior_generator`` ') + return build_prior_generator(cfg, default_args=default_args) diff --git a/segmentation/mmseg_custom/core/anchor/point_generator.py b/segmentation/mmseg_custom/core/anchor/point_generator.py new file mode 100644 index 0000000..34dd51a --- /dev/null +++ b/segmentation/mmseg_custom/core/anchor/point_generator.py @@ -0,0 +1,260 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import numpy as np +import torch +from torch.nn.modules.utils import _pair + +from .builder import PRIOR_GENERATORS + + +@PRIOR_GENERATORS.register_module() +class PointGenerator: + def _meshgrid(self, x, y, row_major=True): + xx = x.repeat(len(y)) + yy = y.view(-1, 1).repeat(1, len(x)).view(-1) + if row_major: + return xx, yy + else: + return yy, xx + + def grid_points(self, featmap_size, stride=16, device='cuda'): + feat_h, feat_w = featmap_size + shift_x = torch.arange(0., feat_w, device=device) * stride + shift_y = torch.arange(0., feat_h, device=device) * stride + shift_xx, shift_yy = self._meshgrid(shift_x, shift_y) + stride = shift_x.new_full((shift_xx.shape[0], ), stride) + shifts = torch.stack([shift_xx, shift_yy, stride], dim=-1) + all_points = shifts.to(device) + return all_points + + def valid_flags(self, featmap_size, valid_size, device='cuda'): + feat_h, feat_w = featmap_size + valid_h, valid_w = valid_size + assert valid_h <= feat_h and valid_w <= feat_w + valid_x = torch.zeros(feat_w, dtype=torch.bool, device=device) + valid_y = torch.zeros(feat_h, dtype=torch.bool, device=device) + valid_x[:valid_w] = 1 + valid_y[:valid_h] = 1 + valid_xx, valid_yy = self._meshgrid(valid_x, valid_y) + valid = valid_xx & valid_yy + return valid + + +@PRIOR_GENERATORS.register_module() +class MlvlPointGenerator: + """Standard points generator for multi-level (Mlvl) feature maps in 2D + points-based detectors. + + Args: + strides (list[int] | list[tuple[int, int]]): Strides of anchors + in multiple feature levels in order (w, h). + offset (float): The offset of points, the value is normalized with + corresponding stride. Defaults to 0.5. + """ + def __init__(self, strides, offset=0.5): + self.strides = [_pair(stride) for stride in strides] + self.offset = offset + + @property + def num_levels(self): + """int: number of feature levels that the generator will be applied""" + return len(self.strides) + + @property + def num_base_priors(self): + """list[int]: The number of priors (points) at a point + on the feature grid""" + return [1 for _ in range(len(self.strides))] + + def _meshgrid(self, x, y, row_major=True): + yy, xx = torch.meshgrid(y, x) + if row_major: + # warning .flatten() would cause error in ONNX exporting + # have to use reshape here + return xx.reshape(-1), yy.reshape(-1) + + else: + return yy.reshape(-1), xx.reshape(-1) + + def grid_priors(self, + featmap_sizes, + dtype=torch.float32, + device='cuda', + with_stride=False): + """Generate grid points of multiple feature levels. + + Args: + featmap_sizes (list[tuple]): List of feature map sizes in + multiple feature levels, each size arrange as + as (h, w). + dtype (:obj:`dtype`): Dtype of priors. Default: torch.float32. + device (str): The device where the anchors will be put on. + with_stride (bool): Whether to concatenate the stride to + the last dimension of points. + + Return: + list[torch.Tensor]: Points of multiple feature levels. + The sizes of each tensor should be (N, 2) when with stride is + ``False``, where N = width * height, width and height + are the sizes of the corresponding feature level, + and the last dimension 2 represent (coord_x, coord_y), + otherwise the shape should be (N, 4), + and the last dimension 4 represent + (coord_x, coord_y, stride_w, stride_h). + """ + + assert self.num_levels == len(featmap_sizes) + multi_level_priors = [] + for i in range(self.num_levels): + priors = self.single_level_grid_priors(featmap_sizes[i], + level_idx=i, + dtype=dtype, + device=device, + with_stride=with_stride) + multi_level_priors.append(priors) + return multi_level_priors + + def single_level_grid_priors(self, + featmap_size, + level_idx, + dtype=torch.float32, + device='cuda', + with_stride=False): + """Generate grid Points of a single level. + + Note: + This function is usually called by method ``self.grid_priors``. + + Args: + featmap_size (tuple[int]): Size of the feature maps, arrange as + (h, w). + level_idx (int): The index of corresponding feature map level. + dtype (:obj:`dtype`): Dtype of priors. Default: torch.float32. + device (str, optional): The device the tensor will be put on. + Defaults to 'cuda'. + with_stride (bool): Concatenate the stride to the last dimension + of points. + + Return: + Tensor: Points of single feature levels. + The shape of tensor should be (N, 2) when with stride is + ``False``, where N = width * height, width and height + are the sizes of the corresponding feature level, + and the last dimension 2 represent (coord_x, coord_y), + otherwise the shape should be (N, 4), + and the last dimension 4 represent + (coord_x, coord_y, stride_w, stride_h). + """ + feat_h, feat_w = featmap_size + stride_w, stride_h = self.strides[level_idx] + shift_x = (torch.arange(0, feat_w, device=device) + + self.offset) * stride_w + # keep featmap_size as Tensor instead of int, so that we + # can convert to ONNX correctly + shift_x = shift_x.to(dtype) + + shift_y = (torch.arange(0, feat_h, device=device) + + self.offset) * stride_h + # keep featmap_size as Tensor instead of int, so that we + # can convert to ONNX correctly + shift_y = shift_y.to(dtype) + shift_xx, shift_yy = self._meshgrid(shift_x, shift_y) + if not with_stride: + shifts = torch.stack([shift_xx, shift_yy], dim=-1) + else: + # use `shape[0]` instead of `len(shift_xx)` for ONNX export + stride_w = shift_xx.new_full((shift_xx.shape[0], ), + stride_w).to(dtype) + stride_h = shift_xx.new_full((shift_yy.shape[0], ), + stride_h).to(dtype) + shifts = torch.stack([shift_xx, shift_yy, stride_w, stride_h], + dim=-1) + all_points = shifts.to(device) + return all_points + + def valid_flags(self, featmap_sizes, pad_shape, device='cuda'): + """Generate valid flags of points of multiple feature levels. + + Args: + featmap_sizes (list(tuple)): List of feature map sizes in + multiple feature levels, each size arrange as + as (h, w). + pad_shape (tuple(int)): The padded shape of the image, + arrange as (h, w). + device (str): The device where the anchors will be put on. + + Return: + list(torch.Tensor): Valid flags of points of multiple levels. + """ + assert self.num_levels == len(featmap_sizes) + multi_level_flags = [] + for i in range(self.num_levels): + point_stride = self.strides[i] + feat_h, feat_w = featmap_sizes[i] + h, w = pad_shape[:2] + valid_feat_h = min(int(np.ceil(h / point_stride[1])), feat_h) + valid_feat_w = min(int(np.ceil(w / point_stride[0])), feat_w) + flags = self.single_level_valid_flags((feat_h, feat_w), + (valid_feat_h, valid_feat_w), + device=device) + multi_level_flags.append(flags) + return multi_level_flags + + def single_level_valid_flags(self, + featmap_size, + valid_size, + device='cuda'): + """Generate the valid flags of points of a single feature map. + + Args: + featmap_size (tuple[int]): The size of feature maps, arrange as + as (h, w). + valid_size (tuple[int]): The valid size of the feature maps. + The size arrange as as (h, w). + device (str, optional): The device where the flags will be put on. + Defaults to 'cuda'. + + Returns: + torch.Tensor: The valid flags of each points in a single level \ + feature map. + """ + feat_h, feat_w = featmap_size + valid_h, valid_w = valid_size + assert valid_h <= feat_h and valid_w <= feat_w + valid_x = torch.zeros(feat_w, dtype=torch.bool, device=device) + valid_y = torch.zeros(feat_h, dtype=torch.bool, device=device) + valid_x[:valid_w] = 1 + valid_y[:valid_h] = 1 + valid_xx, valid_yy = self._meshgrid(valid_x, valid_y) + valid = valid_xx & valid_yy + return valid + + def sparse_priors(self, + prior_idxs, + featmap_size, + level_idx, + dtype=torch.float32, + device='cuda'): + """Generate sparse points according to the ``prior_idxs``. + + Args: + prior_idxs (Tensor): The index of corresponding anchors + in the feature map. + featmap_size (tuple[int]): feature map size arrange as (w, h). + level_idx (int): The level index of corresponding feature + map. + dtype (obj:`torch.dtype`): Date type of points. Defaults to + ``torch.float32``. + device (obj:`torch.device`): The device where the points is + located. + Returns: + Tensor: Anchor with shape (N, 2), N should be equal to + the length of ``prior_idxs``. And last dimension + 2 represent (coord_x, coord_y). + """ + height, width = featmap_size + x = (prior_idxs % width + self.offset) * self.strides[level_idx][0] + y = ((prior_idxs // width) % height + + self.offset) * self.strides[level_idx][1] + prioris = torch.stack([x, y], 1).to(dtype) + prioris = prioris.to(device) + return prioris diff --git a/segmentation/mmseg_custom/core/box/__init__.py b/segmentation/mmseg_custom/core/box/__init__.py new file mode 100644 index 0000000..a6a127f --- /dev/null +++ b/segmentation/mmseg_custom/core/box/__init__.py @@ -0,0 +1,3 @@ +# Copyright (c) Shanghai AI Lab. All rights reserved. +from .builder import * # noqa: F401,F403 +from .samplers import MaskPseudoSampler # noqa: F401,F403 diff --git a/segmentation/mmseg_custom/core/box/builder.py b/segmentation/mmseg_custom/core/box/builder.py new file mode 100644 index 0000000..af4b8a8 --- /dev/null +++ b/segmentation/mmseg_custom/core/box/builder.py @@ -0,0 +1,15 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from mmcv.utils import Registry, build_from_cfg + +BBOX_SAMPLERS = Registry('bbox_sampler') +BBOX_CODERS = Registry('bbox_coder') + + +def build_sampler(cfg, **default_args): + """Builder of box sampler.""" + return build_from_cfg(cfg, BBOX_SAMPLERS, default_args) + + +def build_bbox_coder(cfg, **default_args): + """Builder of box coder.""" + return build_from_cfg(cfg, BBOX_CODERS, default_args) diff --git a/segmentation/mmseg_custom/core/box/samplers/__init__.py b/segmentation/mmseg_custom/core/box/samplers/__init__.py new file mode 100644 index 0000000..d30f999 --- /dev/null +++ b/segmentation/mmseg_custom/core/box/samplers/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Shanghai AI Lab. All rights reserved. +from .mask_pseudo_sampler import MaskPseudoSampler # noqa: F401,F403 diff --git a/segmentation/mmseg_custom/core/box/samplers/base_sampler.py b/segmentation/mmseg_custom/core/box/samplers/base_sampler.py new file mode 100644 index 0000000..dee6497 --- /dev/null +++ b/segmentation/mmseg_custom/core/box/samplers/base_sampler.py @@ -0,0 +1,105 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from abc import ABCMeta, abstractmethod + +import torch + +from .sampling_result import SamplingResult + + +class BaseSampler(metaclass=ABCMeta): + """Base class of samplers.""" + def __init__(self, + num, + pos_fraction, + neg_pos_ub=-1, + add_gt_as_proposals=True, + **kwargs): + self.num = num + self.pos_fraction = pos_fraction + self.neg_pos_ub = neg_pos_ub + self.add_gt_as_proposals = add_gt_as_proposals + self.pos_sampler = self + self.neg_sampler = self + + @abstractmethod + def _sample_pos(self, assign_result, num_expected, **kwargs): + """Sample positive samples.""" + pass + + @abstractmethod + def _sample_neg(self, assign_result, num_expected, **kwargs): + """Sample negative samples.""" + pass + + def sample(self, + assign_result, + bboxes, + gt_bboxes, + gt_labels=None, + **kwargs): + """Sample positive and negative bboxes. + + This is a simple implementation of bbox sampling given candidates, + assigning results and ground truth bboxes. + + Args: + assign_result (:obj:`AssignResult`): Bbox assigning results. + bboxes (Tensor): Boxes to be sampled from. + gt_bboxes (Tensor): Ground truth bboxes. + gt_labels (Tensor, optional): Class labels of ground truth bboxes. + + Returns: + :obj:`SamplingResult`: Sampling result. + + Example: + >>> from mmdet.core.bbox import RandomSampler + >>> from mmdet.core.bbox import AssignResult + >>> from mmdet.core.bbox.demodata import ensure_rng, random_boxes + >>> rng = ensure_rng(None) + >>> assign_result = AssignResult.random(rng=rng) + >>> bboxes = random_boxes(assign_result.num_preds, rng=rng) + >>> gt_bboxes = random_boxes(assign_result.num_gts, rng=rng) + >>> gt_labels = None + >>> self = RandomSampler(num=32, pos_fraction=0.5, neg_pos_ub=-1, + >>> add_gt_as_proposals=False) + >>> self = self.sample(assign_result, bboxes, gt_bboxes, gt_labels) + """ + if len(bboxes.shape) < 2: + bboxes = bboxes[None, :] + + bboxes = bboxes[:, :4] + + gt_flags = bboxes.new_zeros((bboxes.shape[0], ), dtype=torch.uint8) + if self.add_gt_as_proposals and len(gt_bboxes) > 0: + if gt_labels is None: + raise ValueError( + 'gt_labels must be given when add_gt_as_proposals is True') + bboxes = torch.cat([gt_bboxes, bboxes], dim=0) + assign_result.add_gt_(gt_labels) + gt_ones = bboxes.new_ones(gt_bboxes.shape[0], dtype=torch.uint8) + gt_flags = torch.cat([gt_ones, gt_flags]) + + num_expected_pos = int(self.num * self.pos_fraction) + pos_inds = self.pos_sampler._sample_pos(assign_result, + num_expected_pos, + bboxes=bboxes, + **kwargs) + # We found that sampled indices have duplicated items occasionally. + # (may be a bug of PyTorch) + pos_inds = pos_inds.unique() + num_sampled_pos = pos_inds.numel() + num_expected_neg = self.num - num_sampled_pos + if self.neg_pos_ub >= 0: + _pos = max(1, num_sampled_pos) + neg_upper_bound = int(self.neg_pos_ub * _pos) + if num_expected_neg > neg_upper_bound: + num_expected_neg = neg_upper_bound + neg_inds = self.neg_sampler._sample_neg(assign_result, + num_expected_neg, + bboxes=bboxes, + **kwargs) + neg_inds = neg_inds.unique() + + sampling_result = SamplingResult(pos_inds, neg_inds, bboxes, gt_bboxes, + assign_result, gt_flags) + return sampling_result diff --git a/segmentation/mmseg_custom/core/box/samplers/mask_pseudo_sampler.py b/segmentation/mmseg_custom/core/box/samplers/mask_pseudo_sampler.py new file mode 100644 index 0000000..501a201 --- /dev/null +++ b/segmentation/mmseg_custom/core/box/samplers/mask_pseudo_sampler.py @@ -0,0 +1,43 @@ +# Copyright (c) OpenMMLab. All rights reserved. +"""copy from +https://github.com/ZwwWayne/K-Net/blob/main/knet/det/mask_pseudo_sampler.py.""" + +import torch + +from ..builder import BBOX_SAMPLERS +from .base_sampler import BaseSampler +from .mask_sampling_result import MaskSamplingResult + + +@BBOX_SAMPLERS.register_module() +class MaskPseudoSampler(BaseSampler): + """A pseudo sampler that does not do sampling actually.""" + def __init__(self, **kwargs): + pass + + def _sample_pos(self, **kwargs): + """Sample positive samples.""" + raise NotImplementedError + + def _sample_neg(self, **kwargs): + """Sample negative samples.""" + raise NotImplementedError + + def sample(self, assign_result, masks, gt_masks, **kwargs): + """Directly returns the positive and negative indices of samples. + + Args: + assign_result (:obj:`AssignResult`): Assigned results + masks (torch.Tensor): Bounding boxes + gt_masks (torch.Tensor): Ground truth boxes + Returns: + :obj:`SamplingResult`: sampler results + """ + pos_inds = torch.nonzero(assign_result.gt_inds > 0, + as_tuple=False).squeeze(-1).unique() + neg_inds = torch.nonzero(assign_result.gt_inds == 0, + as_tuple=False).squeeze(-1).unique() + gt_flags = masks.new_zeros(masks.shape[0], dtype=torch.uint8) + sampling_result = MaskSamplingResult(pos_inds, neg_inds, masks, + gt_masks, assign_result, gt_flags) + return sampling_result diff --git a/segmentation/mmseg_custom/core/box/samplers/mask_sampling_result.py b/segmentation/mmseg_custom/core/box/samplers/mask_sampling_result.py new file mode 100644 index 0000000..f6c500b --- /dev/null +++ b/segmentation/mmseg_custom/core/box/samplers/mask_sampling_result.py @@ -0,0 +1,59 @@ +# Copyright (c) OpenMMLab. All rights reserved. +"""copy from +https://github.com/ZwwWayne/K-Net/blob/main/knet/det/mask_pseudo_sampler.py.""" + +import torch + +from .sampling_result import SamplingResult + + +class MaskSamplingResult(SamplingResult): + """Mask sampling result.""" + def __init__(self, pos_inds, neg_inds, masks, gt_masks, assign_result, + gt_flags): + self.pos_inds = pos_inds + self.neg_inds = neg_inds + self.pos_masks = masks[pos_inds] + self.neg_masks = masks[neg_inds] + self.pos_is_gt = gt_flags[pos_inds] + + self.num_gts = gt_masks.shape[0] + self.pos_assigned_gt_inds = assign_result.gt_inds[pos_inds] - 1 + + if gt_masks.numel() == 0: + # hack for index error case + assert self.pos_assigned_gt_inds.numel() == 0 + self.pos_gt_masks = torch.empty_like(gt_masks) + else: + self.pos_gt_masks = gt_masks[self.pos_assigned_gt_inds, :] + + if assign_result.labels is not None: + self.pos_gt_labels = assign_result.labels[pos_inds] + else: + self.pos_gt_labels = None + + @property + def masks(self): + """torch.Tensor: concatenated positive and negative boxes""" + return torch.cat([self.pos_masks, self.neg_masks]) + + def __nice__(self): + data = self.info.copy() + data['pos_masks'] = data.pop('pos_masks').shape + data['neg_masks'] = data.pop('neg_masks').shape + parts = [f"'{k}': {v!r}" for k, v in sorted(data.items())] + body = ' ' + ',\n '.join(parts) + return '{\n' + body + '\n}' + + @property + def info(self): + """Returns a dictionary of info about the object.""" + return { + 'pos_inds': self.pos_inds, + 'neg_inds': self.neg_inds, + 'pos_masks': self.pos_masks, + 'neg_masks': self.neg_masks, + 'pos_is_gt': self.pos_is_gt, + 'num_gts': self.num_gts, + 'pos_assigned_gt_inds': self.pos_assigned_gt_inds, + } diff --git a/segmentation/mmseg_custom/core/box/samplers/sampling_result.py b/segmentation/mmseg_custom/core/box/samplers/sampling_result.py new file mode 100644 index 0000000..d1ac578 --- /dev/null +++ b/segmentation/mmseg_custom/core/box/samplers/sampling_result.py @@ -0,0 +1,150 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +from mmdet.utils import util_mixins + + +class SamplingResult(util_mixins.NiceRepr): + """Bbox sampling result. + + Example: + >>> # xdoctest: +IGNORE_WANT + >>> from mmdet.core.bbox.samplers.sampling_result import * # NOQA + >>> self = SamplingResult.random(rng=10) + >>> print(f'self = {self}') + self = + """ + def __init__(self, pos_inds, neg_inds, bboxes, gt_bboxes, assign_result, + gt_flags): + self.pos_inds = pos_inds + self.neg_inds = neg_inds + self.pos_bboxes = bboxes[pos_inds] + self.neg_bboxes = bboxes[neg_inds] + self.pos_is_gt = gt_flags[pos_inds] + + self.num_gts = gt_bboxes.shape[0] + self.pos_assigned_gt_inds = assign_result.gt_inds[pos_inds] - 1 + + if gt_bboxes.numel() == 0: + # hack for index error case + assert self.pos_assigned_gt_inds.numel() == 0 + self.pos_gt_bboxes = torch.empty_like(gt_bboxes).view(-1, 4) + else: + if len(gt_bboxes.shape) < 2: + gt_bboxes = gt_bboxes.view(-1, 4) + + self.pos_gt_bboxes = gt_bboxes[self.pos_assigned_gt_inds.long(), :] + + if assign_result.labels is not None: + self.pos_gt_labels = assign_result.labels[pos_inds] + else: + self.pos_gt_labels = None + + @property + def bboxes(self): + """torch.Tensor: concatenated positive and negative boxes""" + return torch.cat([self.pos_bboxes, self.neg_bboxes]) + + def to(self, device): + """Change the device of the data inplace. + + Example: + >>> self = SamplingResult.random() + >>> print(f'self = {self.to(None)}') + >>> # xdoctest: +REQUIRES(--gpu) + >>> print(f'self = {self.to(0)}') + """ + _dict = self.__dict__ + for key, value in _dict.items(): + if isinstance(value, torch.Tensor): + _dict[key] = value.to(device) + return self + + def __nice__(self): + data = self.info.copy() + data['pos_bboxes'] = data.pop('pos_bboxes').shape + data['neg_bboxes'] = data.pop('neg_bboxes').shape + parts = [f"'{k}': {v!r}" for k, v in sorted(data.items())] + body = ' ' + ',\n '.join(parts) + return '{\n' + body + '\n}' + + @property + def info(self): + """Returns a dictionary of info about the object.""" + return { + 'pos_inds': self.pos_inds, + 'neg_inds': self.neg_inds, + 'pos_bboxes': self.pos_bboxes, + 'neg_bboxes': self.neg_bboxes, + 'pos_is_gt': self.pos_is_gt, + 'num_gts': self.num_gts, + 'pos_assigned_gt_inds': self.pos_assigned_gt_inds, + } + + @classmethod + def random(cls, rng=None, **kwargs): + """ + Args: + rng (None | int | numpy.random.RandomState): seed or state. + kwargs (keyword arguments): + - num_preds: number of predicted boxes + - num_gts: number of true boxes + - p_ignore (float): probability of a predicted box assigned to \ + an ignored truth. + - p_assigned (float): probability of a predicted box not being \ + assigned. + - p_use_label (float | bool): with labels or not. + + Returns: + :obj:`SamplingResult`: Randomly generated sampling result. + + Example: + >>> from mmdet.core.bbox.samplers.sampling_result import * # NOQA + >>> self = SamplingResult.random() + >>> print(self.__dict__) + """ + from mmdet.core.bbox import demodata + from mmdet.core.bbox.assigners.assign_result import AssignResult + from mmdet.core.bbox.samplers.random_sampler import RandomSampler + rng = demodata.ensure_rng(rng) + + # make probabalistic? + num = 32 + pos_fraction = 0.5 + neg_pos_ub = -1 + + assign_result = AssignResult.random(rng=rng, **kwargs) + + # Note we could just compute an assignment + bboxes = demodata.random_boxes(assign_result.num_preds, rng=rng) + gt_bboxes = demodata.random_boxes(assign_result.num_gts, rng=rng) + + if rng.rand() > 0.2: + # sometimes algorithms squeeze their data, be robust to that + gt_bboxes = gt_bboxes.squeeze() + bboxes = bboxes.squeeze() + + if assign_result.labels is None: + gt_labels = None + else: + gt_labels = None # todo + + if gt_labels is None: + add_gt_as_proposals = False + else: + add_gt_as_proposals = True # make probabalistic? + + sampler = RandomSampler(num, + pos_fraction, + neg_pos_ub=neg_pos_ub, + add_gt_as_proposals=add_gt_as_proposals, + rng=rng) + self = sampler.sample(assign_result, bboxes, gt_bboxes, gt_labels) + return self diff --git a/segmentation/mmseg_custom/core/evaluation/__init__.py b/segmentation/mmseg_custom/core/evaluation/__init__.py new file mode 100644 index 0000000..3f02829 --- /dev/null +++ b/segmentation/mmseg_custom/core/evaluation/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Shanghai AI Lab. All rights reserved. +from .panoptic_utils import INSTANCE_OFFSET # noqa: F401,F403 diff --git a/segmentation/mmseg_custom/core/evaluation/panoptic_utils.py b/segmentation/mmseg_custom/core/evaluation/panoptic_utils.py new file mode 100644 index 0000000..10c9ad9 --- /dev/null +++ b/segmentation/mmseg_custom/core/evaluation/panoptic_utils.py @@ -0,0 +1,6 @@ +# Copyright (c) OpenMMLab. All rights reserved. +# A custom value to distinguish instance ID and category ID; need to +# be greater than the number of categories. +# For a pixel in the panoptic result map: +# pan_id = ins_id * INSTANCE_OFFSET + cat_id +INSTANCE_OFFSET = 1000 diff --git a/segmentation/mmseg_custom/core/mask/__init__.py b/segmentation/mmseg_custom/core/mask/__init__.py new file mode 100644 index 0000000..d226c23 --- /dev/null +++ b/segmentation/mmseg_custom/core/mask/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Shanghai AI Lab. All rights reserved. +from .utils import mask2bbox # noqa: F401,F403 diff --git a/segmentation/mmseg_custom/core/mask/utils.py b/segmentation/mmseg_custom/core/mask/utils.py new file mode 100644 index 0000000..90544b3 --- /dev/null +++ b/segmentation/mmseg_custom/core/mask/utils.py @@ -0,0 +1,89 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import mmcv +import numpy as np +import pycocotools.mask as mask_util +import torch + + +def split_combined_polys(polys, poly_lens, polys_per_mask): + """Split the combined 1-D polys into masks. + + A mask is represented as a list of polys, and a poly is represented as + a 1-D array. In dataset, all masks are concatenated into a single 1-D + tensor. Here we need to split the tensor into original representations. + + Args: + polys (list): a list (length = image num) of 1-D tensors + poly_lens (list): a list (length = image num) of poly length + polys_per_mask (list): a list (length = image num) of poly number + of each mask + + Returns: + list: a list (length = image num) of list (length = mask num) of \ + list (length = poly num) of numpy array. + """ + mask_polys_list = [] + for img_id in range(len(polys)): + polys_single = polys[img_id] + polys_lens_single = poly_lens[img_id].tolist() + polys_per_mask_single = polys_per_mask[img_id].tolist() + + split_polys = mmcv.slice_list(polys_single, polys_lens_single) + mask_polys = mmcv.slice_list(split_polys, polys_per_mask_single) + mask_polys_list.append(mask_polys) + return mask_polys_list + + +# TODO: move this function to more proper place +def encode_mask_results(mask_results): + """Encode bitmap mask to RLE code. + + Args: + mask_results (list | tuple[list]): bitmap mask results. + In mask scoring rcnn, mask_results is a tuple of (segm_results, + segm_cls_score). + + Returns: + list | tuple: RLE encoded mask. + """ + if isinstance(mask_results, tuple): # mask scoring + cls_segms, cls_mask_scores = mask_results + else: + cls_segms = mask_results + num_classes = len(cls_segms) + encoded_mask_results = [[] for _ in range(num_classes)] + for i in range(len(cls_segms)): + for cls_segm in cls_segms[i]: + encoded_mask_results[i].append( + mask_util.encode( + np.array( + cls_segm[:, :, np.newaxis], order='F', + dtype='uint8'))[0]) # encoded with RLE + if isinstance(mask_results, tuple): + return encoded_mask_results, cls_mask_scores + else: + return encoded_mask_results + + +def mask2bbox(masks): + """Obtain tight bounding boxes of binary masks. + + Args: + masks (Tensor): Binary mask of shape (n, h, w). + + Returns: + Tensor: Bboxe with shape (n, 4) of \ + positive region in binary mask. + """ + N = masks.shape[0] + bboxes = masks.new_zeros((N, 4), dtype=torch.float32) + x_any = torch.any(masks, dim=1) + y_any = torch.any(masks, dim=2) + for i in range(N): + x = torch.where(x_any[i, :])[0] + y = torch.where(y_any[i, :])[0] + if len(x) > 0 and len(y) > 0: + bboxes[i, :] = bboxes.new_tensor( + [x[0], y[0], x[-1] + 1, y[-1] + 1]) + + return bboxes diff --git a/segmentation/mmseg_custom/core/utils/__init__.py b/segmentation/mmseg_custom/core/utils/__init__.py new file mode 100644 index 0000000..26ff24d --- /dev/null +++ b/segmentation/mmseg_custom/core/utils/__init__.py @@ -0,0 +1,9 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from .dist_utils import (DistOptimizerHook, all_reduce_dict, allreduce_grads, + reduce_mean) +from .misc import add_prefix, multi_apply + +__all__ = [ + 'add_prefix', 'multi_apply', 'DistOptimizerHook', 'allreduce_grads', + 'all_reduce_dict', 'reduce_mean' +] diff --git a/segmentation/mmseg_custom/core/utils/dist_utils.py b/segmentation/mmseg_custom/core/utils/dist_utils.py new file mode 100644 index 0000000..88e519f --- /dev/null +++ b/segmentation/mmseg_custom/core/utils/dist_utils.py @@ -0,0 +1,148 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import functools +import pickle +import warnings +from collections import OrderedDict + +import torch +import torch.distributed as dist +from mmcv.runner import OptimizerHook, get_dist_info +from torch._utils import (_flatten_dense_tensors, _take_tensors, + _unflatten_dense_tensors) + + +def _allreduce_coalesced(tensors, world_size, bucket_size_mb=-1): + if bucket_size_mb > 0: + bucket_size_bytes = bucket_size_mb * 1024 * 1024 + buckets = _take_tensors(tensors, bucket_size_bytes) + else: + buckets = OrderedDict() + for tensor in tensors: + tp = tensor.type() + if tp not in buckets: + buckets[tp] = [] + buckets[tp].append(tensor) + buckets = buckets.values() + + for bucket in buckets: + flat_tensors = _flatten_dense_tensors(bucket) + dist.all_reduce(flat_tensors) + flat_tensors.div_(world_size) + for tensor, synced in zip( + bucket, _unflatten_dense_tensors(flat_tensors, bucket)): + tensor.copy_(synced) + + +def allreduce_grads(params, coalesce=True, bucket_size_mb=-1): + """Allreduce gradients. + + Args: + params (list[torch.Parameters]): List of parameters of a model + coalesce (bool, optional): Whether allreduce parameters as a whole. + Defaults to True. + bucket_size_mb (int, optional): Size of bucket, the unit is MB. + Defaults to -1. + """ + grads = [ + param.grad.data for param in params + if param.requires_grad and param.grad is not None + ] + world_size = dist.get_world_size() + if coalesce: + _allreduce_coalesced(grads, world_size, bucket_size_mb) + else: + for tensor in grads: + dist.all_reduce(tensor.div_(world_size)) + + +class DistOptimizerHook(OptimizerHook): + """Deprecated optimizer hook for distributed training.""" + def __init__(self, *args, **kwargs): + warnings.warn('"DistOptimizerHook" is deprecated, please switch to' + '"mmcv.runner.OptimizerHook".') + super().__init__(*args, **kwargs) + + +def reduce_mean(tensor): + """"Obtain the mean of tensor on different GPUs.""" + if not (dist.is_available() and dist.is_initialized()): + return tensor + tensor = tensor.clone() + dist.all_reduce(tensor.div_(dist.get_world_size()), op=dist.ReduceOp.SUM) + return tensor + + +def obj2tensor(pyobj, device='cuda'): + """Serialize picklable python object to tensor.""" + storage = torch.ByteStorage.from_buffer(pickle.dumps(pyobj)) + return torch.ByteTensor(storage).to(device=device) + + +def tensor2obj(tensor): + """Deserialize tensor to picklable python object.""" + return pickle.loads(tensor.cpu().numpy().tobytes()) + + +@functools.lru_cache() +def _get_global_gloo_group(): + """Return a process group based on gloo backend, containing all the ranks + The result is cached.""" + if dist.get_backend() == 'nccl': + return dist.new_group(backend='gloo') + else: + return dist.group.WORLD + + +def all_reduce_dict(py_dict, op='sum', group=None, to_float=True): + """Apply all reduce function for python dict object. + + The code is modified from https://github.com/Megvii- + BaseDetection/YOLOX/blob/main/yolox/utils/allreduce_norm.py. + + NOTE: make sure that py_dict in different ranks has the same keys and + the values should be in the same shape. + + Args: + py_dict (dict): Dict to be applied all reduce op. + op (str): Operator, could be 'sum' or 'mean'. Default: 'sum' + group (:obj:`torch.distributed.group`, optional): Distributed group, + Default: None. + to_float (bool): Whether to convert all values of dict to float. + Default: True. + + Returns: + OrderedDict: reduced python dict object. + """ + _, world_size = get_dist_info() + if world_size == 1: + return py_dict + if group is None: + # TODO: May try not to use gloo in the future + group = _get_global_gloo_group() + if dist.get_world_size(group) == 1: + return py_dict + + # all reduce logic across different devices. + py_key = list(py_dict.keys()) + py_key_tensor = obj2tensor(py_key) + dist.broadcast(py_key_tensor, src=0) + py_key = tensor2obj(py_key_tensor) + + tensor_shapes = [py_dict[k].shape for k in py_key] + tensor_numels = [py_dict[k].numel() for k in py_key] + + if to_float: + flatten_tensor = torch.cat( + [py_dict[k].flatten().float() for k in py_key]) + else: + flatten_tensor = torch.cat([py_dict[k].flatten() for k in py_key]) + + dist.all_reduce(flatten_tensor, op=dist.ReduceOp.SUM) + if op == 'mean': + flatten_tensor /= world_size + + split_tensors = [ + x.reshape(shape) for x, shape in zip( + torch.split(flatten_tensor, tensor_numels), tensor_shapes) + ] + return OrderedDict({k: v for k, v in zip(py_key, split_tensors)}) diff --git a/segmentation/mmseg_custom/core/utils/misc.py b/segmentation/mmseg_custom/core/utils/misc.py new file mode 100644 index 0000000..9e161fb --- /dev/null +++ b/segmentation/mmseg_custom/core/utils/misc.py @@ -0,0 +1,40 @@ +# Copyright (c) OpenMMLab. All rights reserved. +def multi_apply(func, *args, **kwargs): + """Apply function to a list of arguments. + + Note: + This function applies the ``func`` to multiple inputs and + map the multiple outputs of the ``func`` into different + list. Each list contains the same type of outputs corresponding + to different inputs. + + Args: + func (Function): A function that will be applied to a list of + arguments + + Returns: + tuple(list): A tuple containing multiple list, each list contains \ + a kind of returned results by the function + """ + pfunc = partial(func, **kwargs) if kwargs else func + map_results = map(pfunc, *args) + return tuple(map(list, zip(*map_results))) + + +def add_prefix(inputs, prefix): + """Add prefix for dict. + + Args: + inputs (dict): The input dict with str keys. + prefix (str): The prefix to add. + + Returns: + + dict: The dict with keys updated with ``prefix``. + """ + + outputs = dict() + for name, value in inputs.items(): + outputs[f'{prefix}.{name}'] = value + + return outputs diff --git a/segmentation/mmseg_custom/datasets/__init__.py b/segmentation/mmseg_custom/datasets/__init__.py new file mode 100644 index 0000000..613d2c7 --- /dev/null +++ b/segmentation/mmseg_custom/datasets/__init__.py @@ -0,0 +1,10 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from .mapillary import MapillaryDataset # noqa: F401,F403 +from .nyu_depth_v2 import NYUDepthV2Dataset # noqa: F401,F403 +from .pipelines import * # noqa: F401,F403 +from .dataset_wrappers import ConcatDataset + + +__all__ = [ + 'MapillaryDataset', 'NYUDepthV2Dataset', 'ConcatDataset' +] \ No newline at end of file diff --git a/segmentation/mmseg_custom/datasets/dataset_wrappers.py b/segmentation/mmseg_custom/datasets/dataset_wrappers.py new file mode 100644 index 0000000..1b9843e --- /dev/null +++ b/segmentation/mmseg_custom/datasets/dataset_wrappers.py @@ -0,0 +1,155 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import bisect +from itertools import chain + +import mmcv +import numpy as np +from mmcv.utils import build_from_cfg, print_log +from torch.utils.data.dataset import ConcatDataset as _ConcatDataset + +from mmseg.datasets.builder import DATASETS + + +@DATASETS.register_module(force=True) +class ConcatDataset(_ConcatDataset): + """A wrapper of concatenated dataset. + + Same as :obj:`torch.utils.data.dataset.ConcatDataset`, but + support evaluation and formatting results + + Args: + datasets (list[:obj:`Dataset`]): A list of datasets. + separate_eval (bool): Whether to evaluate the concatenated + dataset results separately, Defaults to True. + """ + + def __init__(self, datasets, separate_eval=True): + super(ConcatDataset, self).__init__(datasets) + self.CLASSES = datasets[0].CLASSES + self.PALETTE = datasets[0].PALETTE + self.separate_eval = separate_eval + assert separate_eval in [True, False], \ + f'separate_eval can only be True or False,' \ + f'but get {separate_eval}' + + def evaluate(self, results, logger=None, **kwargs): + """Evaluate the results. + + Args: + results (list[tuple[torch.Tensor]] | list[str]]): per image + pre_eval results or predict segmentation map for + computing evaluation metric. + logger (logging.Logger | str | None): Logger used for printing + related information during evaluation. Default: None. + + Returns: + dict[str: float]: evaluate results of the total dataset + or each separate + dataset if `self.separate_eval=True`. + """ + assert len(results) == self.cumulative_sizes[-1], \ + ('Dataset and results have different sizes: ' + f'{self.cumulative_sizes[-1]} v.s. {len(results)}') + + # Check whether all the datasets support evaluation + for dataset in self.datasets: + assert hasattr(dataset, 'evaluate'), \ + f'{type(dataset)} does not implement evaluate function' + + if self.separate_eval: + dataset_idx = -1 + total_eval_results = dict() + for size, dataset in zip(self.cumulative_sizes, self.datasets): + start_idx = 0 if dataset_idx == -1 else \ + self.cumulative_sizes[dataset_idx] + end_idx = self.cumulative_sizes[dataset_idx + 1] + + results_per_dataset = results[start_idx:end_idx] + print_log( + f'\nEvaluateing {dataset.img_dir} with ' + f'{len(results_per_dataset)} images now', + logger=logger) + + eval_results_per_dataset = dataset.evaluate( + results_per_dataset, logger=logger, **kwargs) + dataset_idx += 1 + for k, v in eval_results_per_dataset.items(): + total_eval_results.update({f'{dataset_idx}_{k}': v}) + + return total_eval_results + + if len(set([type(ds) for ds in self.datasets])) != 1: + raise NotImplementedError( + 'All the datasets should have same types when ' + 'self.separate_eval=False') + else: + if mmcv.is_list_of(results, np.ndarray) or mmcv.is_list_of( + results, str): + # merge the generators of gt_seg_maps + gt_seg_maps = chain( + *[dataset.get_gt_seg_maps() for dataset in self.datasets]) + else: + # if the results are `pre_eval` results, + # we do not need gt_seg_maps to evaluate + gt_seg_maps = None + eval_results = self.datasets[0].evaluate( + results, gt_seg_maps=gt_seg_maps, logger=logger, **kwargs) + return eval_results + + def get_dataset_idx_and_sample_idx(self, indice): + """Return dataset and sample index when given an indice of + ConcatDataset. + + Args: + indice (int): indice of sample in ConcatDataset + + Returns: + int: the index of sub dataset the sample belong to + int: the index of sample in its corresponding subset + """ + if indice < 0: + if -indice > len(self): + raise ValueError( + 'absolute value of index should not exceed dataset length') + indice = len(self) + indice + dataset_idx = bisect.bisect_right(self.cumulative_sizes, indice) + if dataset_idx == 0: + sample_idx = indice + else: + sample_idx = indice - self.cumulative_sizes[dataset_idx - 1] + return dataset_idx, sample_idx + + def format_results(self, results, imgfile_prefix, indices=None, **kwargs): + """format result for every sample of ConcatDataset.""" + if indices is None: + indices = list(range(len(self))) + + assert isinstance(results, list), 'results must be a list.' + assert isinstance(indices, list), 'indices must be a list.' + + ret_res = [] + for i, indice in enumerate(indices): + dataset_idx, sample_idx = self.get_dataset_idx_and_sample_idx( + indice) + res = self.datasets[dataset_idx].format_results( + [results[i]], + imgfile_prefix + f'/{dataset_idx}', + indices=[sample_idx], + **kwargs) + ret_res.append(res) + return sum(ret_res, []) + + def pre_eval(self, preds, indices): + """do pre eval for every sample of ConcatDataset.""" + # In order to compat with batch inference + if not isinstance(indices, list): + indices = [indices] + if not isinstance(preds, list): + preds = [preds] + ret_res = [] + for i, indice in enumerate(indices): + dataset_idx, sample_idx = self.get_dataset_idx_and_sample_idx( + indice) + res = self.datasets[dataset_idx].pre_eval(preds[i], sample_idx) + ret_res.append(res) + return sum(ret_res, []) diff --git a/segmentation/mmseg_custom/datasets/mapillary.py b/segmentation/mmseg_custom/datasets/mapillary.py new file mode 100644 index 0000000..8732097 --- /dev/null +++ b/segmentation/mmseg_custom/datasets/mapillary.py @@ -0,0 +1,48 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- +from mmseg.datasets.builder import DATASETS +from mmseg.datasets.custom import CustomDataset + + +@DATASETS.register_module() +class MapillaryDataset(CustomDataset): + """Mapillary dataset. + """ + CLASSES = ('Bird', 'Ground Animal', 'Curb', 'Fence', 'Guard Rail', 'Barrier', + 'Wall', 'Bike Lane', 'Crosswalk - Plain', 'Curb Cut', 'Parking', 'Pedestrian Area', + 'Rail Track', 'Road', 'Service Lane', 'Sidewalk', 'Bridge', 'Building', 'Tunnel', + 'Person', 'Bicyclist', 'Motorcyclist', 'Other Rider', 'Lane Marking - Crosswalk', + 'Lane Marking - General', 'Mountain', 'Sand', 'Sky', 'Snow', 'Terrain', 'Vegetation', + 'Water', 'Banner', 'Bench', 'Bike Rack', 'Billboard', 'Catch Basin', 'CCTV Camera', + 'Fire Hydrant', 'Junction Box', 'Mailbox', 'Manhole', 'Phone Booth', 'Pothole', + 'Street Light', 'Pole', 'Traffic Sign Frame', 'Utility Pole', 'Traffic Light', + 'Traffic Sign (Back)', 'Traffic Sign (Front)', 'Trash Can', 'Bicycle', 'Boat', + 'Bus', 'Car', 'Caravan', 'Motorcycle', 'On Rails', 'Other Vehicle', 'Trailer', + 'Truck', 'Wheeled Slow', 'Car Mount', 'Ego Vehicle', 'Unlabeled') + + PALETTE = [[165, 42, 42], [0, 192, 0], [196, 196, 196], [190, 153, 153], + [180, 165, 180], [90, 120, 150], [102, 102, 156], [128, 64, 255], + [140, 140, 200], [170, 170, 170], [250, 170, 160], [96, 96, 96], + [230, 150, 140], [128, 64, 128], [110, 110, 110], [244, 35, 232], + [150, 100, 100], [70, 70, 70], [150, 120, 90], [220, 20, 60], + [255, 0, 0], [255, 0, 100], [255, 0, 200], [200, 128, 128], + [255, 255, 255], [64, 170, 64], [230, 160, 50], [70, 130, 180], + [190, 255, 255], [152, 251, 152], [107, 142, 35], [0, 170, 30], + [255, 255, 128], [250, 0, 30], [100, 140, 180], [220, 220, 220], + [220, 128, 128], [222, 40, 40], [100, 170, 30], [40, 40, 40], + [33, 33, 33], [100, 128, 160], [142, 0, 0], [70, 100, 150], + [210, 170, 100], [153, 153, 153], [128, 128, 128], [0, 0, 80], + [250, 170, 30], [192, 192, 192], [220, 220, 0], [140, 140, 20], + [119, 11, 32], [150, 0, 255], [0, 60, 100], [0, 0, 142], [0, 0, 90], + [0, 0, 230], [0, 80, 100], [128, 64, 64], [0, 0, 110], [0, 0, 70], + [0, 0, 192], [32, 32, 32], [120, 10, 10], [0, 0, 0]] + + def __init__(self, **kwargs): + super(MapillaryDataset, self).__init__( + img_suffix='.jpg', + seg_map_suffix='.png', + reduce_zero_label=False, + **kwargs) \ No newline at end of file diff --git a/segmentation/mmseg_custom/datasets/nyu_depth_v2.py b/segmentation/mmseg_custom/datasets/nyu_depth_v2.py new file mode 100644 index 0000000..897ea86 --- /dev/null +++ b/segmentation/mmseg_custom/datasets/nyu_depth_v2.py @@ -0,0 +1,43 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- +from mmseg.datasets.builder import DATASETS +from mmseg.datasets.custom import CustomDataset + + +@DATASETS.register_module() +class NYUDepthV2Dataset(CustomDataset): + """NYU Depth V2 dataset. + """ + + CLASSES = ('wall', 'floor', 'cabinet', 'bed', 'chair', + 'sofa', 'table', 'door', 'window', 'bookshelf', + 'picture', 'counter', 'blinds', 'desk', 'shelves', + 'curtain', 'dresser', 'pillow', 'mirror', 'floor mat', + 'clothes', 'ceiling', 'books', 'refridgerator', 'television', + 'paper', 'towel', 'shower curtain', 'box', 'whiteboard', + 'person', 'night stand', 'toilet', 'sink', 'lamp', + 'bathtub', 'bag', 'otherstructure', 'otherfurniture', 'otherprop') + + + PALETTE = [[120, 120, 120], [180, 120, 120], [6, 230, 230], [80, 50, 50], + [4, 200, 3], [120, 120, 80], [140, 140, 140], [204, 5, 255], + [230, 230, 230], [4, 250, 7], [224, 5, 255], [235, 255, 7], + [150, 5, 61], [120, 120, 70], [8, 255, 51], [255, 6, 82], + [143, 255, 140], [204, 255, 4], [255, 51, 7], [204, 70, 3], + [0, 102, 200], [61, 230, 250], [255, 6, 51], [11, 102, 255], + [255, 7, 71], [255, 9, 224], [9, 7, 230], [220, 220, 220], + [255, 9, 92], [112, 9, 255], [8, 255, 214], [7, 255, 224], + [255, 184, 6], [10, 255, 71], [255, 41, 10], [7, 255, 255], + [224, 255, 8], [102, 8, 255], [255, 61, 6], [255, 194, 7],] + + def __init__(self, split, **kwargs): + super(NYUDepthV2Dataset, self).__init__( + img_suffix='.png', + seg_map_suffix='.png', + split=split, + reduce_zero_label=True, + **kwargs) + \ No newline at end of file diff --git a/segmentation/mmseg_custom/datasets/pipelines/__init__.py b/segmentation/mmseg_custom/datasets/pipelines/__init__.py new file mode 100644 index 0000000..b0f53e3 --- /dev/null +++ b/segmentation/mmseg_custom/datasets/pipelines/__init__.py @@ -0,0 +1,8 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from .formatting import DefaultFormatBundle, ToMask +from .transform import MapillaryHack, PadShortSide, SETR_Resize + +__all__ = [ + 'DefaultFormatBundle', 'ToMask', 'SETR_Resize', + 'PadShortSide', 'MapillaryHack' +] diff --git a/segmentation/mmseg_custom/datasets/pipelines/formatting.py b/segmentation/mmseg_custom/datasets/pipelines/formatting.py new file mode 100644 index 0000000..d1a41ad --- /dev/null +++ b/segmentation/mmseg_custom/datasets/pipelines/formatting.py @@ -0,0 +1,82 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import numpy as np +from mmcv.parallel import DataContainer as DC +from mmseg.datasets.builder import PIPELINES +from mmseg.datasets.pipelines.formatting import to_tensor + + +@PIPELINES.register_module(force=True) +class DefaultFormatBundle(object): + """Default formatting bundle. + + It simplifies the pipeline of formatting common fields, including "img" + and "gt_semantic_seg". These fields are formatted as follows. + + - img: (1)transpose, (2)to tensor, (3)to DataContainer (stack=True) + - gt_semantic_seg: (1)unsqueeze dim-0 (2)to tensor, + (3)to DataContainer (stack=True) + """ + def __call__(self, results): + """Call function to transform and format common fields in results. + + Args: + results (dict): Result dict contains the data to convert. + + Returns: + dict: The result dict contains the data that is formatted with + default bundle. + """ + + if 'img' in results: + img = results['img'] + if len(img.shape) < 3: + img = np.expand_dims(img, -1) + img = np.ascontiguousarray(img.transpose(2, 0, 1)) + results['img'] = DC(to_tensor(img), stack=True) + if 'gt_semantic_seg' in results: + # convert to long + results['gt_semantic_seg'] = DC(to_tensor( + results['gt_semantic_seg'][None, ...].astype(np.int64)), + stack=True) + if 'gt_masks' in results: + results['gt_masks'] = DC(to_tensor(results['gt_masks'])) + if 'gt_labels' in results: + results['gt_labels'] = DC(to_tensor(results['gt_labels'])) + + return results + + def __repr__(self): + return self.__class__.__name__ + + +@PIPELINES.register_module() +class ToMask(object): + """Transfer gt_semantic_seg to binary mask and generate gt_labels.""" + def __init__(self, ignore_index=255): + self.ignore_index = ignore_index + + def __call__(self, results): + gt_semantic_seg = results['gt_semantic_seg'] + gt_labels = np.unique(gt_semantic_seg) + # remove ignored region + gt_labels = gt_labels[gt_labels != self.ignore_index] + + gt_masks = [] + for class_id in gt_labels: + gt_masks.append(gt_semantic_seg == class_id) + + if len(gt_masks) == 0: + # Some image does not have annotation (all ignored) + gt_masks = np.empty((0, ) + results['pad_shape'][:-1], dtype=np.int64) + gt_labels = np.empty((0, ), dtype=np.int64) + else: + gt_masks = np.asarray(gt_masks, dtype=np.int64) + gt_labels = np.asarray(gt_labels, dtype=np.int64) + + results['gt_labels'] = gt_labels + results['gt_masks'] = gt_masks + return results + + def __repr__(self): + return self.__class__.__name__ + \ + f'(ignore_index={self.ignore_index})' diff --git a/segmentation/mmseg_custom/datasets/pipelines/transform.py b/segmentation/mmseg_custom/datasets/pipelines/transform.py new file mode 100644 index 0000000..b193c9c --- /dev/null +++ b/segmentation/mmseg_custom/datasets/pipelines/transform.py @@ -0,0 +1,350 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import mmcv +import numpy as np +from mmseg.datasets.builder import PIPELINES + + +@PIPELINES.register_module() +class SETR_Resize(object): + """Resize images & seg. + + This transform resizes the input image to some scale. If the input dict + contains the key "scale", then the scale in the input dict is used, + otherwise the specified scale in the init method is used. + + ``img_scale`` can either be a tuple (single-scale) or a list of tuple + (multi-scale). There are 3 multiscale modes: + + - ``ratio_range is not None``: randomly sample a ratio from the ratio range + and multiply it with the image scale. + + - ``ratio_range is None and multiscale_mode == "range"``: randomly sample a + scale from the a range. + + - ``ratio_range is None and multiscale_mode == "value"``: randomly sample a + scale from multiple scales. + + Args: + img_scale (tuple or list[tuple]): Images scales for resizing. + multiscale_mode (str): Either "range" or "value". + ratio_range (tuple[float]): (min_ratio, max_ratio) + keep_ratio (bool): Whether to keep the aspect ratio when resizing the + image. + """ + def __init__(self, + img_scale=None, + multiscale_mode='range', + ratio_range=None, + keep_ratio=True, + crop_size=None, + setr_multi_scale=False): + + if img_scale is None: + self.img_scale = None + else: + if isinstance(img_scale, list): + self.img_scale = img_scale + else: + self.img_scale = [img_scale] + # assert mmcv.is_list_of(self.img_scale, tuple) + + if ratio_range is not None: + # mode 1: given a scale and a range of image ratio + assert len(self.img_scale) == 1 + else: + # mode 2: given multiple scales or a range of scales + assert multiscale_mode in ['value', 'range'] + + self.multiscale_mode = multiscale_mode + self.ratio_range = ratio_range + self.keep_ratio = keep_ratio + self.crop_size = crop_size + self.setr_multi_scale = setr_multi_scale + + @staticmethod + def random_select(img_scales): + """Randomly select an img_scale from given candidates. + + Args: + img_scales (list[tuple]): Images scales for selection. + + Returns: + (tuple, int): Returns a tuple ``(img_scale, scale_dix)``, + where ``img_scale`` is the selected image scale and + ``scale_idx`` is the selected index in the given candidates. + """ + + assert mmcv.is_list_of(img_scales, tuple) + scale_idx = np.random.randint(len(img_scales)) + img_scale = img_scales[scale_idx] + return img_scale, scale_idx + + @staticmethod + def random_sample(img_scales): + """Randomly sample an img_scale when ``multiscale_mode=='range'``. + + Args: + img_scales (list[tuple]): Images scale range for sampling. + There must be two tuples in img_scales, which specify the lower + and uper bound of image scales. + + Returns: + (tuple, None): Returns a tuple ``(img_scale, None)``, where + ``img_scale`` is sampled scale and None is just a placeholder + to be consistent with :func:`random_select`. + """ + + assert mmcv.is_list_of(img_scales, tuple) and len(img_scales) == 2 + img_scale_long = [max(s) for s in img_scales] + img_scale_short = [min(s) for s in img_scales] + long_edge = np.random.randint( + min(img_scale_long), + max(img_scale_long) + 1) + short_edge = np.random.randint( + min(img_scale_short), + max(img_scale_short) + 1) + img_scale = (long_edge, short_edge) + return img_scale, None + + @staticmethod + def random_sample_ratio(img_scale, ratio_range): + """Randomly sample an img_scale when ``ratio_range`` is specified. + + A ratio will be randomly sampled from the range specified by + ``ratio_range``. Then it would be multiplied with ``img_scale`` to + generate sampled scale. + + Args: + img_scale (tuple): Images scale base to multiply with ratio. + ratio_range (tuple[float]): The minimum and maximum ratio to scale + the ``img_scale``. + + Returns: + (tuple, None): Returns a tuple ``(scale, None)``, where + ``scale`` is sampled ratio multiplied with ``img_scale`` and + None is just a placeholder to be consistent with + :func:`random_select`. + """ + + assert isinstance(img_scale, tuple) and len(img_scale) == 2 + min_ratio, max_ratio = ratio_range + assert min_ratio <= max_ratio + ratio = np.random.random_sample() * (max_ratio - min_ratio) + min_ratio + scale = int(img_scale[0] * ratio), int(img_scale[1] * ratio) + return scale, None + + def _random_scale(self, results): + """Randomly sample an img_scale according to ``ratio_range`` and + ``multiscale_mode``. + + If ``ratio_range`` is specified, a ratio will be sampled and be + multiplied with ``img_scale``. + If multiple scales are specified by ``img_scale``, a scale will be + sampled according to ``multiscale_mode``. + Otherwise, single scale will be used. + + Args: + results (dict): Result dict from :obj:`dataset`. + + Returns: + dict: Two new keys 'scale` and 'scale_idx` are added into + ``results``, which would be used by subsequent pipelines. + """ + + if self.ratio_range is not None: + scale, scale_idx = self.random_sample_ratio( + self.img_scale[0], self.ratio_range) + elif len(self.img_scale) == 1: + scale, scale_idx = self.img_scale[0], 0 + elif self.multiscale_mode == 'range': + scale, scale_idx = self.random_sample(self.img_scale) + elif self.multiscale_mode == 'value': + scale, scale_idx = self.random_select(self.img_scale) + else: + raise NotImplementedError + + results['scale'] = scale + results['scale_idx'] = scale_idx + + def _resize_img(self, results): + """Resize images with ``results['scale']``.""" + + if self.keep_ratio: + if self.setr_multi_scale: + if min(results['scale']) < self.crop_size[0]: + new_short = self.crop_size[0] + else: + new_short = min(results['scale']) + + h, w = results['img'].shape[:2] + if h > w: + new_h, new_w = new_short * h / w, new_short + else: + new_h, new_w = new_short, new_short * w / h + results['scale'] = (new_h, new_w) + + img, scale_factor = mmcv.imrescale(results['img'], + results['scale'], + return_scale=True) + # the w_scale and h_scale has minor difference + # a real fix should be done in the mmcv.imrescale in the future + new_h, new_w = img.shape[:2] + h, w = results['img'].shape[:2] + w_scale = new_w / w + h_scale = new_h / h + else: + img, w_scale, h_scale = mmcv.imresize(results['img'], + results['scale'], + return_scale=True) + scale_factor = np.array([w_scale, h_scale, w_scale, h_scale], + dtype=np.float32) + results['img'] = img + results['img_shape'] = img.shape + results['pad_shape'] = img.shape # in case that there is no padding + results['scale_factor'] = scale_factor + results['keep_ratio'] = self.keep_ratio + + def _resize_seg(self, results): + """Resize semantic segmentation map with ``results['scale']``.""" + for key in results.get('seg_fields', []): + if self.keep_ratio: + gt_seg = mmcv.imrescale(results[key], + results['scale'], + interpolation='nearest') + else: + gt_seg = mmcv.imresize(results[key], + results['scale'], + interpolation='nearest') + results['gt_semantic_seg'] = gt_seg + + def __call__(self, results): + """Call function to resize images, bounding boxes, masks, semantic + segmentation map. + + Args: + results (dict): Result dict from loading pipeline. + + Returns: + dict: Resized results, 'img_shape', 'pad_shape', 'scale_factor', + 'keep_ratio' keys are added into result dict. + """ + + if 'scale' not in results: + self._random_scale(results) + self._resize_img(results) + self._resize_seg(results) + return results + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += (f'(img_scale={self.img_scale}, ' + f'multiscale_mode={self.multiscale_mode}, ' + f'ratio_range={self.ratio_range}, ' + f'keep_ratio={self.keep_ratio})') + return repr_str + + +@PIPELINES.register_module() +class PadShortSide(object): + """Pad the image & mask. + + Pad to the minimum size that is equal or larger than a number. + Added keys are "pad_shape", "pad_fixed_size", + + Args: + size (int, optional): Fixed padding size. + pad_val (float, optional): Padding value. Default: 0. + seg_pad_val (float, optional): Padding value of segmentation map. + Default: 255. + """ + def __init__(self, size=None, pad_val=0, seg_pad_val=255): + self.size = size + self.pad_val = pad_val + self.seg_pad_val = seg_pad_val + # only one of size and size_divisor should be valid + assert size is not None + + def _pad_img(self, results): + """Pad images according to ``self.size``.""" + h, w = results['img'].shape[:2] + new_h = max(h, self.size) + new_w = max(w, self.size) + padded_img = mmcv.impad(results['img'], + shape=(new_h, new_w), + pad_val=self.pad_val) + + results['img'] = padded_img + results['pad_shape'] = padded_img.shape + # results['unpad_shape'] = (h, w) + + def _pad_seg(self, results): + """Pad masks according to ``results['pad_shape']``.""" + for key in results.get('seg_fields', []): + results[key] = mmcv.impad(results[key], + shape=results['pad_shape'][:2], + pad_val=self.seg_pad_val) + + def __call__(self, results): + """Call function to pad images, masks, semantic segmentation maps. + + Args: + results (dict): Result dict from loading pipeline. + + Returns: + dict: Updated result dict. + """ + h, w = results['img'].shape[:2] + if h >= self.size and w >= self.size: # 短边比窗口大，跳过 + pass + else: + self._pad_img(results) + self._pad_seg(results) + return results + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += f'(size={self.size}, pad_val={self.pad_val})' + return repr_str + + +@PIPELINES.register_module() +class MapillaryHack(object): + """map MV 65 class to 19 class like Cityscapes.""" + def __init__(self): + self.map = [[13, 24, 41], [2, 15], [17], [6], [3], + [45, 47], [48], [50], [30], [29], [27], [19], [20, 21, 22], + [55], [61], [54], [58], [57], [52]] + + self.others = [i for i in range(66)] + for i in self.map: + for j in i: + if j in self.others: + self.others.remove(j) + + def __call__(self, results): + """Call function to process the image with gamma correction. + + Args: + results (dict): Result dict from loading pipeline. + + Returns: + dict: Processed results. + """ + gt_map = results['gt_semantic_seg'] + # others -> 255 + new_gt_map = np.zeros_like(gt_map) + + for value in self.others: + new_gt_map[gt_map == value] = 255 + + for index, map in enumerate(self.map): + for value in map: + new_gt_map[gt_map == value] = index + + results['gt_semantic_seg'] = new_gt_map + + return results + + def __repr__(self): + repr_str = self.__class__.__name__ + return repr_str diff --git a/segmentation/mmseg_custom/models/__init__.py b/segmentation/mmseg_custom/models/__init__.py new file mode 100644 index 0000000..d88de89 --- /dev/null +++ b/segmentation/mmseg_custom/models/__init__.py @@ -0,0 +1,12 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + +from .backbones import * # noqa: F401,F403 +from .decode_heads import * # noqa: F401,F403 +from .losses import * # noqa: F401,F403 +from .plugins import * # noqa: F401,F403 +from .segmentors import * # noqa: F401,F403 +from .utils import * # noqa: F401,F403 \ No newline at end of file diff --git a/segmentation/mmseg_custom/models/backbones/__init__.py b/segmentation/mmseg_custom/models/backbones/__init__.py new file mode 100644 index 0000000..352c244 --- /dev/null +++ b/segmentation/mmseg_custom/models/backbones/__init__.py @@ -0,0 +1,9 @@ +# -------------------------------------------------------- +# FlashInternImage +# Copyright (c) 2023 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + +from .flash_intern_image import FlashInternImage + +__all__ = ['FlashInternImage'] diff --git a/segmentation/mmseg_custom/models/backbones/flash_intern_image.py b/segmentation/mmseg_custom/models/backbones/flash_intern_image.py new file mode 100644 index 0000000..cca3bca --- /dev/null +++ b/segmentation/mmseg_custom/models/backbones/flash_intern_image.py @@ -0,0 +1,763 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + +import torch +import torch.nn as nn +from collections import OrderedDict +import torch.utils.checkpoint as checkpoint +from timm.models.layers import trunc_normal_, DropPath +from mmcv.runner import _load_checkpoint +from mmcv.cnn import constant_init, trunc_normal_init +from mmseg.utils import get_root_logger +from mmseg.models.builder import BACKBONES +import torch.nn.functional as F +import DCNv4 + + +class to_channels_first(nn.Module): + + def __init__(self): + super().__init__() + + def forward(self, x): + return x.permute(0, 3, 1, 2) + + +class to_channels_last(nn.Module): + + def __init__(self): + super().__init__() + + def forward(self, x): + return x.permute(0, 2, 3, 1) + + +def build_norm_layer(dim, + norm_layer, + in_format='channels_last', + out_format='channels_last', + eps=1e-6): + layers = [] + if norm_layer == 'BN': + if in_format == 'channels_last': + layers.append(to_channels_first()) + layers.append(nn.BatchNorm2d(dim)) + if out_format == 'channels_last': + layers.append(to_channels_last()) + elif norm_layer == 'LN': + if in_format == 'channels_first': + layers.append(to_channels_last()) + layers.append(nn.LayerNorm(dim, eps=eps)) + if out_format == 'channels_first': + layers.append(to_channels_first()) + else: + raise NotImplementedError( + f'build_norm_layer does not support {norm_layer}') + return nn.Sequential(*layers) + + +def build_act_layer(act_layer): + if act_layer == 'ReLU': + return nn.ReLU(inplace=True) + elif act_layer == 'SiLU': + return nn.SiLU(inplace=True) + elif act_layer == 'GELU': + return nn.GELU() + + raise NotImplementedError(f'build_act_layer does not support {act_layer}') + + +class CrossAttention(nn.Module): + r""" Cross Attention Module + Args: + dim (int): Number of input channels. + num_heads (int): Number of attention heads. Default: 8 + qkv_bias (bool, optional): If True, add a learnable bias to q, k, v. + Default: False. + qk_scale (float | None, optional): Override default qk scale of + head_dim ** -0.5 if set. Default: None. + attn_drop (float, optional): Dropout ratio of attention weight. + Default: 0.0 + proj_drop (float, optional): Dropout ratio of output. Default: 0.0 + attn_head_dim (int, optional): Dimension of attention head. + out_dim (int, optional): Dimension of output. + """ + + def __init__(self, + dim, + num_heads=8, + qkv_bias=False, + qk_scale=None, + attn_drop=0., + proj_drop=0., + attn_head_dim=None, + out_dim=None): + super().__init__() + if out_dim is None: + out_dim = dim + self.num_heads = num_heads + head_dim = dim // num_heads + if attn_head_dim is not None: + head_dim = attn_head_dim + all_head_dim = head_dim * self.num_heads + self.scale = qk_scale or head_dim ** -0.5 + assert all_head_dim == dim + + self.q = nn.Linear(dim, all_head_dim, bias=False) + self.k = nn.Linear(dim, all_head_dim, bias=False) + self.v = nn.Linear(dim, all_head_dim, bias=False) + + if qkv_bias: + self.q_bias = nn.Parameter(torch.zeros(all_head_dim)) + self.k_bias = nn.Parameter(torch.zeros(all_head_dim)) + self.v_bias = nn.Parameter(torch.zeros(all_head_dim)) + else: + self.q_bias = None + self.k_bias = None + self.v_bias = None + + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(all_head_dim, out_dim) + self.proj_drop = nn.Dropout(proj_drop) + + def forward(self, x, k=None, v=None): + B, N, C = x.shape + N_k = k.shape[1] + N_v = v.shape[1] + + q_bias, k_bias, v_bias = None, None, None + if self.q_bias is not None: + q_bias = self.q_bias + k_bias = self.k_bias + v_bias = self.v_bias + + q = F.linear(input=x, weight=self.q.weight, bias=q_bias) + q = q.reshape(B, N, 1, self.num_heads, + -1).permute(2, 0, 3, 1, + 4).squeeze(0) # (B, N_head, N_q, dim) + + k = F.linear(input=k, weight=self.k.weight, bias=k_bias) + k = k.reshape(B, N_k, 1, self.num_heads, -1).permute(2, 0, 3, 1, + 4).squeeze(0) + + v = F.linear(input=v, weight=self.v.weight, bias=v_bias) + v = v.reshape(B, N_v, 1, self.num_heads, -1).permute(2, 0, 3, 1, + 4).squeeze(0) + + q = q * self.scale + attn = (q @ k.transpose(-2, -1)) # (B, N_head, N_q, N_k) + + attn = attn.softmax(dim=-1) + attn = self.attn_drop(attn) + + x = (attn @ v).transpose(1, 2).reshape(B, N, -1) + x = self.proj(x) + x = self.proj_drop(x) + + return x + + +class AttentiveBlock(nn.Module): + r"""Attentive Block + Args: + dim (int): Number of input channels. + num_heads (int): Number of attention heads. Default: 8 + qkv_bias (bool, optional): If True, add a learnable bias to q, k, v. + Default: False. + qk_scale (float | None, optional): Override default qk scale of + head_dim ** -0.5 if set. Default: None. + drop (float, optional): Dropout rate. Default: 0.0. + attn_drop (float, optional): Attention dropout rate. Default: 0.0. + drop_path (float | tuple[float], optional): Stochastic depth rate. + Default: 0.0. + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm. + attn_head_dim (int, optional): Dimension of attention head. Default: None. + out_dim (int, optional): Dimension of output. Default: None. + """ + + def __init__(self, + dim, + num_heads, + qkv_bias=False, + qk_scale=None, + drop=0., + attn_drop=0., + drop_path=0., + norm_layer="LN", + attn_head_dim=None, + out_dim=None): + super().__init__() + + self.norm1_q = build_norm_layer(dim, norm_layer, eps=1e-6) + self.norm1_k = build_norm_layer(dim, norm_layer, eps=1e-6) + self.norm1_v = build_norm_layer(dim, norm_layer, eps=1e-6) + self.cross_dcn = CrossAttention(dim, + num_heads=num_heads, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + attn_drop=attn_drop, + proj_drop=drop, + attn_head_dim=attn_head_dim, + out_dim=out_dim) + + self.drop_path = DropPath( + drop_path) if drop_path > 0. else nn.Identity() + + def forward(self, + x_q, + x_kv, + pos_q, + pos_k, + bool_masked_pos, + rel_pos_bias=None): + x_q = self.norm1_q(x_q + pos_q) + x_k = self.norm1_k(x_kv + pos_k) + x_v = self.norm1_v(x_kv) + + x = self.cross_dcn(x_q, k=x_k, v=x_v) + + return x + + +class AttentionPoolingBlock(AttentiveBlock): + + def forward(self, x): + x_q = x.mean(1, keepdim=True) + x_kv = x + pos_q, pos_k = 0, 0 + x = super().forward(x_q, x_kv, pos_q, pos_k, + bool_masked_pos=None, + rel_pos_bias=None) + x = x.squeeze(1) + return x + + +class StemLayer(nn.Module): + r""" Stem layer of InternImage + Args: + in_chans (int): number of input channels + out_chans (int): number of output channels + act_layer (str): activation layer + norm_layer (str): normalization layer + """ + + def __init__(self, + in_chans=3, + out_chans=96, + act_layer='GELU', + norm_layer='BN'): + super().__init__() + self.conv1 = nn.Conv2d(in_chans, + out_chans // 2, + kernel_size=3, + stride=2, + padding=1) + self.norm1 = build_norm_layer(out_chans // 2, norm_layer, + 'channels_first', 'channels_first') + self.act = build_act_layer(act_layer) + self.conv2 = nn.Conv2d(out_chans // 2, + out_chans, + kernel_size=3, + stride=2, + padding=1) + self.norm2 = build_norm_layer(out_chans, norm_layer, 'channels_first', + 'channels_last') + + def forward(self, x): + x = self.conv1(x) + x = self.norm1(x) + x = self.act(x) + x = self.conv2(x) + x = self.norm2(x) + return x + + + +class DownsampleLayer(nn.Module): + r""" Downsample layer of InternImage + Args: + channels (int): number of input channels + norm_layer (str): normalization layer + """ + + def __init__(self, channels, norm_layer='LN'): + super().__init__() + self.conv = nn.Conv2d(channels, + 2 * channels, + kernel_size=3, + stride=2, + padding=1, + bias=False) + self.norm = build_norm_layer(2 * channels, norm_layer, + 'channels_first', 'channels_first') + + + def forward(self, x, shape=None): + H, W = shape + N, HW, C = x.shape + + x = x.view(N, H, W, C) + x = self.conv(x.permute(0, 3, 1, 2)) + x = self.norm(x) # B C H W + H, W = x.size(2), x.size(3) + x = x.flatten(2).permute(0, 2, 1) + + return x, (H, W) + + + +class MLPLayer(nn.Module): + r""" MLP layer of InternImage + Args: + in_features (int): number of input features + hidden_features (int): number of hidden features + out_features (int): number of output features + act_layer (str): activation layer + drop (float): dropout rate + """ + + def __init__(self, + in_features, + hidden_features=None, + out_features=None, + act_layer='GELU', + mlp_fc2_bias=False, + drop=0.): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = nn.Linear(in_features, hidden_features, bias=True) + self.act = build_act_layer(act_layer) + self.fc2 = nn.Linear(hidden_features, out_features, bias=mlp_fc2_bias) + self.drop = nn.Dropout(drop) + + + def forward(self, x, shape, level_idx=0): + x = self.fc1(x) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + return x + + +class InternImageLayer(nn.Module): + r""" Basic layer of InternImage + Args: + core_op (nn.Module): core operation of InternImage + channels (int): number of input channels + groups (list): Groups of each block. + mlp_ratio (float): ratio of mlp hidden features to input channels + drop (float): dropout rate + drop_path (float): drop path rate + act_layer (str): activation layer + norm_layer (str): normalization layer + post_norm (bool): whether to use post normalization + layer_scale (float): layer scale + offset_scale (float): offset scale + with_cp (bool): whether to use checkpoint + """ + + def __init__(self, + core_op, + channels, + groups, + mlp_ratio=4., + drop=0., + drop_path=0., + act_layer='GELU', + norm_layer='LN', + post_norm=False, + layer_scale=None, + offset_scale=1.0, + with_cp=False, + dcn_output_bias=False, + mlp_fc2_bias=False, + dw_kernel_size=None, # for InternImage-H/G + res_post_norm=False, # for InternImage-H/G + center_feature_scale=False): # for InternImage-H/G + super().__init__() + self.channels = channels + self.groups = groups + self.mlp_ratio = mlp_ratio + self.with_cp = with_cp + + self.norm1 = build_norm_layer(channels, 'LN') + self.post_norm = post_norm + self.dcn = core_op( + channels=channels, + group=groups, + offset_scale=offset_scale, + dw_kernel_size=dw_kernel_size, + output_bias=dcn_output_bias, + ) + self.drop_path = DropPath(drop_path) if drop_path > 0. \ + else nn.Identity() + self.norm2 = build_norm_layer(channels, 'LN') + self.mlp = MLPLayer(in_features=channels, + hidden_features=int(channels * mlp_ratio), + act_layer=act_layer, + drop=drop, + mlp_fc2_bias=mlp_fc2_bias + ) + self.layer_scale = layer_scale is not None + if self.layer_scale: + self.gamma1 = nn.Parameter(layer_scale * torch.ones(channels), + requires_grad=True) + self.gamma2 = nn.Parameter(layer_scale * torch.ones(channels), + requires_grad=True) + self.res_post_norm = res_post_norm + if res_post_norm: + self.res_post_norm1 = build_norm_layer(channels, 'LN') + self.res_post_norm2 = build_norm_layer(channels, 'LN') + def forward(self, x, shape, level_idx=0): + + def _inner_forward(x, shape, level_idx): + if not self.layer_scale: + if self.post_norm: + x = x + self.drop_path(self.norm1(self.dcn(x, shape, level_idx))) + x = x + self.drop_path(self.norm2(self.mlp(x, shape, level_idx))) + elif self.res_post_norm: # for InternImage-H/G + x = x + self.drop_path(self.res_post_norm1(self.dcn(self.norm1(x), shape, level_idx))) + x = x + self.drop_path(self.res_post_norm2(self.mlp(self.norm2(x), shape, level_idx))) + + else: + x = x + self.drop_path(self.dcn(self.norm1(x), shape, level_idx)) + x = x + self.drop_path(self.mlp(self.norm2(x), shape, level_idx)) + return x + if self.post_norm: + x = x + self.drop_path(self.gamma1 * self.norm1(self.dcn(x, shape))) + x = x + self.drop_path(self.gamma2 * self.norm2(self.mlp(x, shape, level_idx))) + else: + x = x + self.drop_path(self.gamma1 * self.dcn(self.norm1(x), shape)) + x = x + self.drop_path(self.gamma2 * self.mlp(self.norm2(x), shape, level_idx)) + return x + + if self.with_cp and x.requires_grad: + x = checkpoint.checkpoint(_inner_forward, x, shape, level_idx) + else: + x = _inner_forward(x, shape, level_idx) + + return x + + +class InternImageBlock(nn.Module): + r""" Block of InternImage + Args: + core_op (nn.Module): core operation of InternImage + channels (int): number of input channels + depths (list): Depth of each block. + groups (list): Groups of each block. + mlp_ratio (float): ratio of mlp hidden features to input channels + drop (float): dropout rate + drop_path (float): drop path rate + act_layer (str): activation layer + norm_layer (str): normalization layer + post_norm (bool): whether to use post normalization + layer_scale (float): layer scale + offset_scale (float): offset scale + with_cp (bool): whether to use checkpoint + """ + + def __init__(self, + core_op, + channels, + depth, + groups, + downsample=True, + downsample_layer=DownsampleLayer, + mlp_ratio=4., + drop=0., + drop_path=0., + act_layer='GELU', + norm_layer='LN', + post_norm=False, + offset_scale=0.5, + layer_scale=None, + with_cp=False, + dcn_output_bias=False, + mlp_fc2_bias=False, + dw_kernel_size=None, # for InternImage-H/G + post_norm_block_ids=None, # for InternImage-H/G + res_post_norm=False, # for InternImage-H/G + center_feature_scale=False): # for InternImage-H/G + super().__init__() + self.channels = channels + self.depth = depth + self.post_norm = post_norm + self.center_feature_scale = center_feature_scale + + self.blocks = nn.ModuleList([ + InternImageLayer( + core_op=core_op, + channels=channels, + groups=groups, + mlp_ratio=mlp_ratio, + drop=drop, + drop_path=drop_path[i] if isinstance( + drop_path, list) else drop_path, + act_layer=act_layer, + norm_layer=norm_layer, + post_norm=post_norm, + layer_scale=layer_scale, + offset_scale=offset_scale, + with_cp=with_cp, + dcn_output_bias=dcn_output_bias, + mlp_fc2_bias=mlp_fc2_bias, + dw_kernel_size=dw_kernel_size, # for InternImage-H/G + res_post_norm=res_post_norm, # for InternImage-H/G + center_feature_scale=center_feature_scale # for InternImage-H/G + ) for i in range(depth) + ]) + if not self.post_norm or center_feature_scale: + self.norm = build_norm_layer(channels, 'LN') + self.post_norm_block_ids = post_norm_block_ids + if post_norm_block_ids is not None: # for InternImage-H/G + self.post_norms = nn.ModuleList( + [build_norm_layer(channels, 'LN', eps=1e-6) for _ in post_norm_block_ids] + ) + self.downsample = downsample_layer( + channels=channels, norm_layer=norm_layer) if downsample else None + + + def forward(self, x, return_wo_downsample=False, shape=None, level_idx=0 + ): + for i, blk in enumerate(self.blocks): + x = blk(x, shape=shape, level_idx=level_idx) + if (self.post_norm_block_ids is not None) and (i in self.post_norm_block_ids): + index = self.post_norm_block_ids.index(i) + x = self.post_norms[index](x) # for InternImage-H/G + if not self.post_norm or self.center_feature_scale: + x = self.norm(x) + if return_wo_downsample: + x_ = x.clone() + if self.downsample is not None: + x, shape = self.downsample(x, shape=shape) + + if return_wo_downsample: + return x, x_, shape + return x, shape + +@BACKBONES.register_module() +class FlashInternImage(nn.Module): + r""" FlashInternImage + A PyTorch impl based on : + `InternImage: Exploring Large-Scale Vision Foundation Models with Deformable Convolutions` - + https://arxiv.org/pdf/2103.14030 + 'DCNv4': TODO: add arxiv + Args: + core_op (str): Core operator. Default: 'DCNv4' + channels (int): Number of the first stage. Default: 64 + depths (list): Depth of each block. Default: [3, 4, 18, 5] + groups (list): Groups of each block. Default: [3, 6, 12, 24] + num_classes (int): Number of classes. Default: 1000 + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. Default: 4. + drop_rate (float): Probability of an element to be zeroed. Default: 0. + drop_path_rate (float): Stochastic depth rate. Default: 0.2 + act_layer (str): Activation layer. Default: 'GELU' + norm_layer (str): Normalization layer. Default: 'LN' + layer_scale (bool): Whether to use layer scale. Default: False + cls_scale (bool): Whether to use class scale. Default: False + with_cp (bool): Use checkpoint or not. Using checkpoint will save some + dw_kernel_size (int): Size of the dwconv. Default: None + use_clip_projector (bool): Whether to use clip projector. Default: False + level2_post_norm (bool): Whether to use level2 post norm. Default: False + level2_post_norm_block_ids (list): Indexes of post norm blocks. Default: None + res_post_norm (bool): Whether to use res post norm. Default: False + center_feature_scale (bool): Whether to use center feature scale. Default: False + """ + + def __init__(self, + core_op='DCNv4', + channels=64, + depths=[3, 4, 18, 5], + groups=[3, 6, 12, 24], + num_classes=1000, + mlp_ratio=4., + drop_rate=0., + drop_path_rate=0.2, + drop_path_type='linear', + act_layer='GELU', + norm_layer='LN', + layer_scale=None, + offset_scale=0.5, + post_norm=False, + with_cp=False, + mlp_fc2_bias=False, + dcn_output_bias=False, + dw_kernel_size=None, # for InternImage-H/G + level2_post_norm=False, # for InternImage-H/G + level2_post_norm_block_ids=None, # for InternImage-H/G + res_post_norm=False, # for InternImage-H/G + center_feature_scale=False, # for InternImage-H/G + out_indices=(0, 1, 2, 3), + init_cfg=None, + **kwargs): + super().__init__() + self.core_op = core_op + self.num_levels = len(depths) + self.depths = depths + self.channels = channels + self.num_features = int(channels * 2**(self.num_levels - 1)) + self.post_norm = post_norm + self.mlp_ratio = mlp_ratio + self.init_cfg = init_cfg + self.out_indices = out_indices + self.level2_post_norm_block_ids = level2_post_norm_block_ids + logger = get_root_logger() + logger.info(f'using core type: {core_op}') + logger.info(f'using activation layer: {act_layer}') + logger.info(f'using main norm layer: {norm_layer}') + logger.info(f'using dpr: {drop_path_type}, {drop_path_rate}') + logger.info(f"level2_post_norm: {level2_post_norm}") + logger.info(f"level2_post_norm_block_ids: {level2_post_norm_block_ids}") + logger.info(f"res_post_norm: {res_post_norm}") + + in_chans = 3 + self.patch_embed = StemLayer(in_chans=in_chans, + out_chans=channels, + act_layer=act_layer, + norm_layer=norm_layer) + self.pos_drop = nn.Dropout(p=drop_rate) + + dpr = [ + x.item() for x in torch.linspace(0, drop_path_rate, sum(depths)) + ] + if drop_path_type == 'uniform': + for i in range(len(dpr)): + dpr[i] = drop_path_rate + + self.levels = nn.ModuleList() + for i in range(self.num_levels): + post_norm_block_ids = level2_post_norm_block_ids if level2_post_norm and ( + i == 2) else None # for InternImage-H/G + + level = InternImageBlock( + core_op=getattr(DCNv4, core_op), + channels=int(channels * 2**i), + depth=depths[i], + groups=groups[i], + mlp_ratio=self.mlp_ratio, + drop=drop_rate, + drop_path=dpr[sum(depths[:i]):sum(depths[:i + 1])], + act_layer=act_layer, + norm_layer=norm_layer, + post_norm=post_norm, + downsample=(i < self.num_levels - 1), + downsample_layer = DownsampleLayer, + layer_scale=layer_scale, + offset_scale=offset_scale, + with_cp=with_cp, + mlp_fc2_bias=mlp_fc2_bias, + dcn_output_bias=dcn_output_bias, + dw_kernel_size=dw_kernel_size, # for InternImage-H/G + post_norm_block_ids=post_norm_block_ids, # for InternImage-H/G + res_post_norm=res_post_norm, # for InternImage-H/G + center_feature_scale=center_feature_scale # for InternImage-H/G + ) + self.levels.append(level) + + self.num_layers = len(depths) + self.apply(self._init_weights) + self.apply(self._init_deform_weights) + + def init_weights(self): + logger = get_root_logger() + if self.init_cfg is None: + logger.warn(f'No pre-trained weights for ' + f'{self.__class__.__name__}, ' + f'training start from scratch') + for m in self.modules(): + if isinstance(m, nn.Linear): + trunc_normal_init(m, std=.02, bias=0.) + elif isinstance(m, nn.LayerNorm): + constant_init(m, 1.0) + else: + assert 'checkpoint' in self.init_cfg, f'Only support ' \ + f'specify `Pretrained` in ' \ + f'`init_cfg` in ' \ + f'{self.__class__.__name__} ' + ckpt = _load_checkpoint(self.init_cfg.checkpoint, + logger=logger, + map_location='cpu') + if 'state_dict' in ckpt: + _state_dict = ckpt['state_dict'] + elif 'model_ema' in ckpt: + _state_dict = ckpt['model_ema'] + elif 'model' in ckpt: + _state_dict = ckpt['model'] + else: + _state_dict = ckpt + + state_dict = OrderedDict() + for k, v in _state_dict.items(): + if k.startswith('backbone.'): + state_dict[k[9:]] = v + else: + state_dict[k] = v + + # strip prefix of state_dict + if list(state_dict.keys())[0].startswith('module.'): + state_dict = {k[7:]: v for k, v in state_dict.items()} + + # load state_dict + meg = self.load_state_dict(state_dict, False) + logger.info(meg) + + def _init_weights(self, m): + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + + def _init_deform_weights(self, m): + if isinstance(m, getattr(DCNv4, self.core_op)): + m._reset_parameters() + + @torch.jit.ignore + def lr_decay_keywards(self, decay_ratio=0.87): + lr_ratios = {} + + # blocks + idx = 0 + for i in range(4): + layer_num = 3 - i # 3 2 1 0 + for j in range(self.depths[layer_num]): + block_num = self.depths[layer_num] - j - 1 + tag = 'levels.{}.blocks.{}.'.format(layer_num, block_num) + decay = 1.0 * (decay_ratio**idx) + lr_ratios[tag] = decay + idx += 1 + # patch_embed (before stage-1) + lr_ratios["patch_embed"] = lr_ratios['levels.0.blocks.0.'] + # levels.0.downsample (between stage-1 and stage-2) + lr_ratios["levels.0.downsample"] = lr_ratios['levels.1.blocks.0.'] + lr_ratios["levels.0.norm"] = lr_ratios['levels.1.blocks.0.'] + # levels.1.downsample (between stage-2 and stage-3) + lr_ratios["levels.1.downsample"] = lr_ratios['levels.2.blocks.0.'] + lr_ratios["levels.1.norm"] = lr_ratios['levels.2.blocks.0.'] + # levels.2.downsample (between stage-3 and stage-4) + lr_ratios["levels.2.downsample"] = lr_ratios['levels.3.blocks.0.'] + lr_ratios["levels.2.norm"] = lr_ratios['levels.3.blocks.0.'] + return lr_ratios + + def forward(self, x): + x = self.patch_embed(x) + N, H, W, C = x.shape + x = x.view(N, H*W, C) + + shape=(H, W) + seq_out = [] + for level_idx, level in enumerate(self.levels): + old_shape = shape + x, x_ , shape = level(x, return_wo_downsample=True, shape=shape, level_idx=level_idx) + if level_idx in self.out_indices: + h, w= old_shape + seq_out.append(x_.reshape(N, h, w, -1).permute(0, 3, 1, 2)) + return seq_out diff --git a/segmentation/mmseg_custom/models/builder.py b/segmentation/mmseg_custom/models/builder.py new file mode 100644 index 0000000..0c8e28d --- /dev/null +++ b/segmentation/mmseg_custom/models/builder.py @@ -0,0 +1,23 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import warnings # noqa: F401,F403 + +from mmcv.utils import Registry + +TRANSFORMER = Registry('Transformer') +MASK_ASSIGNERS = Registry('mask_assigner') +MATCH_COST = Registry('match_cost') + + +def build_match_cost(cfg): + """Build Match Cost.""" + return MATCH_COST.build(cfg) + + +def build_assigner(cfg): + """Build Assigner.""" + return MASK_ASSIGNERS.build(cfg) + + +def build_transformer(cfg): + """Build Transformer.""" + return TRANSFORMER.build(cfg) diff --git a/segmentation/mmseg_custom/models/decode_heads/__init__.py b/segmentation/mmseg_custom/models/decode_heads/__init__.py new file mode 100644 index 0000000..d0599d0 --- /dev/null +++ b/segmentation/mmseg_custom/models/decode_heads/__init__.py @@ -0,0 +1,8 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from .mask2former_head import Mask2FormerHead +from .maskformer_head import MaskFormerHead +from .msda import CustomMultiScaleDeformableAttention +__all__ = [ + 'MaskFormerHead', + 'Mask2FormerHead', +] diff --git a/segmentation/mmseg_custom/models/decode_heads/mask2former_head.py b/segmentation/mmseg_custom/models/decode_heads/mask2former_head.py new file mode 100644 index 0000000..719c547 --- /dev/null +++ b/segmentation/mmseg_custom/models/decode_heads/mask2former_head.py @@ -0,0 +1,579 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import copy + +import torch +import torch.nn as nn +import torch.nn.functional as F +from mmcv.cnn import Conv2d, build_plugin_layer, caffe2_xavier_init +from mmcv.cnn.bricks.transformer import (build_positional_encoding, + build_transformer_layer_sequence) +from mmcv.ops import point_sample +from mmcv.runner import ModuleList, force_fp32 +from mmseg.models.builder import HEADS, build_loss +from mmseg.models.decode_heads.decode_head import BaseDecodeHead + +from ...core import build_sampler, multi_apply, reduce_mean +from ..builder import build_assigner +from ..utils import get_uncertain_point_coords_with_randomness + + +@HEADS.register_module() +class Mask2FormerHead(BaseDecodeHead): + """Implements the Mask2Former head. + + See `Masked-attention Mask Transformer for Universal Image + Segmentation `_ for details. + + Args: + in_channels (list[int]): Number of channels in the input feature map. + feat_channels (int): Number of channels for features. + out_channels (int): Number of channels for output. + num_classes (int): Number of classes. + num_things_classes (int): Number of things. + num_stuff_classes (int): Number of stuff. + num_queries (int): Number of query in Transformer decoder. + pixel_decoder (:obj:`mmcv.ConfigDict` | dict): Config for pixel + decoder. Defaults to None. + enforce_decoder_input_project (bool, optional): Whether to add + a layer to change the embed_dim of tranformer encoder in + pixel decoder to the embed_dim of transformer decoder. + Defaults to False. + transformer_decoder (:obj:`mmcv.ConfigDict` | dict): Config for + transformer decoder. Defaults to None. + positional_encoding (:obj:`mmcv.ConfigDict` | dict): Config for + transformer decoder position encoding. Defaults to None. + loss_cls (:obj:`mmcv.ConfigDict` | dict): Config of the classification + loss. Defaults to None. + loss_mask (:obj:`mmcv.ConfigDict` | dict): Config of the mask loss. + Defaults to None. + loss_dice (:obj:`mmcv.ConfigDict` | dict): Config of the dice loss. + Defaults to None. + train_cfg (:obj:`mmcv.ConfigDict` | dict): Training config of + Mask2Former head. + test_cfg (:obj:`mmcv.ConfigDict` | dict): Testing config of + Mask2Former head. + init_cfg (dict or list[dict], optional): Initialization config dict. + Defaults to None. + """ + def __init__(self, + in_channels, + feat_channels, + out_channels, + num_classes=80, + num_things_classes=None, + num_stuff_classes=None, + num_queries=100, + num_transformer_feat_level=3, + pixel_decoder=None, + enforce_decoder_input_project=False, + transformer_decoder=None, + positional_encoding=None, + loss_cls=None, + loss_mask=None, + loss_dice=None, + train_cfg=None, + test_cfg=None, + init_cfg=None, + **kwargs): + super(Mask2FormerHead, self).__init__( + in_channels=in_channels, + channels=feat_channels, + num_classes=num_classes, + init_cfg=init_cfg, + input_transform='multiple_select', + **kwargs) + self.num_classes = num_classes + self.num_queries = num_queries + self.num_transformer_feat_level = num_transformer_feat_level + self.num_heads = transformer_decoder.transformerlayers. \ + attn_cfgs.num_heads + self.num_transformer_decoder_layers = transformer_decoder.num_layers + assert pixel_decoder.encoder.transformerlayers. \ + attn_cfgs.num_levels == num_transformer_feat_level + pixel_decoder_ = copy.deepcopy(pixel_decoder) + pixel_decoder_.update( + in_channels=in_channels, + feat_channels=feat_channels, + out_channels=out_channels) + self.pixel_decoder = build_plugin_layer(pixel_decoder_)[1] + self.transformer_decoder = build_transformer_layer_sequence( + transformer_decoder) + self.decoder_embed_dims = self.transformer_decoder.embed_dims + + self.decoder_input_projs = ModuleList() + # from low resolution to high resolution + for _ in range(num_transformer_feat_level): + if (self.decoder_embed_dims != feat_channels + or enforce_decoder_input_project): + self.decoder_input_projs.append( + Conv2d( + feat_channels, self.decoder_embed_dims, kernel_size=1)) + else: + self.decoder_input_projs.append(nn.Identity()) + self.decoder_positional_encoding = build_positional_encoding( + positional_encoding) + self.query_embed = nn.Embedding(self.num_queries, feat_channels) + self.query_feat = nn.Embedding(self.num_queries, feat_channels) + # from low resolution to high resolution + self.level_embed = nn.Embedding(self.num_transformer_feat_level, + feat_channels) + + self.cls_embed = nn.Linear(feat_channels, self.num_classes + 1) + self.mask_embed = nn.Sequential( + nn.Linear(feat_channels, feat_channels), nn.ReLU(inplace=True), + nn.Linear(feat_channels, feat_channels), nn.ReLU(inplace=True), + nn.Linear(feat_channels, out_channels)) + self.conv_seg = None # fix a bug here (conv_seg is not used) + + self.test_cfg = test_cfg + self.train_cfg = train_cfg + if train_cfg: + self.assigner = build_assigner(self.train_cfg.assigner) + self.sampler = build_sampler(self.train_cfg.sampler, context=self) + self.num_points = self.train_cfg.get('num_points', 12544) + self.oversample_ratio = self.train_cfg.get('oversample_ratio', 3.0) + self.importance_sample_ratio = self.train_cfg.get( + 'importance_sample_ratio', 0.75) + + self.class_weight = loss_cls.class_weight + self.loss_cls = build_loss(loss_cls) + self.loss_mask = build_loss(loss_mask) + self.loss_dice = build_loss(loss_dice) + + def init_weights(self): + for m in self.decoder_input_projs: + if isinstance(m, Conv2d): + caffe2_xavier_init(m, bias=0) + + self.pixel_decoder.init_weights() + + for p in self.transformer_decoder.parameters(): + if p.dim() > 1: + nn.init.xavier_normal_(p) + + def get_targets(self, cls_scores_list, mask_preds_list, gt_labels_list, + gt_masks_list, img_metas): + """Compute classification and mask targets for all images for a decoder + layer. + + Args: + cls_scores_list (list[Tensor]): Mask score logits from a single + decoder layer for all images. Each with shape [num_queries, + cls_out_channels]. + mask_preds_list (list[Tensor]): Mask logits from a single decoder + layer for all images. Each with shape [num_queries, h, w]. + gt_labels_list (list[Tensor]): Ground truth class indices for all + images. Each with shape (n, ), n is the sum of number of stuff + type and number of instance in a image. + gt_masks_list (list[Tensor]): Ground truth mask for each image, + each with shape (n, h, w). + img_metas (list[dict]): List of image meta information. + + Returns: + tuple[list[Tensor]]: a tuple containing the following targets. + + - labels_list (list[Tensor]): Labels of all images. + Each with shape [num_queries, ]. + - label_weights_list (list[Tensor]): Label weights of all + images.Each with shape [num_queries, ]. + - mask_targets_list (list[Tensor]): Mask targets of all images. + Each with shape [num_queries, h, w]. + - mask_weights_list (list[Tensor]): Mask weights of all images. + Each with shape [num_queries, ]. + - num_total_pos (int): Number of positive samples in all + images. + - num_total_neg (int): Number of negative samples in all + images. + """ + (labels_list, label_weights_list, mask_targets_list, mask_weights_list, + pos_inds_list, + neg_inds_list) = multi_apply(self._get_target_single, cls_scores_list, + mask_preds_list, gt_labels_list, + gt_masks_list, img_metas) + + num_total_pos = sum((inds.numel() for inds in pos_inds_list)) + num_total_neg = sum((inds.numel() for inds in neg_inds_list)) + return (labels_list, label_weights_list, mask_targets_list, + mask_weights_list, num_total_pos, num_total_neg) + + def _get_target_single(self, cls_score, mask_pred, gt_labels, gt_masks, + img_metas): + """Compute classification and mask targets for one image. + + Args: + cls_score (Tensor): Mask score logits from a single decoder layer + for one image. Shape (num_queries, cls_out_channels). + mask_pred (Tensor): Mask logits for a single decoder layer for one + image. Shape (num_queries, h, w). + gt_labels (Tensor): Ground truth class indices for one image with + shape (num_gts, ). + gt_masks (Tensor): Ground truth mask for each image, each with + shape (num_gts, h, w). + img_metas (dict): Image informtation. + + Returns: + tuple[Tensor]: A tuple containing the following for one image. + + - labels (Tensor): Labels of each image. \ + shape (num_queries, ). + - label_weights (Tensor): Label weights of each image. \ + shape (num_queries, ). + - mask_targets (Tensor): Mask targets of each image. \ + shape (num_queries, h, w). + - mask_weights (Tensor): Mask weights of each image. \ + shape (num_queries, ). + - pos_inds (Tensor): Sampled positive indices for each \ + image. + - neg_inds (Tensor): Sampled negative indices for each \ + image. + """ + # sample points + num_queries = cls_score.shape[0] + num_gts = gt_labels.shape[0] + + point_coords = torch.rand((1, self.num_points, 2), + device=cls_score.device) + # shape (num_queries, num_points) + mask_points_pred = point_sample( + mask_pred.unsqueeze(1), point_coords.repeat(num_queries, 1, + 1)).squeeze(1) + # shape (num_gts, num_points) + gt_points_masks = point_sample( + gt_masks.unsqueeze(1).float(), point_coords.repeat(num_gts, 1, + 1)).squeeze(1) + + # assign and sample + assign_result = self.assigner.assign(cls_score, mask_points_pred, + gt_labels, gt_points_masks, + img_metas) + sampling_result = self.sampler.sample(assign_result, mask_pred, + gt_masks) + pos_inds = sampling_result.pos_inds + neg_inds = sampling_result.neg_inds + + # label target + labels = gt_labels.new_full((self.num_queries, ), + self.num_classes, + dtype=torch.long) + labels[pos_inds] = gt_labels[sampling_result.pos_assigned_gt_inds] + label_weights = gt_labels.new_ones((self.num_queries, )) + + # mask target + mask_targets = gt_masks[sampling_result.pos_assigned_gt_inds] + mask_weights = mask_pred.new_zeros((self.num_queries, )) + mask_weights[pos_inds] = 1.0 + + return (labels, label_weights, mask_targets, mask_weights, pos_inds, + neg_inds) + + def loss_single(self, cls_scores, mask_preds, gt_labels_list, + gt_masks_list, img_metas): + """Loss function for outputs from a single decoder layer. + + Args: + cls_scores (Tensor): Mask score logits from a single decoder layer + for all images. Shape (batch_size, num_queries, + cls_out_channels). Note `cls_out_channels` should includes + background. + mask_preds (Tensor): Mask logits for a pixel decoder for all + images. Shape (batch_size, num_queries, h, w). + gt_labels_list (list[Tensor]): Ground truth class indices for each + image, each with shape (num_gts, ). + gt_masks_list (list[Tensor]): Ground truth mask for each image, + each with shape (num_gts, h, w). + img_metas (list[dict]): List of image meta information. + + Returns: + tuple[Tensor]: Loss components for outputs from a single \ + decoder layer. + """ + num_imgs = cls_scores.size(0) + cls_scores_list = [cls_scores[i] for i in range(num_imgs)] + mask_preds_list = [mask_preds[i] for i in range(num_imgs)] + (labels_list, label_weights_list, mask_targets_list, mask_weights_list, + num_total_pos, + num_total_neg) = self.get_targets(cls_scores_list, mask_preds_list, + gt_labels_list, gt_masks_list, + img_metas) + # shape (batch_size, num_queries) + labels = torch.stack(labels_list, dim=0) + # shape (batch_size, num_queries) + label_weights = torch.stack(label_weights_list, dim=0) + # shape (num_total_gts, h, w) + mask_targets = torch.cat(mask_targets_list, dim=0) + # shape (batch_size, num_queries) + mask_weights = torch.stack(mask_weights_list, dim=0) + + # classfication loss + # shape (batch_size * num_queries, ) + cls_scores = cls_scores.flatten(0, 1) + labels = labels.flatten(0, 1) + label_weights = label_weights.flatten(0, 1) + + class_weight = cls_scores.new_tensor(self.class_weight) + loss_cls = self.loss_cls( + cls_scores, + labels, + label_weights, + avg_factor=class_weight[labels].sum()) + + num_total_masks = reduce_mean(cls_scores.new_tensor([num_total_pos])) + num_total_masks = max(num_total_masks, 1) + + # extract positive ones + # shape (batch_size, num_queries, h, w) -> (num_total_gts, h, w) + mask_preds = mask_preds[mask_weights > 0] + + if mask_targets.shape[0] == 0: + # zero match + loss_dice = mask_preds.sum() + loss_mask = mask_preds.sum() + return loss_cls, loss_mask, loss_dice + + with torch.no_grad(): + points_coords = get_uncertain_point_coords_with_randomness( + mask_preds.unsqueeze(1), None, self.num_points, + self.oversample_ratio, self.importance_sample_ratio) + # shape (num_total_gts, h, w) -> (num_total_gts, num_points) + mask_point_targets = point_sample( + mask_targets.unsqueeze(1).float(), points_coords).squeeze(1) + # shape (num_queries, h, w) -> (num_queries, num_points) + mask_point_preds = point_sample( + mask_preds.unsqueeze(1), points_coords).squeeze(1) + + # dice loss + loss_dice = self.loss_dice( + mask_point_preds, mask_point_targets, avg_factor=num_total_masks) + + # mask loss + # shape (num_queries, num_points) -> (num_queries * num_points, ) + mask_point_preds = mask_point_preds.reshape(-1,1) + # shape (num_total_gts, num_points) -> (num_total_gts * num_points, ) + mask_point_targets = mask_point_targets.reshape(-1) + loss_mask = self.loss_mask( + mask_point_preds, + mask_point_targets, + avg_factor=num_total_masks * self.num_points) + + return loss_cls, loss_mask, loss_dice + + @force_fp32(apply_to=('all_cls_scores', 'all_mask_preds')) + def loss(self, all_cls_scores, all_mask_preds, gt_labels_list, + gt_masks_list, img_metas): + """Loss function. + + Args: + all_cls_scores (Tensor): Classification scores for all decoder + layers with shape [num_decoder, batch_size, num_queries, + cls_out_channels]. + all_mask_preds (Tensor): Mask scores for all decoder layers with + shape [num_decoder, batch_size, num_queries, h, w]. + gt_labels_list (list[Tensor]): Ground truth class indices for each + image with shape (n, ). n is the sum of number of stuff type + and number of instance in a image. + gt_masks_list (list[Tensor]): Ground truth mask for each image with + shape (n, h, w). + img_metas (list[dict]): List of image meta information. + + Returns: + dict[str, Tensor]: A dictionary of loss components. + """ + num_dec_layers = len(all_cls_scores) + all_gt_labels_list = [gt_labels_list for _ in range(num_dec_layers)] + all_gt_masks_list = [gt_masks_list for _ in range(num_dec_layers)] + img_metas_list = [img_metas for _ in range(num_dec_layers)] + losses_cls, losses_mask, losses_dice = multi_apply( + self.loss_single, all_cls_scores, all_mask_preds, + all_gt_labels_list, all_gt_masks_list, img_metas_list) + + loss_dict = dict() + # loss from the last decoder layer + loss_dict['loss_cls'] = losses_cls[-1] + loss_dict['loss_mask'] = losses_mask[-1] + loss_dict['loss_dice'] = losses_dice[-1] + # loss from other decoder layers + num_dec_layer = 0 + for loss_cls_i, loss_mask_i, loss_dice_i in zip( + losses_cls[:-1], losses_mask[:-1], losses_dice[:-1]): + loss_dict[f'd{num_dec_layer}.loss_cls'] = loss_cls_i + loss_dict[f'd{num_dec_layer}.loss_mask'] = loss_mask_i + loss_dict[f'd{num_dec_layer}.loss_dice'] = loss_dice_i + num_dec_layer += 1 + return loss_dict + + def forward_head(self, decoder_out, mask_feature, attn_mask_target_size): + """Forward for head part which is called after every decoder layer. + + Args: + decoder_out (Tensor): in shape (num_queries, batch_size, c). + mask_feature (Tensor): in shape (batch_size, c, h, w). + attn_mask_target_size (tuple[int, int]): target attention + mask size. + + Returns: + tuple: A tuple contain three elements. + + - cls_pred (Tensor): Classification scores in shape \ + (batch_size, num_queries, cls_out_channels). \ + Note `cls_out_channels` should includes background. + - mask_pred (Tensor): Mask scores in shape \ + (batch_size, num_queries,h, w). + - attn_mask (Tensor): Attention mask in shape \ + (batch_size * num_heads, num_queries, h, w). + """ + decoder_out = self.transformer_decoder.post_norm(decoder_out) + decoder_out = decoder_out.transpose(0, 1) + # shape (num_queries, batch_size, c) + cls_pred = self.cls_embed(decoder_out) + # shape (num_queries, batch_size, c) + mask_embed = self.mask_embed(decoder_out) + # shape (num_queries, batch_size, h, w) + mask_pred = torch.einsum('bqc,bchw->bqhw', mask_embed, mask_feature) + attn_mask = F.interpolate( + mask_pred, + attn_mask_target_size, + mode='bilinear', + align_corners=False) + # shape (num_queries, batch_size, h, w) -> + # (batch_size * num_head, num_queries, h, w) + attn_mask = attn_mask.flatten(2).unsqueeze(1).repeat( + (1, self.num_heads, 1, 1)).flatten(0, 1) + attn_mask = attn_mask.sigmoid() < 0.5 + attn_mask = attn_mask.detach() + + return cls_pred, mask_pred, attn_mask + + def forward(self, feats, img_metas): + """Forward function. + + Args: + feats (list[Tensor]): Multi scale Features from the + upstream network, each is a 4D-tensor. + img_metas (list[dict]): List of image information. + + Returns: + tuple: A tuple contains two elements. + + - cls_pred_list (list[Tensor)]: Classification logits \ + for each decoder layer. Each is a 3D-tensor with shape \ + (batch_size, num_queries, cls_out_channels). \ + Note `cls_out_channels` should includes background. + - mask_pred_list (list[Tensor]): Mask logits for each \ + decoder layer. Each with shape (batch_size, num_queries, \ + h, w). + """ + batch_size = len(img_metas) + mask_features, multi_scale_memorys = self.pixel_decoder(feats) + # multi_scale_memorys (from low resolution to high resolution) + decoder_inputs = [] + decoder_positional_encodings = [] + for i in range(self.num_transformer_feat_level): + decoder_input = self.decoder_input_projs[i](multi_scale_memorys[i]) + # shape (batch_size, c, h, w) -> (h*w, batch_size, c) + decoder_input = decoder_input.flatten(2).permute(2, 0, 1) + level_embed = self.level_embed.weight[i].view(1, 1, -1) + decoder_input = decoder_input + level_embed + # shape (batch_size, c, h, w) -> (h*w, batch_size, c) + mask = decoder_input.new_zeros( + (batch_size, ) + multi_scale_memorys[i].shape[-2:], + dtype=torch.bool) + decoder_positional_encoding = self.decoder_positional_encoding( + mask) + decoder_positional_encoding = decoder_positional_encoding.flatten( + 2).permute(2, 0, 1) + decoder_inputs.append(decoder_input) + decoder_positional_encodings.append(decoder_positional_encoding) + # shape (num_queries, c) -> (num_queries, batch_size, c) + query_feat = self.query_feat.weight.unsqueeze(1).repeat( + (1, batch_size, 1)) + query_embed = self.query_embed.weight.unsqueeze(1).repeat( + (1, batch_size, 1)) + + cls_pred_list = [] + mask_pred_list = [] + cls_pred, mask_pred, attn_mask = self.forward_head( + query_feat, mask_features, multi_scale_memorys[0].shape[-2:]) + cls_pred_list.append(cls_pred) + mask_pred_list.append(mask_pred) + + for i in range(self.num_transformer_decoder_layers): + level_idx = i % self.num_transformer_feat_level + # if a mask is all True(all background), then set it all False. + attn_mask[torch.where( + attn_mask.sum(-1) == attn_mask.shape[-1])] = False + + # cross_attn + self_attn + layer = self.transformer_decoder.layers[i] + attn_masks = [attn_mask, None] + query_feat = layer( + query=query_feat, + key=decoder_inputs[level_idx], + value=decoder_inputs[level_idx], + query_pos=query_embed, + key_pos=decoder_positional_encodings[level_idx], + attn_masks=attn_masks, + query_key_padding_mask=None, + # here we do not apply masking on padded region + key_padding_mask=None) + cls_pred, mask_pred, attn_mask = self.forward_head( + query_feat, mask_features, multi_scale_memorys[ + (i + 1) % self.num_transformer_feat_level].shape[-2:]) + + cls_pred_list.append(cls_pred) + mask_pred_list.append(mask_pred) + + return cls_pred_list, mask_pred_list + + def forward_train(self, x, img_metas, gt_semantic_seg, gt_labels, + gt_masks): + """Forward function for training mode. + + Args: + x (list[Tensor]): Multi-level features from the upstream network, + each is a 4D-tensor. + img_metas (list[Dict]): List of image information. + gt_semantic_seg (list[tensor]):Each element is the ground truth + of semantic segmentation with the shape (N, H, W). + train_cfg (dict): The training config, which not been used in + maskformer. + gt_labels (list[Tensor]): Each element is ground truth labels of + each box, shape (num_gts,). + gt_masks (list[BitmapMasks]): Each element is masks of instances + of a image, shape (num_gts, h, w). + + Returns: + losses (dict[str, Tensor]): a dictionary of loss components + """ + + # forward + all_cls_scores, all_mask_preds = self(x, img_metas) + + # loss + losses = self.loss(all_cls_scores, all_mask_preds, gt_labels, gt_masks, + img_metas) + + return losses + + def forward_test(self, inputs, img_metas, test_cfg): + """Test segment without test-time aumengtation. + + Only the output of last decoder layers was used. + + Args: + inputs (list[Tensor]): Multi-level features from the + upstream network, each is a 4D-tensor. + img_metas (list[dict]): List of image information. + test_cfg (dict): Testing config. + + Returns: + seg_mask (Tensor): Predicted semantic segmentation logits. + """ + all_cls_scores, all_mask_preds = self(inputs, img_metas) + cls_score, mask_pred = all_cls_scores[-1], all_mask_preds[-1] + ori_h, ori_w, _ = img_metas[0]['ori_shape'] + + # semantic inference + cls_score = F.softmax(cls_score, dim=-1)[..., :-1] + mask_pred = mask_pred.sigmoid() + seg_mask = torch.einsum('bqc,bqhw->bchw', cls_score, mask_pred) + return seg_mask diff --git a/segmentation/mmseg_custom/models/decode_heads/maskformer_head.py b/segmentation/mmseg_custom/models/decode_heads/maskformer_head.py new file mode 100644 index 0000000..aa8509a --- /dev/null +++ b/segmentation/mmseg_custom/models/decode_heads/maskformer_head.py @@ -0,0 +1,519 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +from mmcv.cnn import Conv2d, build_plugin_layer, kaiming_init +from mmcv.cnn.bricks.transformer import (build_positional_encoding, + build_transformer_layer_sequence) +from mmcv.runner import force_fp32 +from mmseg.models.builder import HEADS, build_loss +from mmseg.models.decode_heads.decode_head import BaseDecodeHead + +from ...core import multi_apply, reduce_mean +from ..builder import build_assigner, build_transformer + + +@HEADS.register_module() +class MaskFormerHead(BaseDecodeHead): + """Implements the MaskFormer head. + + See `paper: Per-Pixel Classification is Not All You Need + for Semantic Segmentation` + for details. + + Args: + in_channels (list[int]): Number of channels in the input feature map. + feat_channels (int): Number channels for feature. + out_channels (int): Number channels for output. + num_things_classes (int): Number of things. + num_stuff_classes (int): Number of stuff. + num_queries (int): Number of query in Transformer. + pixel_decoder (obj:`mmcv.ConfigDict`|dict): Config for pixel decoder. + Defaults to None. + enforce_decoder_input_project (bool, optional): Whether to add a layer + to change the embed_dim of tranformer encoder in pixel decoder to + the embed_dim of transformer decoder. Defaults to False. + transformer_decoder (obj:`mmcv.ConfigDict`|dict): Config for + transformer decoder. Defaults to None. + positional_encoding (obj:`mmcv.ConfigDict`|dict): Config for + transformer decoder position encoding. Defaults to None. + loss_cls (obj:`mmcv.ConfigDict`|dict): Config of the classification + loss. Defaults to `CrossEntropyLoss`. + loss_mask (obj:`mmcv.ConfigDict`|dict): Config of the mask loss. + Defaults to `FocalLoss`. + loss_dice (obj:`mmcv.ConfigDict`|dict): Config of the dice loss. + Defaults to `DiceLoss`. + train_cfg (obj:`mmcv.ConfigDict`|dict): Training config of Maskformer + head. + test_cfg (obj:`mmcv.ConfigDict`|dict): Testing config of Maskformer + head. + init_cfg (dict or list[dict], optional): Initialization config dict. + Defaults to None. + """ + def __init__(self, + out_channels, + num_queries=100, + pixel_decoder=None, + enforce_decoder_input_project=False, + transformer_decoder=None, + positional_encoding=None, + loss_cls=dict( + type='CrossEntropyLoss', + bg_cls_weight=0.1, + use_sigmoid=False, + loss_weight=1.0, + class_weight=1.0), + loss_mask=dict( + type='FocalLoss', + use_sigmoid=True, + gamma=2.0, + alpha=0.25, + loss_weight=20.0), + loss_dice=dict( + type='DiceLoss', + use_sigmoid=True, + activate=True, + naive_dice=True, + loss_weight=1.0), + assigner=dict( + type='MaskHungarianAssigner', + cls_cost=dict(type='ClassificationCost', weight=1.), + dice_cost=dict(type='DiceCost', weight=1.0, pred_act=True, + eps=1.0), + mask_cost=dict(type='MaskFocalLossCost', weight=20.0)), + **kwargs): + super(MaskFormerHead, self).__init__(input_transform='multiple_select', + **kwargs) + self.num_queries = num_queries + + pixel_decoder.update( + in_channels=self.in_channels, + feat_channels=self.channels, + out_channels=out_channels) + self.pixel_decoder = build_plugin_layer(pixel_decoder)[1] + self.transformer_decoder = build_transformer_layer_sequence( + transformer_decoder) + self.decoder_embed_dims = self.transformer_decoder.embed_dims + pixel_decoder_type = pixel_decoder.get('type') + if pixel_decoder_type == 'PixelDecoder' and ( + self.decoder_embed_dims != self.in_channels[-1] + or enforce_decoder_input_project): + self.decoder_input_proj = Conv2d( + self.in_channels[-1], self.decoder_embed_dims, kernel_size=1) + else: + self.decoder_input_proj = nn.Identity() + self.decoder_pe = build_positional_encoding(positional_encoding) + self.query_embed = nn.Embedding(self.num_queries, out_channels) + + self.cls_embed = nn.Linear(self.channels, self.num_classes + 1) + self.mask_embed = nn.Sequential( + nn.Linear(self.channels, self.channels), nn.ReLU(inplace=True), + nn.Linear(self.channels, self.channels), nn.ReLU(inplace=True), + nn.Linear(self.channels, out_channels)) + + self.assigner = build_assigner(assigner) + + self.bg_cls_weight = 0 + class_weight = loss_cls.get('class_weight', None) + if class_weight is not None and (self.__class__ is MaskFormerHead): + assert isinstance(class_weight, float), 'Expected ' \ + 'class_weight to have type float. Found ' \ + f'{type(class_weight)}.' + # NOTE following the official MaskFormerHead repo, bg_cls_weight + # means relative classification weight of the VOID class. + bg_cls_weight = loss_cls.get('bg_cls_weight', class_weight) + assert isinstance(bg_cls_weight, float), 'Expected ' \ + 'bg_cls_weight to have type float. Found ' \ + f'{type(bg_cls_weight)}.' + class_weight = (self.num_classes + 1) * [class_weight] + # set VOID class as the last indice + class_weight[self.num_classes] = bg_cls_weight + loss_cls.update({'class_weight': class_weight}) + if 'bg_cls_weight' in loss_cls: + loss_cls.pop('bg_cls_weight') + self.bg_cls_weight = bg_cls_weight + + assert loss_cls['loss_weight'] == assigner['cls_cost']['weight'], \ + 'The classification weight for loss and matcher should be' \ + 'exactly the same.' + assert loss_dice['loss_weight'] == assigner['dice_cost']['weight'], \ + f'The dice weight for loss and matcher' \ + f'should be exactly the same.' + assert loss_mask['loss_weight'] == assigner['mask_cost']['weight'], \ + 'The focal weight for loss and matcher should be' \ + 'exactly the same.' + self.loss_cls = build_loss(loss_cls) + self.loss_mask = build_loss(loss_mask) + self.loss_dice = build_loss(loss_dice) + + self.init_weights() + + def init_weights(self): + kaiming_init(self.decoder_input_proj, a=1) + + def get_targets(self, cls_scores_list, mask_preds_list, gt_labels_list, + gt_masks_list, img_metas): + """Compute classification and mask targets for all images for a decoder + layer. + + Args: + cls_scores_list (list[Tensor]): Mask score logits from a single + decoder layer for all images. Each with shape [num_queries, + cls_out_channels]. + mask_preds_list (list[Tensor]): Mask logits from a single decoder + layer for all images. Each with shape [num_queries, h, w]. + gt_labels_list (list[Tensor]): Ground truth class indices for all + images. Each with shape (n, ), n is the sum of number of stuff + type and number of instance in a image. + gt_masks_list (list[Tensor]): Ground truth mask for each image, + each with shape (n, h, w). + img_metas (list[dict]): List of image meta information. + + Returns: + tuple[list[Tensor]]: a tuple containing the following targets. + + - labels_list (list[Tensor]): Labels of all images. + Each with shape [num_queries, ]. + - label_weights_list (list[Tensor]): Label weights of all + images.Each with shape [num_queries, ]. + - mask_targets_list (list[Tensor]): Mask targets of all images. + Each with shape [num_queries, h, w]. + - mask_weights_list (list[Tensor]): Mask weights of all images. + Each with shape [num_queries, ]. + - num_total_pos (int): Number of positive samples in all + images. + - num_total_neg (int): Number of negative samples in all + images. + """ + (labels_list, label_weights_list, mask_targets_list, mask_weights_list, + pos_inds_list, + neg_inds_list) = multi_apply(self._get_target_single, cls_scores_list, + mask_preds_list, gt_labels_list, + gt_masks_list, img_metas) + + num_total_pos = sum((inds.numel() for inds in pos_inds_list)) + num_total_neg = sum((inds.numel() for inds in neg_inds_list)) + return (labels_list, label_weights_list, mask_targets_list, + mask_weights_list, num_total_pos, num_total_neg) + + def _get_target_single(self, cls_score, mask_pred, gt_labels, gt_masks, + img_metas): + """Compute classification and mask targets for one image. + + Args: + cls_score (Tensor): Mask score logits from a single decoder layer + for one image. Shape [num_queries, cls_out_channels]. + mask_pred (Tensor): Mask logits for a single decoder layer for one + image. Shape [num_queries, h, w]. + gt_labels (Tensor): Ground truth class indices for one image with + shape (n, ). n is the sum of number of stuff type and number + of instance in a image. + gt_masks (Tensor): Ground truth mask for each image, each with + shape (n, h, w). + img_metas (dict): Image informtation. + + Returns: + tuple[Tensor]: a tuple containing the following for one image. + + - labels (Tensor): Labels of each image. + shape [num_queries, ]. + - label_weights (Tensor): Label weights of each image. + shape [num_queries, ]. + - mask_targets (Tensor): Mask targets of each image. + shape [num_queries, h, w]. + - mask_weights (Tensor): Mask weights of each image. + shape [num_queries, ]. + - pos_inds (Tensor): Sampled positive indices for each image. + - neg_inds (Tensor): Sampled negative indices for each image. + """ + target_shape = mask_pred.shape[-2:] + gt_masks_downsampled = F.interpolate( + gt_masks.unsqueeze(1).float(), target_shape, + mode='nearest').squeeze(1).long() + # assign and sample + assign_result = self.assigner.assign(cls_score, mask_pred, gt_labels, + gt_masks_downsampled, img_metas) + # pos_ind: range from 1 to (self.num_classes) + # which represents the positive index + pos_inds = torch.nonzero(assign_result.gt_inds > 0, + as_tuple=False).squeeze(-1).unique() + neg_inds = torch.nonzero(assign_result.gt_inds == 0, + as_tuple=False).squeeze(-1).unique() + pos_assigned_gt_inds = assign_result.gt_inds[pos_inds] - 1 + + # label target + labels = gt_labels.new_full((self.num_queries, ), + self.num_classes, + dtype=torch.long) + labels[pos_inds] = gt_labels[pos_assigned_gt_inds] + label_weights = gt_labels.new_ones(self.num_queries) + + # mask target + mask_targets = gt_masks[pos_assigned_gt_inds, :] + mask_weights = mask_pred.new_zeros((self.num_queries, )) + mask_weights[pos_inds] = 1.0 + + return (labels, label_weights, mask_targets, mask_weights, pos_inds, + neg_inds) + + @force_fp32(apply_to=('all_cls_scores', 'all_mask_preds')) + def loss(self, all_cls_scores, all_mask_preds, gt_labels_list, + gt_masks_list, img_metas): + """Loss function. + + Args: + all_cls_scores (Tensor): Classification scores for all decoder + layers with shape [num_decoder, batch_size, num_queries, + cls_out_channels]. + all_mask_preds (Tensor): Mask scores for all decoder layers with + shape [num_decoder, batch_size, num_queries, h, w]. + gt_labels_list (list[Tensor]): Ground truth class indices for each + image with shape (n, ). n is the sum of number of stuff type + and number of instance in a image. + gt_masks_list (list[Tensor]): Ground truth mask for each image with + shape (n, h, w). + img_metas (list[dict]): List of image meta information. + + Returns: + dict[str, Tensor]: A dictionary of loss components. + """ + num_dec_layers = len(all_cls_scores) + all_gt_labels_list = [gt_labels_list for _ in range(num_dec_layers)] + all_gt_masks_list = [gt_masks_list for _ in range(num_dec_layers)] + img_metas_list = [img_metas for _ in range(num_dec_layers)] + losses_cls, losses_mask, losses_dice = multi_apply( + self.loss_single, all_cls_scores, all_mask_preds, + all_gt_labels_list, all_gt_masks_list, img_metas_list) + + loss_dict = dict() + # loss from the last decoder layer + loss_dict['loss_cls'] = losses_cls[-1] + loss_dict['loss_mask'] = losses_mask[-1] + loss_dict['loss_dice'] = losses_dice[-1] + # loss from other decoder layers + num_dec_layer = 0 + for loss_cls_i, loss_mask_i, loss_dice_i in zip( + losses_cls[:-1], losses_mask[:-1], losses_dice[:-1]): + loss_dict[f'd{num_dec_layer}.loss_cls'] = loss_cls_i + loss_dict[f'd{num_dec_layer}.loss_mask'] = loss_mask_i + loss_dict[f'd{num_dec_layer}.loss_dice'] = loss_dice_i + num_dec_layer += 1 + return loss_dict + + def loss_single(self, cls_scores, mask_preds, gt_labels_list, + gt_masks_list, img_metas): + """Loss function for outputs from a single decoder layer. + + Args: + cls_scores (Tensor): Mask score logits from a single decoder layer + for all images. Shape [batch_size, num_queries, + cls_out_channels]. + mask_preds (Tensor): Mask logits for a pixel decoder for all + images. Shape [batch_size, num_queries, h, w]. + gt_labels_list (list[Tensor]): Ground truth class indices for each + image, each with shape (n, ). n is the sum of number of stuff + types and number of instances in a image. + gt_masks_list (list[Tensor]): Ground truth mask for each image, + each with shape (n, h, w). + img_metas (list[dict]): List of image meta information. + + Returns: + tuple[Tensor]:Loss components for outputs from a single decoder + layer. + """ + num_imgs = cls_scores.size(0) + cls_scores_list = [cls_scores[i] for i in range(num_imgs)] + mask_preds_list = [mask_preds[i] for i in range(num_imgs)] + + (labels_list, label_weights_list, mask_targets_list, mask_weights_list, + num_total_pos, + num_total_neg) = self.get_targets(cls_scores_list, mask_preds_list, + gt_labels_list, gt_masks_list, + img_metas) + # shape [batch_size, num_queries] + labels = torch.stack(labels_list, dim=0) + # shape [batch_size, num_queries] + label_weights = torch.stack(label_weights_list, dim=0) + # shape [num_gts, h, w] + mask_targets = torch.cat(mask_targets_list, dim=0) + # shape [batch_size, num_queries] + mask_weights = torch.stack(mask_weights_list, dim=0) + + # classfication loss + # shape [batch_size * num_queries, ] + cls_scores = cls_scores.flatten(0, 1) + # shape [batch_size * num_queries, ] + labels = labels.flatten(0, 1) + # shape [batch_size* num_queries, ] + label_weights = label_weights.flatten(0, 1) + + class_weight = cls_scores.new_ones(self.num_classes + 1) + class_weight[-1] = self.bg_cls_weight + + loss_cls = self.loss_cls( + cls_scores, + labels, + label_weights, + avg_factor=class_weight[labels].sum()) + + num_total_masks = reduce_mean(cls_scores.new_tensor([num_total_pos])) + num_total_masks = max(num_total_masks, 1) + + # extract positive ones + mask_preds = mask_preds[mask_weights > 0] + target_shape = mask_targets.shape[-2:] + + if mask_targets.shape[0] == 0: + # zero match + loss_dice = mask_preds.sum() + loss_mask = mask_preds.sum() + return loss_cls, loss_mask, loss_dice + + # upsample to shape of target + # shape [num_gts, h, w] + mask_preds = F.interpolate( + mask_preds.unsqueeze(1), + target_shape, + mode='bilinear', + align_corners=False).squeeze(1) + + # dice loss + loss_dice = self.loss_dice( + mask_preds, mask_targets, avg_factor=num_total_masks) + + # mask loss + # FocalLoss support input of shape [n, num_class] + h, w = mask_preds.shape[-2:] + # shape [num_gts, h, w] -> [num_gts * h * w, 1] + mask_preds = mask_preds.reshape(-1, 1) + # shape [num_gts, h, w] -> [num_gts * h * w] + mask_targets = mask_targets.reshape(-1) + # target is (1 - mask_targets) !!! + print("mask_pred:", mask_preds.shape) + print("mask_targets:", mask_targets.shape) + loss_mask = self.loss_mask( + mask_preds, 1 - mask_targets, avg_factor=num_total_masks * h * w) + + return loss_cls, loss_mask, loss_dice + + def forward(self, feats, img_metas): + """Forward function. + + Args: + feats (list[Tensor]): Features from the upstream network, each + is a 4D-tensor. + img_metas (list[dict]): List of image information. + + Returns: + all_cls_scores (Tensor): Classification scores for each + scale level. Each is a 4D-tensor with shape + [num_decoder, batch_size, num_queries, cls_out_channels]. + Note `cls_out_channels` should includes background. + all_mask_preds (Tensor): Mask scores for each decoder + layer. Each with shape [num_decoder, batch_size, + num_queries, h, w]. + """ + batch_size = len(img_metas) + input_img_h, input_img_w = img_metas[0]['pad_shape'][:-1] + # input_img_h, input_img_w = img_metas[0]['batch_input_shape'] + padding_mask = feats[-1].new_ones( + (batch_size, input_img_h, input_img_w), dtype=torch.float32) + for i in range(batch_size): + img_h, img_w, _ = img_metas[i]['img_shape'] + padding_mask[i, :img_h, :img_w] = 0 + padding_mask = F.interpolate( + padding_mask.unsqueeze(1), + size=feats[-1].shape[-2:], + mode='nearest').to(torch.bool).squeeze(1) + # when backbone is swin, memory is output of last stage of swin. + # when backbone is r50, memory is output of tranformer encoder. + mask_features, memory = self.pixel_decoder(feats, img_metas) + pos_embed = self.decoder_pe(padding_mask) + memory = self.decoder_input_proj(memory) + # shape [batch_size, c, h, w] -> [h*w, batch_size, c] + memory = memory.flatten(2).permute(2, 0, 1) + pos_embed = pos_embed.flatten(2).permute(2, 0, 1) + # shape [batch_size, h * w] + padding_mask = padding_mask.flatten(1) + # shape = [num_queries, embed_dims] + query_embed = self.query_embed.weight + # shape = [num_queries, batch_size, embed_dims] + query_embed = query_embed.unsqueeze(1).repeat(1, batch_size, 1) + target = torch.zeros_like(query_embed) + # shape [num_decoder, num_queries, batch_size, embed_dims] + out_dec = self.transformer_decoder( + query=target, + key=memory, + value=memory, + key_pos=pos_embed, + query_pos=query_embed, + key_padding_mask=padding_mask) + # shape [num_decoder, batch_size, num_queries, embed_dims] + out_dec = out_dec.transpose(1, 2) + + # cls_scores + all_cls_scores = self.cls_embed(out_dec) + + # mask_preds + mask_embed = self.mask_embed(out_dec) + all_mask_preds = torch.einsum('lbqc,bchw->lbqhw', mask_embed, + mask_features) + + return all_cls_scores, all_mask_preds + + def forward_train(self, + x, + img_metas, + gt_semantic_seg, + gt_labels, + gt_masks): + """Forward function for training mode. + + Args: + x (list[Tensor]): Multi-level features from the upstream network, + each is a 4D-tensor. + img_metas (list[Dict]): List of image information. + gt_semantic_seg (list[tensor]):Each element is the ground truth + of semantic segmentation with the shape (N, H, W). + train_cfg (dict): The training config, which not been used in + maskformer. + gt_labels (list[Tensor]): Each element is ground truth labels of + each box, shape (num_gts,). + gt_masks (list[BitmapMasks]): Each element is masks of instances + of a image, shape (num_gts, h, w). + + Returns: + losses (dict[str, Tensor]): a dictionary of loss components + """ + + # forward + all_cls_scores, all_mask_preds = self(x, img_metas) + + # loss + losses = self.loss(all_cls_scores, all_mask_preds, gt_labels, gt_masks, + img_metas) + + return losses + + def forward_test(self, inputs, img_metas, test_cfg): + """Test segment without test-time aumengtation. + + Only the output of last decoder layers was used. + + Args: + inputs (list[Tensor]): Multi-level features from the + upstream network, each is a 4D-tensor. + img_metas (list[dict]): List of image information. + test_cfg (dict): Testing config. + + Returns: + seg_mask (Tensor): Predicted semantic segmentation logits. + """ + all_cls_scores, all_mask_preds = self(inputs, img_metas) + cls_score, mask_pred = all_cls_scores[-1], all_mask_preds[-1] + ori_h, ori_w, _ = img_metas[0]['ori_shape'] + + # semantic inference + cls_score = F.softmax(cls_score, dim=-1)[..., :-1] + mask_pred = mask_pred.sigmoid() + seg_mask = torch.einsum('bqc,bqhw->bchw', cls_score, mask_pred) + return seg_mask diff --git a/segmentation/mmseg_custom/models/decode_heads/msda.py b/segmentation/mmseg_custom/models/decode_heads/msda.py new file mode 100644 index 0000000..0a4d755 --- /dev/null +++ b/segmentation/mmseg_custom/models/decode_heads/msda.py @@ -0,0 +1,355 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + + +import torch +from torch.cuda.amp import custom_bwd, custom_fwd +from torch.autograd.function import Function, once_differentiable +from mmcv.utils import ext_loader +ext_module = ext_loader.load_ext( + '_ext', ['ms_deform_attn_backward', 'ms_deform_attn_forward']) + +class MultiScaleDeformableAttnFunction_fp16(Function): + + @staticmethod + @custom_fwd(cast_inputs=torch.float16) + def forward(ctx, value, value_spatial_shapes, value_level_start_index, + sampling_locations, attention_weights, im2col_step): + """GPU version of multi-scale deformable attention. + + Args: + value (Tensor): The value has shape + (bs, num_keys, mum_heads, embed_dims//num_heads) + value_spatial_shapes (Tensor): Spatial shape of + each feature map, has shape (num_levels, 2), + last dimension 2 represent (h, w) + sampling_locations (Tensor): The location of sampling points, + has shape + (bs ,num_queries, num_heads, num_levels, num_points, 2), + the last dimension 2 represent (x, y). + attention_weights (Tensor): The weight of sampling points used + when calculate the attention, has shape + (bs ,num_queries, num_heads, num_levels, num_points), + im2col_step (Tensor): The step used in image to column. + + Returns: + Tensor: has shape (bs, num_queries, embed_dims) + """ + ctx.im2col_step = im2col_step + output = ext_module.ms_deform_attn_forward( + value, + value_spatial_shapes, + value_level_start_index, + sampling_locations, + attention_weights, + im2col_step=ctx.im2col_step) + ctx.save_for_backward(value, value_spatial_shapes, + value_level_start_index, sampling_locations, + attention_weights) + return output + + @staticmethod + @once_differentiable + @custom_bwd + def backward(ctx, grad_output): + """GPU version of backward function. + + Args: + grad_output (Tensor): Gradient + of output tensor of forward. + + Returns: + Tuple[Tensor]: Gradient + of input tensors in forward. + """ + value, value_spatial_shapes, value_level_start_index, \ + sampling_locations, attention_weights = ctx.saved_tensors + grad_value = torch.zeros_like(value) + grad_sampling_loc = torch.zeros_like(sampling_locations) + grad_attn_weight = torch.zeros_like(attention_weights) + + ext_module.ms_deform_attn_backward( + value, + value_spatial_shapes, + value_level_start_index, + sampling_locations, + attention_weights, + grad_output.contiguous(), + grad_value, + grad_sampling_loc, + grad_attn_weight, + im2col_step=ctx.im2col_step) + + return grad_value, None, None, \ + grad_sampling_loc, grad_attn_weight, None + + + + +class MultiScaleDeformableAttnFunction_fp32_old(Function): + + @staticmethod + @custom_fwd(cast_inputs=torch.float32) + def forward(ctx, value, value_spatial_shapes, value_level_start_index, + sampling_locations, attention_weights, im2col_step): + """GPU version of multi-scale deformable attention. + + Args: + value (Tensor): The value has shape + (bs, num_keys, mum_heads, embed_dims//num_heads) + value_spatial_shapes (Tensor): Spatial shape of + each feature map, has shape (num_levels, 2), + last dimension 2 represent (h, w) + sampling_locations (Tensor): The location of sampling points, + has shape + (bs ,num_queries, num_heads, num_levels, num_points, 2), + the last dimension 2 represent (x, y). + attention_weights (Tensor): The weight of sampling points used + when calculate the attention, has shape + (bs ,num_queries, num_heads, num_levels, num_points), + im2col_step (Tensor): The step used in image to column. + + Returns: + Tensor: has shape (bs, num_queries, embed_dims) + """ + + ctx.im2col_step = im2col_step + output = ext_module.ms_deform_attn_forward( + value.to(torch.float), + value_spatial_shapes, + value_level_start_index, + sampling_locations.to(torch.float), + attention_weights.to(torch.float), + im2col_step=ctx.im2col_step).to(torch.float16) + ctx.save_for_backward(value, value_spatial_shapes, + value_level_start_index, sampling_locations, + attention_weights) + return output + + @staticmethod + @once_differentiable + @custom_bwd + def backward(ctx, grad_output): + """GPU version of backward function. + + Args: + grad_output (Tensor): Gradient + of output tensor of forward. + + Returns: + Tuple[Tensor]: Gradient + of input tensors in forward. + """ + value, value_spatial_shapes, value_level_start_index, \ + sampling_locations, attention_weights = ctx.saved_tensors + grad_value = torch.zeros_like(value) + grad_sampling_loc = torch.zeros_like(sampling_locations) + grad_attn_weight = torch.zeros_like(attention_weights) + + ext_module.ms_deform_attn_backward( + value, + value_spatial_shapes, + value_level_start_index, + sampling_locations, + attention_weights, + grad_output.contiguous(), + grad_value, + grad_sampling_loc, + grad_attn_weight, + im2col_step=ctx.im2col_step) + + return grad_value, None, None, \ + grad_sampling_loc, grad_attn_weight, None + + +# Copyright (c) OpenMMLab. All rights reserved. +import math +import warnings + +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.autograd.function import Function, once_differentiable + +from mmcv import deprecated_api_warning +from mmcv.cnn import constant_init, xavier_init +from mmcv.cnn.bricks.registry import ATTENTION +from mmcv.runner import BaseModule + +ext_module = ext_loader.load_ext( + '_ext', ['ms_deform_attn_backward', 'ms_deform_attn_forward']) +import functools +import time +from collections import defaultdict +import torch +from mmcv.ops import MultiScaleDeformableAttention +@ATTENTION.register_module() +class CustomMultiScaleDeformableAttention(MultiScaleDeformableAttention): + """An attention module used in Deformable-Detr. + + `Deformable DETR: Deformable Transformers for End-to-End Object Detection. + `_. + + Args: + embed_dims (int): The embedding dimension of Attention. + Default: 256. + num_heads (int): Parallel attention heads. Default: 64. + num_levels (int): The number of feature map used in + Attention. Default: 4. + num_points (int): The number of sampling points for + each query in each head. Default: 4. + im2col_step (int): The step used in image_to_column. + Default: 64. + dropout (float): A Dropout layer on `inp_identity`. + Default: 0.1. + batch_first (bool): Key, Query and Value are shape of + (batch, n, embed_dim) + or (n, batch, embed_dim). Default to False. + norm_cfg (dict): Config dict for normalization layer. + Default: None. + init_cfg (obj:`mmcv.ConfigDict`): The Config for initialization. + Default: None. + """ + + def __init__(self, + use_flash=False, + use_softmax=True, + **kwargs +): + super().__init__(**kwargs) + + self.use_flash = use_flash + self.use_softmax = use_softmax + + @deprecated_api_warning({'residual': 'identity'}, + cls_name='FlashMultiScaleDeformableAttention') + # @run_time('ms_attention') + def forward(self, + query, + key=None, + value=None, + identity=None, + query_pos=None, + key_padding_mask=None, + reference_points=None, + spatial_shapes=None, + level_start_index=None, + **kwargs): + """Forward Function of MultiScaleDeformAttention. + + Args: + query (torch.Tensor): Query of Transformer with shape + (num_query, bs, embed_dims). + key (torch.Tensor): The key tensor with shape + `(num_key, bs, embed_dims)`. + value (torch.Tensor): The value tensor with shape + `(num_key, bs, embed_dims)`. + identity (torch.Tensor): The tensor used for addition, with the + same shape as `query`. Default None. If None, + `query` will be used. + query_pos (torch.Tensor): The positional encoding for `query`. + Default: None. + key_pos (torch.Tensor): The positional encoding for `key`. Default + None. + reference_points (torch.Tensor): The normalized reference + points with shape (bs, num_query, num_levels, 2), + all elements is range in [0, 1], top-left (0,0), + bottom-right (1, 1), including padding area. + or (N, Length_{query}, num_levels, 4), add + additional two dimensions is (w, h) to + form reference boxes. + key_padding_mask (torch.Tensor): ByteTensor for `query`, with + shape [bs, num_key]. + spatial_shapes (torch.Tensor): Spatial shape of features in + different levels. With shape (num_levels, 2), + last dimension represents (h, w). + level_start_index (torch.Tensor): The start index of each level. + A tensor has shape ``(num_levels, )`` and can be represented + as [0, h_0*w_0, h_0*w_0+h_1*w_1, ...]. + + Returns: + torch.Tensor: forwarded results with shape + [num_query, bs, embed_dims]. + """ + + if value is None: + value = query + + if identity is None: + identity = query + if query_pos is not None: + query = query + query_pos + if not self.batch_first: + # change to (bs, num_query ,embed_dims) + query = query.permute(1, 0, 2) + value = value.permute(1, 0, 2) + + bs, num_query, _ = query.shape + bs, num_value, _ = value.shape + assert (spatial_shapes[:, 0] * spatial_shapes[:, 1]).sum() == num_value + + value = self.value_proj(value) + if key_padding_mask is not None: + value = value.masked_fill(key_padding_mask[..., None], 0.0) + value = value.view(bs, num_value, self.num_heads, -1) + query = query.to(value.dtype) + sampling_offsets = self.sampling_offsets(query).view( + bs, num_query, self.num_heads, self.num_levels, self.num_points, 2) + attention_weights = self.attention_weights(query).view( + bs, num_query, self.num_heads, self.num_levels * self.num_points) + + if not self.use_flash: + if self.use_softmax: + attention_weights = attention_weights.softmax(-1) + attention_weights = attention_weights.view(bs, num_query, + self.num_heads, + self.num_levels, + self.num_points) + + else: + attention_weights = attention_weights.view(bs, num_query, + self.num_heads, + self.num_levels, + self.num_points, 1) + + if reference_points.shape[-1] == 2: + offset_normalizer = torch.stack( + [spatial_shapes[..., 1], spatial_shapes[..., 0]], -1) + sampling_locations = reference_points[:, :, None, :, None, :] \ + + sampling_offsets \ + / offset_normalizer[None, None, None, :, None, :] + elif reference_points.shape[-1] == 4: + sampling_locations = reference_points[:, :, None, :, None, :2] \ + + sampling_offsets / self.num_points \ + * reference_points[:, :, None, :, None, 2:] \ + * 0.5 + else: + raise ValueError( + f'Last dim of reference_points must be' + f' 2 or 4, but get {reference_points.shape[-1]} instead.') + sampling_locations = sampling_locations.to(sampling_offsets.dtype) + if torch.cuda.is_available() and value.is_cuda: + if self.use_flash: + assert False + else: + MultiScaleDeformableAttnFunction = MultiScaleDeformableAttnFunction_fp32_old + + output = MultiScaleDeformableAttnFunction.apply( + value, spatial_shapes, level_start_index, sampling_locations, + attention_weights, self.im2col_step) + + else: + output = multi_scale_deformable_attn_pytorch( + value, spatial_shapes, sampling_locations, attention_weights) + + output = self.output_proj(output.to(value.dtype)) + + if not self.batch_first: + # (num_query, bs ,embed_dims) + output = output.permute(1, 0, 2) + + return self.dropout(output) + identity + diff --git a/segmentation/mmseg_custom/models/losses/__init__.py b/segmentation/mmseg_custom/models/losses/__init__.py new file mode 100644 index 0000000..50ed88a --- /dev/null +++ b/segmentation/mmseg_custom/models/losses/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from .cross_entropy_loss import (CrossEntropyLoss, binary_cross_entropy, + cross_entropy, mask_cross_entropy) +from .dice_loss import DiceLoss +from .focal_loss import FocalLoss +from .match_costs import (ClassificationCost, CrossEntropyLossCost, DiceCost, + MaskFocalLossCost) + +__all__ = [ + 'cross_entropy', 'binary_cross_entropy', 'mask_cross_entropy', + 'CrossEntropyLoss', 'DiceLoss', 'FocalLoss', 'ClassificationCost', + 'MaskFocalLossCost', 'DiceCost', 'CrossEntropyLossCost' +] diff --git a/segmentation/mmseg_custom/models/losses/cross_entropy_loss.py b/segmentation/mmseg_custom/models/losses/cross_entropy_loss.py new file mode 100644 index 0000000..5766ea6 --- /dev/null +++ b/segmentation/mmseg_custom/models/losses/cross_entropy_loss.py @@ -0,0 +1,291 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import warnings + +import torch +import torch.nn as nn +import torch.nn.functional as F +from mmseg.models.builder import LOSSES +from mmseg.models.losses.utils import get_class_weight, weight_reduce_loss + + +def cross_entropy(pred, + label, + weight=None, + class_weight=None, + reduction='mean', + avg_factor=None, + ignore_index=-100, + avg_non_ignore=False): + """cross_entropy. The wrapper function for :func:`F.cross_entropy` + + Args: + pred (torch.Tensor): The prediction with shape (N, 1). + label (torch.Tensor): The learning label of the prediction. + weight (torch.Tensor, optional): Sample-wise loss weight. + Default: None. + class_weight (list[float], optional): The weight for each class. + Default: None. + reduction (str, optional): The method used to reduce the loss. + Options are 'none', 'mean' and 'sum'. Default: 'mean'. + avg_factor (int, optional): Average factor that is used to average + the loss. Default: None. + ignore_index (int): Specifies a target value that is ignored and + does not contribute to the input gradients. When + ``avg_non_ignore `` is ``True``, and the ``reduction`` is + ``''mean''``, the loss is averaged over non-ignored targets. + Defaults: -100. + avg_non_ignore (bool): The flag decides to whether the loss is + only averaged over non-ignored targets. Default: False. + `New in version 0.23.0.` + """ + + # class_weight is a manual rescaling weight given to each class. + # If given, has to be a Tensor of size C element-wise losses + loss = F.cross_entropy( + pred, + label, + weight=class_weight, + reduction='none', + ignore_index=ignore_index) + + # apply weights and do the reduction + # average loss over non-ignored elements + # pytorch's official cross_entropy average loss over non-ignored elements + # refer to https://github.com/pytorch/pytorch/blob/56b43f4fec1f76953f15a627694d4bba34588969/torch/nn/functional.py#L2660 # noqa + if (avg_factor is None) and avg_non_ignore and reduction == 'mean': + avg_factor = label.numel() - (label == ignore_index).sum().item() + if weight is not None: + weight = weight.float() + loss = weight_reduce_loss( + loss, weight=weight, reduction=reduction, avg_factor=avg_factor) + + return loss + + +def _expand_onehot_labels(labels, label_weights, target_shape, ignore_index): + """Expand onehot labels to match the size of prediction.""" + bin_labels = labels.new_zeros(target_shape) + valid_mask = (labels >= 0) & (labels != ignore_index) + inds = torch.nonzero(valid_mask, as_tuple=True) + + if inds[0].numel() > 0: + if labels.dim() == 3: + bin_labels[inds[0], labels[valid_mask], inds[1], inds[2]] = 1 + else: + bin_labels[inds[0], labels[valid_mask]] = 1 + + valid_mask = valid_mask.unsqueeze(1).expand(target_shape).float() + + if label_weights is None: + bin_label_weights = valid_mask + else: + bin_label_weights = label_weights.unsqueeze(1).expand(target_shape) + bin_label_weights = bin_label_weights * valid_mask + + return bin_labels, bin_label_weights, valid_mask + + +def binary_cross_entropy(pred, + label, + weight=None, + reduction='mean', + avg_factor=None, + class_weight=None, + ignore_index=-100, + avg_non_ignore=False, + **kwargs): + """Calculate the binary CrossEntropy loss. + + Args: + pred (torch.Tensor): The prediction with shape (N, 1). + label (torch.Tensor): The learning label of the prediction. + Note: In bce loss, label < 0 is invalid. + weight (torch.Tensor, optional): Sample-wise loss weight. + reduction (str, optional): The method used to reduce the loss. + Options are "none", "mean" and "sum". + avg_factor (int, optional): Average factor that is used to average + the loss. Defaults to None. + class_weight (list[float], optional): The weight for each class. + ignore_index (int): The label index to be ignored. Default: -100. + avg_non_ignore (bool): The flag decides to whether the loss is + only averaged over non-ignored targets. Default: False. + `New in version 0.23.0.` + + Returns: + torch.Tensor: The calculated loss + """ + if pred.size(1) == 1: + # For binary class segmentation, the shape of pred is + # [N, 1, H, W] and that of label is [N, H, W]. + assert label.max() <= 1, \ + 'For pred with shape [N, 1, H, W], its label must have at ' \ + 'most 2 classes' + pred = pred.squeeze() + if pred.dim() != label.dim(): + assert (pred.dim() == 2 and label.dim() == 1) or ( + pred.dim() == 4 and label.dim() == 3), \ + 'Only pred shape [N, C], label shape [N] or pred shape [N, C, ' \ + 'H, W], label shape [N, H, W] are supported' + # `weight` returned from `_expand_onehot_labels` + # has been treated for valid (non-ignore) pixels + label, weight, valid_mask = _expand_onehot_labels( + label, weight, pred.shape, ignore_index) + else: + # should mask out the ignored elements + valid_mask = ((label >= 0) & (label != ignore_index)).float() + if weight is not None: + weight = weight * valid_mask + else: + weight = valid_mask + # average loss over non-ignored and valid elements + if reduction == 'mean' and avg_factor is None and avg_non_ignore: + avg_factor = valid_mask.sum().item() + + loss = F.binary_cross_entropy_with_logits( + pred, label.float(), pos_weight=class_weight, reduction='none') + # do the reduction for the weighted loss + loss = weight_reduce_loss( + loss, weight, reduction=reduction, avg_factor=avg_factor) + + return loss + + +def mask_cross_entropy(pred, + target, + label, + reduction='mean', + avg_factor=None, + class_weight=None, + ignore_index=None, + **kwargs): + """Calculate the CrossEntropy loss for masks. + + Args: + pred (torch.Tensor): The prediction with shape (N, C), C is the number + of classes. + target (torch.Tensor): The learning label of the prediction. + label (torch.Tensor): ``label`` indicates the class label of the mask' + corresponding object. This will be used to select the mask in the + of the class which the object belongs to when the mask prediction + if not class-agnostic. + reduction (str, optional): The method used to reduce the loss. + Options are "none", "mean" and "sum". + avg_factor (int, optional): Average factor that is used to average + the loss. Defaults to None. + class_weight (list[float], optional): The weight for each class. + ignore_index (None): Placeholder, to be consistent with other loss. + Default: None. + + Returns: + torch.Tensor: The calculated loss + """ + assert ignore_index is None, 'BCE loss does not support ignore_index' + # TODO: handle these two reserved arguments + assert reduction == 'mean' and avg_factor is None + num_rois = pred.size()[0] + inds = torch.arange(0, num_rois, dtype=torch.long, device=pred.device) + pred_slice = pred[inds, label].squeeze(1) + return F.binary_cross_entropy_with_logits( + pred_slice, target, weight=class_weight, reduction='mean')[None] + + +@LOSSES.register_module(force=True) +class CrossEntropyLoss(nn.Module): + """CrossEntropyLoss. + + Args: + use_sigmoid (bool, optional): Whether the prediction uses sigmoid + of softmax. Defaults to False. + use_mask (bool, optional): Whether to use mask cross entropy loss. + Defaults to False. + reduction (str, optional): . Defaults to 'mean'. + Options are "none", "mean" and "sum". + class_weight (list[float] | str, optional): Weight of each class. If in + str format, read them from a file. Defaults to None. + loss_weight (float, optional): Weight of the loss. Defaults to 1.0. + loss_name (str, optional): Name of the loss item. If you want this loss + item to be included into the backward graph, `loss_` must be the + prefix of the name. Defaults to 'loss_ce'. + avg_non_ignore (bool): The flag decides to whether the loss is + only averaged over non-ignored targets. Default: False. + `New in version 0.23.0.` + """ + def __init__(self, + use_sigmoid=False, + use_mask=False, + reduction='mean', + class_weight=None, + loss_weight=1.0, + loss_name='loss_ce', + avg_non_ignore=False): + super(CrossEntropyLoss, self).__init__() + assert (use_sigmoid is False) or (use_mask is False) + self.use_sigmoid = use_sigmoid + self.use_mask = use_mask + self.reduction = reduction + self.loss_weight = loss_weight + self.class_weight = get_class_weight(class_weight) + self.avg_non_ignore = avg_non_ignore + if not self.avg_non_ignore and self.reduction == 'mean': + warnings.warn( + 'Default ``avg_non_ignore`` is False, if you would like to ' + 'ignore the certain label and average loss over non-ignore ' + 'labels, which is the same with PyTorch official ' + 'cross_entropy, set ``avg_non_ignore=True``.') + + if self.use_sigmoid: + self.cls_criterion = binary_cross_entropy + elif self.use_mask: + self.cls_criterion = mask_cross_entropy + else: + self.cls_criterion = cross_entropy + self._loss_name = loss_name + + def extra_repr(self): + """Extra repr.""" + s = f'avg_non_ignore={self.avg_non_ignore}' + return s + + def forward(self, + cls_score, + label, + weight=None, + avg_factor=None, + reduction_override=None, + ignore_index=-100, + **kwargs): + """Forward function.""" + assert reduction_override in (None, 'none', 'mean', 'sum') + reduction = (reduction_override + if reduction_override else self.reduction) + if self.class_weight is not None: + class_weight = cls_score.new_tensor(self.class_weight) + else: + class_weight = None + # Note: for BCE loss, label < 0 is invalid. + loss_cls = self.loss_weight * self.cls_criterion( + cls_score, + label, + weight, + class_weight=class_weight, + reduction=reduction, + avg_factor=avg_factor, + avg_non_ignore=self.avg_non_ignore, + ignore_index=ignore_index, + **kwargs) + return loss_cls + + @property + def loss_name(self): + """Loss Name. + + This function must be implemented and will return the name of this + loss function. This name will be used to combine different loss items + by simple sum operation. In addition, if you want this loss item to be + included into the backward graph, `loss_` must be the prefix of the + name. + + Returns: + str: The name of this loss item. + """ + return self._loss_name diff --git a/segmentation/mmseg_custom/models/losses/dice_loss.py b/segmentation/mmseg_custom/models/losses/dice_loss.py new file mode 100644 index 0000000..e84b07e --- /dev/null +++ b/segmentation/mmseg_custom/models/losses/dice_loss.py @@ -0,0 +1,179 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +import torch.nn as nn +from mmseg.models.builder import LOSSES +from mmseg.models.losses.utils import weight_reduce_loss + + +def dice_loss(pred, + target, + weight=None, + eps=1e-3, + reduction='mean', + avg_factor=None): + """Calculate dice loss, which is proposed in + `V-Net: Fully Convolutional Neural Networks for Volumetric + Medical Image Segmentation `_. + + Args: + pred (torch.Tensor): The prediction, has a shape (n, *) + target (torch.Tensor): The learning label of the prediction, + shape (n, *), same shape of pred. + weight (torch.Tensor, optional): The weight of loss for each + prediction, has a shape (n,). Defaults to None. + eps (float): Avoid dividing by zero. Default: 1e-3. + reduction (str, optional): The method used to reduce the loss into + a scalar. Defaults to 'mean'. + Options are "none", "mean" and "sum". + avg_factor (int, optional): Average factor that is used to average + the loss. Defaults to None. + """ + + input = pred.flatten(1) + target = target.flatten(1).float() + + a = torch.sum(input * target, 1) + b = torch.sum(input * input, 1) + eps + c = torch.sum(target * target, 1) + eps + d = (2 * a) / (b + c) + loss = 1 - d + if weight is not None: + assert weight.ndim == loss.ndim + assert len(weight) == len(pred) + loss = weight_reduce_loss(loss, weight, reduction, avg_factor) + return loss + + +def naive_dice_loss(pred, + target, + weight=None, + eps=1e-3, + reduction='mean', + avg_factor=None): + """Calculate naive dice loss, the coefficient in the denominator is the + first power instead of the second power. + + Args: + pred (torch.Tensor): The prediction, has a shape (n, *) + target (torch.Tensor): The learning label of the prediction, + shape (n, *), same shape of pred. + weight (torch.Tensor, optional): The weight of loss for each + prediction, has a shape (n,). Defaults to None. + eps (float): Avoid dividing by zero. Default: 1e-3. + reduction (str, optional): The method used to reduce the loss into + a scalar. Defaults to 'mean'. + Options are "none", "mean" and "sum". + avg_factor (int, optional): Average factor that is used to average + the loss. Defaults to None. + """ + input = pred.flatten(1) + target = target.flatten(1).float() + + a = torch.sum(input * target, 1) + b = torch.sum(input, 1) + c = torch.sum(target, 1) + d = (2 * a + eps) / (b + c + eps) + loss = 1 - d + if weight is not None: + assert weight.ndim == loss.ndim + assert len(weight) == len(pred) + loss = weight_reduce_loss(loss, weight, reduction, avg_factor) + return loss + + +@LOSSES.register_module(force=True) +class DiceLoss(nn.Module): + def __init__(self, + use_sigmoid=True, + activate=True, + reduction='mean', + naive_dice=False, + loss_weight=1.0, + eps=1e-3): + """Dice Loss, there are two forms of dice loss is supported: + + - the one proposed in `V-Net: Fully Convolutional Neural + Networks for Volumetric Medical Image Segmentation + `_. + - the dice loss in which the power of the number in the + denominator is the first power instead of the second + power. + + Args: + use_sigmoid (bool, optional): Whether to the prediction is + used for sigmoid or softmax. Defaults to True. + activate (bool): Whether to activate the predictions inside, + this will disable the inside sigmoid operation. + Defaults to True. + reduction (str, optional): The method used + to reduce the loss. Options are "none", + "mean" and "sum". Defaults to 'mean'. + naive_dice (bool, optional): If false, use the dice + loss defined in the V-Net paper, otherwise, use the + naive dice loss in which the power of the number in the + denominator is the first power instead of the second + power.Defaults to False. + loss_weight (float, optional): Weight of loss. Defaults to 1.0. + eps (float): Avoid dividing by zero. Defaults to 1e-3. + """ + + super(DiceLoss, self).__init__() + self.use_sigmoid = use_sigmoid + self.reduction = reduction + self.naive_dice = naive_dice + self.loss_weight = loss_weight + self.eps = eps + self.activate = activate + + def forward(self, + pred, + target, + weight=None, + reduction_override=None, + avg_factor=None): + """Forward function. + + Args: + pred (torch.Tensor): The prediction, has a shape (n, *). + target (torch.Tensor): The label of the prediction, + shape (n, *), same shape of pred. + weight (torch.Tensor, optional): The weight of loss for each + prediction, has a shape (n,). Defaults to None. + avg_factor (int, optional): Average factor that is used to average + the loss. Defaults to None. + reduction_override (str, optional): The reduction method used to + override the original reduction method of the loss. + Options are "none", "mean" and "sum". + + Returns: + torch.Tensor: The calculated loss + """ + + assert reduction_override in (None, 'none', 'mean', 'sum') + reduction = (reduction_override + if reduction_override else self.reduction) + + if self.activate: + if self.use_sigmoid: + pred = pred.sigmoid() + else: + raise NotImplementedError + + if self.naive_dice: + loss = self.loss_weight * naive_dice_loss( + pred, + target, + weight, + eps=self.eps, + reduction=reduction, + avg_factor=avg_factor) + else: + loss = self.loss_weight * dice_loss( + pred, + target, + weight, + eps=self.eps, + reduction=reduction, + avg_factor=avg_factor) + + return loss \ No newline at end of file diff --git a/segmentation/mmseg_custom/models/losses/focal_loss.py b/segmentation/mmseg_custom/models/losses/focal_loss.py new file mode 100644 index 0000000..3d48a2b --- /dev/null +++ b/segmentation/mmseg_custom/models/losses/focal_loss.py @@ -0,0 +1,180 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +import torch.nn as nn +import torch.nn.functional as F +from mmcv.ops import sigmoid_focal_loss as _sigmoid_focal_loss +from mmseg.models.builder import LOSSES +from mmseg.models.losses.utils import weight_reduce_loss + + +# This method is only for debugging +def py_sigmoid_focal_loss(pred, + target, + weight=None, + gamma=2.0, + alpha=0.25, + reduction='mean', + avg_factor=None): + """PyTorch version of `Focal Loss `_. + + Args: + pred (torch.Tensor): The prediction with shape (N, C), C is the + number of classes + target (torch.Tensor): The learning label of the prediction. + weight (torch.Tensor, optional): Sample-wise loss weight. + gamma (float, optional): The gamma for calculating the modulating + factor. Defaults to 2.0. + alpha (float, optional): A balanced form for Focal Loss. + Defaults to 0.25. + reduction (str, optional): The method used to reduce the loss into + a scalar. Defaults to 'mean'. + avg_factor (int, optional): Average factor that is used to average + the loss. Defaults to None. + """ + pred_sigmoid = pred.sigmoid() + target = target.type_as(pred) + pt = (1 - pred_sigmoid) * target + pred_sigmoid * (1 - target) + focal_weight = (alpha * target + (1 - alpha) * + (1 - target)) * pt.pow(gamma) + loss = F.binary_cross_entropy_with_logits( + pred, target, reduction='none') * focal_weight + if weight is not None: + if weight.shape != loss.shape: + if weight.size(0) == loss.size(0): + # For most cases, weight is of shape (num_priors, ), + # which means it does not have the second axis num_class + weight = weight.view(-1, 1) + else: + # Sometimes, weight per anchor per class is also needed. e.g. + # in FSAF. But it may be flattened of shape + # (num_priors x num_class, ), while loss is still of shape + # (num_priors, num_class). + assert weight.numel() == loss.numel() + weight = weight.view(loss.size(0), -1) + assert weight.ndim == loss.ndim + loss = weight_reduce_loss(loss, weight, reduction, avg_factor) + return loss + + +def sigmoid_focal_loss(pred, + target, + weight=None, + gamma=2.0, + alpha=0.25, + reduction='mean', + avg_factor=None): + r"""A warpper of cuda version `Focal Loss + `_. + + Args: + pred (torch.Tensor): The prediction with shape (N, C), C is the number + of classes. + target (torch.Tensor): The learning label of the prediction. + weight (torch.Tensor, optional): Sample-wise loss weight. + gamma (float, optional): The gamma for calculating the modulating + factor. Defaults to 2.0. + alpha (float, optional): A balanced form for Focal Loss. + Defaults to 0.25. + reduction (str, optional): The method used to reduce the loss into + a scalar. Defaults to 'mean'. Options are "none", "mean" and "sum". + avg_factor (int, optional): Average factor that is used to average + the loss. Defaults to None. + """ + # Function.apply does not accept keyword arguments, so the decorator + # "weighted_loss" is not applicable + loss = _sigmoid_focal_loss(pred.contiguous(), target.contiguous(), gamma, + alpha, None, 'none') + if weight is not None: + if weight.shape != loss.shape: + if weight.size(0) == loss.size(0): + # For most cases, weight is of shape (num_priors, ), + # which means it does not have the second axis num_class + weight = weight.view(-1, 1) + else: + # Sometimes, weight per anchor per class is also needed. e.g. + # in FSAF. But it may be flattened of shape + # (num_priors x num_class, ), while loss is still of shape + # (num_priors, num_class). + assert weight.numel() == loss.numel() + weight = weight.view(loss.size(0), -1) + assert weight.ndim == loss.ndim + loss = weight_reduce_loss(loss, weight, reduction, avg_factor) + return loss + + +@LOSSES.register_module(force=True) +class FocalLoss(nn.Module): + def __init__(self, + use_sigmoid=True, + gamma=2.0, + alpha=0.25, + reduction='mean', + loss_weight=1.0): + """`Focal Loss `_ + + Args: + use_sigmoid (bool, optional): Whether to the prediction is + used for sigmoid or softmax. Defaults to True. + gamma (float, optional): The gamma for calculating the modulating + factor. Defaults to 2.0. + alpha (float, optional): A balanced form for Focal Loss. + Defaults to 0.25. + reduction (str, optional): The method used to reduce the loss into + a scalar. Defaults to 'mean'. Options are "none", "mean" and + "sum". + loss_weight (float, optional): Weight of loss. Defaults to 1.0. + """ + super(FocalLoss, self).__init__() + assert use_sigmoid is True, 'Only sigmoid focal loss supported now.' + self.use_sigmoid = use_sigmoid + self.gamma = gamma + self.alpha = alpha + self.reduction = reduction + self.loss_weight = loss_weight + + def forward(self, + pred, + target, + weight=None, + avg_factor=None, + reduction_override=None): + """Forward function. + + Args: + pred (torch.Tensor): The prediction. + target (torch.Tensor): The learning label of the prediction. + weight (torch.Tensor, optional): The weight of loss for each + prediction. Defaults to None. + avg_factor (int, optional): Average factor that is used to average + the loss. Defaults to None. + reduction_override (str, optional): The reduction method used to + override the original reduction method of the loss. + Options are "none", "mean" and "sum". + + Returns: + torch.Tensor: The calculated loss + """ + assert reduction_override in (None, 'none', 'mean', 'sum') + reduction = ( + reduction_override if reduction_override else self.reduction) + if self.use_sigmoid: + if torch.cuda.is_available() and pred.is_cuda: + calculate_loss_func = sigmoid_focal_loss + else: + num_classes = pred.size(1) + target = F.one_hot(target, num_classes=num_classes + 1) + target = target[:, :num_classes] + calculate_loss_func = py_sigmoid_focal_loss + + loss_cls = self.loss_weight * calculate_loss_func( + pred, + target, + weight, + gamma=self.gamma, + alpha=self.alpha, + reduction=reduction, + avg_factor=avg_factor) + + else: + raise NotImplementedError + return loss_cls diff --git a/segmentation/mmseg_custom/models/losses/match_costs.py b/segmentation/mmseg_custom/models/losses/match_costs.py new file mode 100644 index 0000000..9c04862 --- /dev/null +++ b/segmentation/mmseg_custom/models/losses/match_costs.py @@ -0,0 +1,233 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +import torch.nn as nn +import torch.nn.functional as F + +from ..builder import MATCH_COST + + +@MATCH_COST.register_module() +class FocalLossCost: + """FocalLossCost. + + Args: + weight (int | float, optional): loss_weight + alpha (int | float, optional): focal_loss alpha + gamma (int | float, optional): focal_loss gamma + eps (float, optional): default 1e-12 + + Examples: + >>> from mmdet.core.bbox.match_costs.match_cost import FocalLossCost + >>> import torch + >>> self = FocalLossCost() + >>> cls_pred = torch.rand(4, 3) + >>> gt_labels = torch.tensor([0, 1, 2]) + >>> factor = torch.tensor([10, 8, 10, 8]) + >>> self(cls_pred, gt_labels) + tensor([[-0.3236, -0.3364, -0.2699], + [-0.3439, -0.3209, -0.4807], + [-0.4099, -0.3795, -0.2929], + [-0.1950, -0.1207, -0.2626]]) + """ + def __init__(self, weight=1., alpha=0.25, gamma=2, eps=1e-12): + self.weight = weight + self.alpha = alpha + self.gamma = gamma + self.eps = eps + + def __call__(self, cls_pred, gt_labels): + """ + Args: + cls_pred (Tensor): Predicted classification logits, shape + [num_query, num_class]. + gt_labels (Tensor): Label of `gt_bboxes`, shape (num_gt,). + + Returns: + torch.Tensor: cls_cost value with weight + """ + cls_pred = cls_pred.sigmoid() + neg_cost = -(1 - cls_pred + self.eps).log() * ( + 1 - self.alpha) * cls_pred.pow(self.gamma) + pos_cost = -(cls_pred + self.eps).log() * self.alpha * ( + 1 - cls_pred).pow(self.gamma) + cls_cost = pos_cost[:, gt_labels] - neg_cost[:, gt_labels] + return cls_cost * self.weight + + +@MATCH_COST.register_module() +class MaskFocalLossCost(FocalLossCost): + """Cost of mask assignments based on focal losses. + + Args: + weight (int | float, optional): loss_weight. + alpha (int | float, optional): focal_loss alpha. + gamma (int | float, optional): focal_loss gamma. + eps (float, optional): default 1e-12. + """ + def __call__(self, cls_pred, gt_labels): + """ + Args: + cls_pred (Tensor): Predicted classfication logits + in shape (N1, H, W), dtype=torch.float32. + gt_labels (Tensor): Ground truth in shape (N2, H, W), + dtype=torch.long. + + Returns: + Tensor: classification cost matrix in shape (N1, N2). + """ + cls_pred = cls_pred.reshape((cls_pred.shape[0], -1)) + gt_labels = gt_labels.reshape((gt_labels.shape[0], -1)).float() + hw = cls_pred.shape[1] + cls_pred = cls_pred.sigmoid() + neg_cost = -(1 - cls_pred + self.eps).log() * ( + 1 - self.alpha) * cls_pred.pow(self.gamma) + pos_cost = -(cls_pred + self.eps).log() * self.alpha * ( + 1 - cls_pred).pow(self.gamma) + + cls_cost = torch.einsum('nc,mc->nm', pos_cost, gt_labels) + \ + torch.einsum('nc,mc->nm', neg_cost, (1 - gt_labels)) + return cls_cost / hw * self.weight + + +@MATCH_COST.register_module() +class ClassificationCost: + """ClsSoftmaxCost.Borrow from + mmdet.core.bbox.match_costs.match_cost.ClassificationCost. + + Args: + weight (int | float, optional): loss_weight + + Examples: + >>> import torch + >>> self = ClassificationCost() + >>> cls_pred = torch.rand(4, 3) + >>> gt_labels = torch.tensor([0, 1, 2]) + >>> factor = torch.tensor([10, 8, 10, 8]) + >>> self(cls_pred, gt_labels) + tensor([[-0.3430, -0.3525, -0.3045], + [-0.3077, -0.2931, -0.3992], + [-0.3664, -0.3455, -0.2881], + [-0.3343, -0.2701, -0.3956]]) + """ + def __init__(self, weight=1.): + self.weight = weight + + def __call__(self, cls_pred, gt_labels): + """ + Args: + cls_pred (Tensor): Predicted classification logits, shape + [num_query, num_class]. + gt_labels (Tensor): Label of `gt_bboxes`, shape (num_gt,). + + Returns: + torch.Tensor: cls_cost value with weight + """ + # Following the official DETR repo, contrary to the loss that + # NLL is used, we approximate it in 1 - cls_score[gt_label]. + # The 1 is a constant that doesn't change the matching, + # so it can be omitted. + cls_score = cls_pred.softmax(-1) + cls_cost = -cls_score[:, gt_labels] + return cls_cost * self.weight + + +@MATCH_COST.register_module() +class DiceCost: + """Cost of mask assignments based on dice losses. + + Args: + weight (int | float, optional): loss_weight. Defaults to 1. + pred_act (bool, optional): Whether to apply sigmoid to mask_pred. + Defaults to False. + eps (float, optional): default 1e-12. + """ + def __init__(self, weight=1., pred_act=False, eps=1e-3): + self.weight = weight + self.pred_act = pred_act + self.eps = eps + + def binary_mask_dice_loss(self, mask_preds, gt_masks): + """ + Args: + mask_preds (Tensor): Mask prediction in shape (N1, H, W). + gt_masks (Tensor): Ground truth in shape (N2, H, W) + store 0 or 1, 0 for negative class and 1 for + positive class. + + Returns: + Tensor: Dice cost matrix in shape (N1, N2). + """ + mask_preds = mask_preds.reshape((mask_preds.shape[0], -1)) + gt_masks = gt_masks.reshape((gt_masks.shape[0], -1)).float() + numerator = 2 * torch.einsum('nc,mc->nm', mask_preds, gt_masks) + denominator = mask_preds.sum(-1)[:, None] + gt_masks.sum(-1)[None, :] + loss = 1 - (numerator + self.eps) / (denominator + self.eps) + return loss + + def __call__(self, mask_preds, gt_masks): + """ + Args: + mask_preds (Tensor): Mask prediction logits in shape (N1, H, W). + gt_masks (Tensor): Ground truth in shape (N2, H, W). + + Returns: + Tensor: Dice cost matrix in shape (N1, N2). + """ + if self.pred_act: + mask_preds = mask_preds.sigmoid() + dice_cost = self.binary_mask_dice_loss(mask_preds, gt_masks) + return dice_cost * self.weight + + +@MATCH_COST.register_module() +class CrossEntropyLossCost: + """CrossEntropyLossCost. + + Args: + weight (int | float, optional): loss weight. Defaults to 1. + use_sigmoid (bool, optional): Whether the prediction uses sigmoid + of softmax. Defaults to True. + """ + def __init__(self, weight=1., use_sigmoid=True): + assert use_sigmoid, 'use_sigmoid = False is not supported yet.' + self.weight = weight + self.use_sigmoid = use_sigmoid + + def _binary_cross_entropy(self, cls_pred, gt_labels): + """ + Args: + cls_pred (Tensor): The prediction with shape (num_query, 1, *) or + (num_query, *). + gt_labels (Tensor): The learning label of prediction with + shape (num_gt, *). + Returns: + Tensor: Cross entropy cost matrix in shape (num_query, num_gt). + """ + cls_pred = cls_pred.flatten(1).float() + gt_labels = gt_labels.flatten(1).float() + n = cls_pred.shape[1] + pos = F.binary_cross_entropy_with_logits( + cls_pred, torch.ones_like(cls_pred), reduction='none') + neg = F.binary_cross_entropy_with_logits( + cls_pred, torch.zeros_like(cls_pred), reduction='none') + cls_cost = torch.einsum('nc,mc->nm', pos, gt_labels) + \ + torch.einsum('nc,mc->nm', neg, 1 - gt_labels) + cls_cost = cls_cost / n + + return cls_cost + + def __call__(self, cls_pred, gt_labels): + """ + Args: + cls_pred (Tensor): Predicted classification logits. + gt_labels (Tensor): Labels. + Returns: + Tensor: Cross entropy cost matrix with weight in + shape (num_query, num_gt). + """ + if self.use_sigmoid: + cls_cost = self._binary_cross_entropy(cls_pred, gt_labels) + else: + raise NotImplementedError + + return cls_cost * self.weight diff --git a/segmentation/mmseg_custom/models/losses/match_loss.py b/segmentation/mmseg_custom/models/losses/match_loss.py new file mode 100644 index 0000000..3c53839 --- /dev/null +++ b/segmentation/mmseg_custom/models/losses/match_loss.py @@ -0,0 +1,179 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +import torch.nn as nn +import torch.nn.functional as F + +from ..builder import MATCH_COST + + +@MATCH_COST.register_module() +class FocalLossCost: + """FocalLossCost. + + Args: + weight (int | float, optional): loss_weight + alpha (int | float, optional): focal_loss alpha + gamma (int | float, optional): focal_loss gamma + eps (float, optional): default 1e-12 + + Examples: + >>> from mmdet.core.bbox.match_costs.match_cost import FocalLossCost + >>> import torch + >>> self = FocalLossCost() + >>> cls_pred = torch.rand(4, 3) + >>> gt_labels = torch.tensor([0, 1, 2]) + >>> factor = torch.tensor([10, 8, 10, 8]) + >>> self(cls_pred, gt_labels) + tensor([[-0.3236, -0.3364, -0.2699], + [-0.3439, -0.3209, -0.4807], + [-0.4099, -0.3795, -0.2929], + [-0.1950, -0.1207, -0.2626]]) + """ + def __init__(self, weight=1., alpha=0.25, gamma=2, eps=1e-12): + self.weight = weight + self.alpha = alpha + self.gamma = gamma + self.eps = eps + + def __call__(self, cls_pred, gt_labels): + """ + Args: + cls_pred (Tensor): Predicted classification logits, shape + [num_query, num_class]. + gt_labels (Tensor): Label of `gt_bboxes`, shape (num_gt,). + + Returns: + torch.Tensor: cls_cost value with weight + """ + cls_pred = cls_pred.sigmoid() + neg_cost = -(1 - cls_pred + self.eps).log() * ( + 1 - self.alpha) * cls_pred.pow(self.gamma) + pos_cost = -(cls_pred + self.eps).log() * self.alpha * ( + 1 - cls_pred).pow(self.gamma) + cls_cost = pos_cost[:, gt_labels] - neg_cost[:, gt_labels] + return cls_cost * self.weight + + +@MATCH_COST.register_module() +class MaskFocalLossCost(FocalLossCost): + """Cost of mask assignments based on focal losses. + + Args: + weight (int | float, optional): loss_weight. + alpha (int | float, optional): focal_loss alpha. + gamma (int | float, optional): focal_loss gamma. + eps (float, optional): default 1e-12. + """ + def __call__(self, cls_pred, gt_labels): + """ + Args: + cls_pred (Tensor): Predicted classfication logits + in shape (N1, H, W), dtype=torch.float32. + gt_labels (Tensor): Ground truth in shape (N2, H, W), + dtype=torch.long. + + Returns: + Tensor: classification cost matrix in shape (N1, N2). + """ + cls_pred = cls_pred.reshape((cls_pred.shape[0], -1)) + gt_labels = gt_labels.reshape((gt_labels.shape[0], -1)).float() + hw = cls_pred.shape[1] + cls_pred = cls_pred.sigmoid() + neg_cost = -(1 - cls_pred + self.eps).log() * ( + 1 - self.alpha) * cls_pred.pow(self.gamma) + pos_cost = -(cls_pred + self.eps).log() * self.alpha * ( + 1 - cls_pred).pow(self.gamma) + + cls_cost = torch.einsum('nc,mc->nm', pos_cost, gt_labels) + \ + torch.einsum('nc,mc->nm', neg_cost, (1 - gt_labels)) + return cls_cost / hw * self.weight + + +@MATCH_COST.register_module() +class ClassificationCost: + """ClsSoftmaxCost.Borrow from + mmdet.core.bbox.match_costs.match_cost.ClassificationCost. + + Args: + weight (int | float, optional): loss_weight + + Examples: + >>> import torch + >>> self = ClassificationCost() + >>> cls_pred = torch.rand(4, 3) + >>> gt_labels = torch.tensor([0, 1, 2]) + >>> factor = torch.tensor([10, 8, 10, 8]) + >>> self(cls_pred, gt_labels) + tensor([[-0.3430, -0.3525, -0.3045], + [-0.3077, -0.2931, -0.3992], + [-0.3664, -0.3455, -0.2881], + [-0.3343, -0.2701, -0.3956]]) + """ + def __init__(self, weight=1.): + self.weight = weight + + def __call__(self, cls_pred, gt_labels): + """ + Args: + cls_pred (Tensor): Predicted classification logits, shape + [num_query, num_class]. + gt_labels (Tensor): Label of `gt_bboxes`, shape (num_gt,). + + Returns: + torch.Tensor: cls_cost value with weight + """ + # Following the official DETR repo, contrary to the loss that + # NLL is used, we approximate it in 1 - cls_score[gt_label]. + # The 1 is a constant that doesn't change the matching, + # so it can be omitted. + cls_score = cls_pred.softmax(-1) + cls_cost = -cls_score[:, gt_labels] + return cls_cost * self.weight + + +@MATCH_COST.register_module() +class DiceCost: + """Cost of mask assignments based on dice losses. + + Args: + weight (int | float, optional): loss_weight. Defaults to 1. + pred_act (bool, optional): Whether to apply sigmoid to mask_pred. + Defaults to False. + eps (float, optional): default 1e-12. + """ + def __init__(self, weight=1., pred_act=False, eps=1e-3): + self.weight = weight + self.pred_act = pred_act + self.eps = eps + + def binary_mask_dice_loss(self, mask_preds, gt_masks): + """ + Args: + mask_preds (Tensor): Mask prediction in shape (N1, H, W). + gt_masks (Tensor): Ground truth in shape (N2, H, W) + store 0 or 1, 0 for negative class and 1 for + positive class. + + Returns: + Tensor: Dice cost matrix in shape (N1, N2). + """ + mask_preds = mask_preds.reshape((mask_preds.shape[0], -1)) + gt_masks = gt_masks.reshape((gt_masks.shape[0], -1)).float() + numerator = 2 * torch.einsum('nc,mc->nm', mask_preds, gt_masks) + denominator = mask_preds.sum(-1)[:, None] + gt_masks.sum(-1)[None, :] + loss = 1 - (numerator + self.eps) / (denominator + self.eps) + return loss + + def __call__(self, mask_preds, gt_masks): + """ + Args: + mask_preds (Tensor): Mask prediction logits in shape (N1, H, W). + gt_masks (Tensor): Ground truth in shape (N2, H, W). + + Returns: + Tensor: Dice cost matrix in shape (N1, N2). + """ + if self.pred_act: + mask_preds = mask_preds.sigmoid() + dice_cost = self.binary_mask_dice_loss(mask_preds, gt_masks) + return dice_cost * self.weight diff --git a/segmentation/mmseg_custom/models/plugins/__init__.py b/segmentation/mmseg_custom/models/plugins/__init__.py new file mode 100644 index 0000000..a1bb9ad --- /dev/null +++ b/segmentation/mmseg_custom/models/plugins/__init__.py @@ -0,0 +1,8 @@ +# Copyright (c) Shanghai AI Lab. All rights reserved. +from .msdeformattn_pixel_decoder import MSDeformAttnPixelDecoder +from .pixel_decoder import PixelDecoder, TransformerEncoderPixelDecoder + +__all__ = [ + 'PixelDecoder', 'TransformerEncoderPixelDecoder', + 'MSDeformAttnPixelDecoder' +] diff --git a/segmentation/mmseg_custom/models/plugins/msdeformattn_pixel_decoder.py b/segmentation/mmseg_custom/models/plugins/msdeformattn_pixel_decoder.py new file mode 100644 index 0000000..3a4ea63 --- /dev/null +++ b/segmentation/mmseg_custom/models/plugins/msdeformattn_pixel_decoder.py @@ -0,0 +1,268 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +import torch.nn as nn +import torch.nn.functional as F +from mmcv.cnn import (PLUGIN_LAYERS, Conv2d, ConvModule, caffe2_xavier_init, + normal_init, xavier_init) +from mmcv.cnn.bricks.transformer import (build_positional_encoding, + build_transformer_layer_sequence) +from mmcv.runner import BaseModule, ModuleList + +from ...core.anchor import MlvlPointGenerator +from ..utils.transformer import MultiScaleDeformableAttention + + +@PLUGIN_LAYERS.register_module() +class MSDeformAttnPixelDecoder(BaseModule): + """Pixel decoder with multi-scale deformable attention. + + Args: + in_channels (list[int] | tuple[int]): Number of channels in the + input feature maps. + strides (list[int] | tuple[int]): Output strides of feature from + backbone. + feat_channels (int): Number of channels for feature. + out_channels (int): Number of channels for output. + num_outs (int): Number of output scales. + norm_cfg (:obj:`mmcv.ConfigDict` | dict): Config for normalization. + Defaults to dict(type='GN', num_groups=32). + act_cfg (:obj:`mmcv.ConfigDict` | dict): Config for activation. + Defaults to dict(type='ReLU'). + encoder (:obj:`mmcv.ConfigDict` | dict): Config for transformer + encoder. Defaults to `DetrTransformerEncoder`. + positional_encoding (:obj:`mmcv.ConfigDict` | dict): Config for + transformer encoder position encoding. Defaults to + dict(type='SinePositionalEncoding', num_feats=128, + normalize=True). + init_cfg (:obj:`mmcv.ConfigDict` | dict): Initialization config dict. + """ + def __init__(self, + in_channels=[256, 512, 1024, 2048], + strides=[4, 8, 16, 32], + feat_channels=256, + out_channels=256, + num_outs=3, + norm_cfg=dict(type='GN', num_groups=32), + act_cfg=dict(type='ReLU'), + encoder=dict( + type='DetrTransformerEncoder', + num_layers=6, + transformerlayers=dict( + type='BaseTransformerLayer', + attn_cfgs=dict( + type='MultiScaleDeformableAttention', + embed_dims=256, + num_heads=8, + num_levels=3, + num_points=4, + im2col_step=64, + dropout=0.0, + batch_first=False, + norm_cfg=None, + init_cfg=None), + feedforward_channels=1024, + ffn_dropout=0.0, + operation_order=('self_attn', 'norm', 'ffn', 'norm')), + init_cfg=None), + positional_encoding=dict( + type='SinePositionalEncoding', + num_feats=128, + normalize=True), + init_cfg=None): + super().__init__(init_cfg=init_cfg) + self.strides = strides + self.num_input_levels = len(in_channels) + self.num_encoder_levels = \ + encoder.transformerlayers.attn_cfgs.num_levels + assert self.num_encoder_levels >= 1, \ + 'num_levels in attn_cfgs must be at least one' + input_conv_list = [] + # from top to down (low to high resolution) + for i in range(self.num_input_levels - 1, + self.num_input_levels - self.num_encoder_levels - 1, + -1): + input_conv = ConvModule( + in_channels[i], + feat_channels, + kernel_size=1, + norm_cfg=norm_cfg, + act_cfg=None, + bias=True) + input_conv_list.append(input_conv) + self.input_convs = ModuleList(input_conv_list) + + self.encoder = build_transformer_layer_sequence(encoder) + self.postional_encoding = build_positional_encoding( + positional_encoding) + # high resolution to low resolution + self.level_encoding = nn.Embedding(self.num_encoder_levels, + feat_channels) + + # fpn-like structure + self.lateral_convs = ModuleList() + self.output_convs = ModuleList() + self.use_bias = norm_cfg is None + # from top to down (low to high resolution) + # fpn for the rest features that didn't pass in encoder + for i in range(self.num_input_levels - self.num_encoder_levels - 1, -1, + -1): + lateral_conv = ConvModule( + in_channels[i], + feat_channels, + kernel_size=1, + bias=self.use_bias, + norm_cfg=norm_cfg, + act_cfg=None) + output_conv = ConvModule( + feat_channels, + feat_channels, + kernel_size=3, + stride=1, + padding=1, + bias=self.use_bias, + norm_cfg=norm_cfg, + act_cfg=act_cfg) + self.lateral_convs.append(lateral_conv) + self.output_convs.append(output_conv) + + self.mask_feature = Conv2d( + feat_channels, out_channels, kernel_size=1, stride=1, padding=0) + + self.num_outs = num_outs + self.point_generator = MlvlPointGenerator(strides) + + def init_weights(self): + """Initialize weights.""" + for i in range(0, self.num_encoder_levels): + xavier_init( + self.input_convs[i].conv, + gain=1, + bias=0, + distribution='uniform') + + for i in range(0, self.num_input_levels - self.num_encoder_levels): + caffe2_xavier_init(self.lateral_convs[i].conv, bias=0) + caffe2_xavier_init(self.output_convs[i].conv, bias=0) + + caffe2_xavier_init(self.mask_feature, bias=0) + + normal_init(self.level_encoding, mean=0, std=1) + for p in self.encoder.parameters(): + if p.dim() > 1: + nn.init.xavier_normal_(p) + + # init_weights defined in MultiScaleDeformableAttention + for layer in self.encoder.layers: + for attn in layer.attentions: + if isinstance(attn, MultiScaleDeformableAttention): + attn.init_weights() + + def forward(self, feats): + """ + Args: + feats (list[Tensor]): Feature maps of each level. Each has + shape of (batch_size, c, h, w). + + Returns: + tuple: A tuple containing the following: + + - mask_feature (Tensor): shape (batch_size, c, h, w). + - multi_scale_features (list[Tensor]): Multi scale \ + features, each in shape (batch_size, c, h, w). + """ + # generate padding mask for each level, for each image + batch_size = feats[0].shape[0] + encoder_input_list = [] + padding_mask_list = [] + level_positional_encoding_list = [] + spatial_shapes = [] + reference_points_list = [] + for i in range(self.num_encoder_levels): + level_idx = self.num_input_levels - i - 1 + feat = feats[level_idx] + feat_projected = self.input_convs[i](feat) + h, w = feat.shape[-2:] + + # no padding + padding_mask_resized = feat.new_zeros( + (batch_size, ) + feat.shape[-2:], dtype=torch.bool) + pos_embed = self.postional_encoding(padding_mask_resized) + level_embed = self.level_encoding.weight[i] + level_pos_embed = level_embed.view(1, -1, 1, 1) + pos_embed + # (h_i * w_i, 2) + reference_points = self.point_generator.single_level_grid_priors( + feat.shape[-2:], level_idx, device=feat.device) + # normalize + factor = feat.new_tensor([[w, h]]) * self.strides[level_idx] + reference_points = reference_points / factor + + # shape (batch_size, c, h_i, w_i) -> (h_i * w_i, batch_size, c) + feat_projected = feat_projected.flatten(2).permute(2, 0, 1) + level_pos_embed = level_pos_embed.flatten(2).permute(2, 0, 1) + padding_mask_resized = padding_mask_resized.flatten(1) + + encoder_input_list.append(feat_projected) + padding_mask_list.append(padding_mask_resized) + level_positional_encoding_list.append(level_pos_embed) + spatial_shapes.append(feat.shape[-2:]) + reference_points_list.append(reference_points) + # shape (batch_size, total_num_query), + # total_num_query=sum([., h_i * w_i,.]) + padding_masks = torch.cat(padding_mask_list, dim=1) + # shape (total_num_query, batch_size, c) + encoder_inputs = torch.cat(encoder_input_list, dim=0) + level_positional_encodings = torch.cat( + level_positional_encoding_list, dim=0) + device = encoder_inputs.device + # shape (num_encoder_levels, 2), from low + # resolution to high resolution + spatial_shapes = torch.as_tensor( + spatial_shapes, dtype=torch.long, device=device) + # shape (0, h_0*w_0, h_0*w_0+h_1*w_1, ...) + level_start_index = torch.cat((spatial_shapes.new_zeros( + (1, )), spatial_shapes.prod(1).cumsum(0)[:-1])) + reference_points = torch.cat(reference_points_list, dim=0) + reference_points = reference_points[None, :, None].repeat( + batch_size, 1, self.num_encoder_levels, 1) + valid_radios = reference_points.new_ones( + (batch_size, self.num_encoder_levels, 2)) + # shape (num_total_query, batch_size, c) + memory = self.encoder( + query=encoder_inputs, + key=None, + value=None, + query_pos=level_positional_encodings, + key_pos=None, + attn_masks=None, + key_padding_mask=None, + query_key_padding_mask=padding_masks, + spatial_shapes=spatial_shapes, + reference_points=reference_points, + level_start_index=level_start_index, + valid_radios=valid_radios) + # (num_total_query, batch_size, c) -> (batch_size, c, num_total_query) + memory = memory.permute(1, 2, 0) + + # from low resolution to high resolution + num_query_per_level = [e[0] * e[1] for e in spatial_shapes] + outs = torch.split(memory, num_query_per_level, dim=-1) + outs = [ + x.reshape(batch_size, -1, spatial_shapes[i][0], + spatial_shapes[i][1]) for i, x in enumerate(outs) + ] + + for i in range(self.num_input_levels - self.num_encoder_levels - 1, -1, + -1): + x = feats[i] + cur_feat = self.lateral_convs[i](x) + y = cur_feat + F.interpolate( + outs[-1], + size=cur_feat.shape[-2:], + mode='bilinear', + align_corners=False) + y = self.output_convs[i](y) + outs.append(y) + multi_scale_features = outs[:self.num_outs] + + mask_feature = self.mask_feature(outs[-1]) + return mask_feature, multi_scale_features \ No newline at end of file diff --git a/segmentation/mmseg_custom/models/plugins/pixel_decoder.py b/segmentation/mmseg_custom/models/plugins/pixel_decoder.py new file mode 100644 index 0000000..62e488f --- /dev/null +++ b/segmentation/mmseg_custom/models/plugins/pixel_decoder.py @@ -0,0 +1,237 @@ +import torch +import torch.nn.functional as F +from mmcv.cnn import PLUGIN_LAYERS, Conv2d, ConvModule, kaiming_init +from mmcv.cnn.bricks.transformer import (build_positional_encoding, + build_transformer_layer_sequence) +from mmcv.runner import BaseModule, ModuleList + + +@PLUGIN_LAYERS.register_module() +class PixelDecoder(BaseModule): + """Pixel decoder with a structure like fpn. + + Args: + in_channels (list[int] | tuple[int]): Number of channels in the + input feature maps. + feat_channels (int): Number channels for feature. + out_channels (int): Number channels for output. + norm_cfg (obj:`mmcv.ConfigDict`|dict): Config for normalization. + Defaults to dict(type='GN', num_groups=32). + act_cfg (obj:`mmcv.ConfigDict`|dict): Config for activation. + Defaults to dict(type='ReLU'). + encoder (obj:`mmcv.ConfigDict`|dict): Config for transorformer + encoder.Defaults to None. + positional_encoding (obj:`mmcv.ConfigDict`|dict): Config for + transformer encoder position encoding. Defaults to + dict(type='SinePositionalEncoding', num_feats=128, + normalize=True). + init_cfg (obj:`mmcv.ConfigDict`|dict): Initialization config dict. + Default: None + """ + def __init__(self, + in_channels, + feat_channels, + out_channels, + norm_cfg=dict(type='GN', num_groups=32), + act_cfg=dict(type='ReLU'), + init_cfg=None): + super().__init__(init_cfg=init_cfg) + self.in_channels = in_channels + self.num_inputs = len(in_channels) + self.lateral_convs = ModuleList() + self.output_convs = ModuleList() + self.use_bias = norm_cfg is None + for i in range(0, self.num_inputs - 1): + l_conv = ConvModule( + in_channels[i], + feat_channels, + kernel_size=1, + bias=self.use_bias, + norm_cfg=norm_cfg, + act_cfg=None) + o_conv = ConvModule( + feat_channels, + feat_channels, + kernel_size=3, + stride=1, + padding=1, + bias=self.use_bias, + norm_cfg=norm_cfg, + act_cfg=act_cfg) + self.lateral_convs.append(l_conv) + self.output_convs.append(o_conv) + + self.last_feat_conv = ConvModule( + in_channels[-1], + feat_channels, + kernel_size=3, + padding=1, + stride=1, + bias=self.use_bias, + norm_cfg=norm_cfg, + act_cfg=act_cfg) + self.mask_feature = Conv2d( + feat_channels, out_channels, kernel_size=3, stride=1, padding=1) + + def init_weights(self): + """Initialize weights.""" + for i in range(0, self.num_inputs - 2): + kaiming_init(self.lateral_convs[i].conv, a=1) + kaiming_init(self.output_convs[i].conv, a=1) + + kaiming_init(self.mask_feature, a=1) + kaiming_init(self.last_feat_conv, a=1) + + def forward(self, feats, img_metas): + """ + Args: + feats (list[Tensor]): Feature maps of each level. Each has + shape of [bs, c, h, w]. + img_metas (list[dict]): List of image information. Pass in + for creating more accurate padding mask. #! not used here. + + Returns: + tuple: a tuple containing the following: + + - mask_feature (Tensor): Shape [bs, c, h, w]. + - memory (Tensor): Output of last stage of backbone. + Shape [bs, c, h, w]. + """ + y = self.last_feat_conv(feats[-1]) + for i in range(self.num_inputs - 2, -1, -1): + x = feats[i] + cur_fpn = self.lateral_convs[i](x) + y = cur_fpn + \ + F.interpolate(y, size=cur_fpn.shape[-2:], mode='nearest') + y = self.output_convs[i](y) + + mask_feature = self.mask_feature(y) + memory = feats[-1] + return mask_feature, memory + + +@PLUGIN_LAYERS.register_module() +class TransformerEncoderPixelDecoder(PixelDecoder): + """Pixel decoder with transormer encoder inside. + + Args: + in_channels (list[int] | tuple[int]): Number of channels in the + input feature maps. + feat_channels (int): Number channels for feature. + out_channels (int): Number channels for output. + norm_cfg (obj:`mmcv.ConfigDict`|dict): Config for normalization. + Defaults to dict(type='GN', num_groups=32). + act_cfg (obj:`mmcv.ConfigDict`|dict): Config for activation. + Defaults to dict(type='ReLU'). + encoder (obj:`mmcv.ConfigDict`|dict): Config for transorformer + encoder.Defaults to None. + positional_encoding (obj:`mmcv.ConfigDict`|dict): Config for + transformer encoder position encoding. Defaults to + dict(type='SinePositionalEncoding', num_feats=128, + normalize=True). + init_cfg (obj:`mmcv.ConfigDict`|dict): Initialization config dict. + Default: None + """ + def __init__(self, + in_channels, + feat_channels, + out_channels, + norm_cfg=dict(type='GN', num_groups=32), + act_cfg=dict(type='ReLU'), + encoder=None, + positional_encoding=dict( + type='SinePositionalEncoding', + num_feats=128, + normalize=True), + init_cfg=None): + super(TransformerEncoderPixelDecoder, self).__init__( + in_channels, + feat_channels, + out_channels, + norm_cfg, + act_cfg, + init_cfg=init_cfg) + self.last_feat_conv = None + + self.encoder = build_transformer_layer_sequence(encoder) + self.encoder_embed_dims = self.encoder.embed_dims + assert self.encoder_embed_dims == feat_channels, 'embed_dims({}) of ' \ + 'tranformer encoder must equal to feat_channels({})'.format( + feat_channels, self.encoder_embed_dims) + self.positional_encoding = build_positional_encoding( + positional_encoding) + self.encoder_in_proj = Conv2d( + in_channels[-1], feat_channels, kernel_size=1) + self.encoder_out_proj = ConvModule( + feat_channels, + feat_channels, + kernel_size=3, + stride=1, + padding=1, + bias=self.use_bias, + norm_cfg=norm_cfg, + act_cfg=act_cfg) + + def init_weights(self): + """Initialize weights.""" + for i in range(0, self.num_inputs - 2): + kaiming_init(self.lateral_convs[i].conv, a=1) + kaiming_init(self.output_convs[i].conv, a=1) + + kaiming_init(self.mask_feature, a=1) + kaiming_init(self.encoder_in_proj, a=1) + kaiming_init(self.encoder_out_proj.conv, a=1) + + def forward(self, feats, img_metas): + """ + Args: + feats (list[Tensor]): Feature maps of each level. Each has + shape of [bs, c, h, w]. + img_metas (list[dict]): List of image information. Pass in + for creating more accurate padding mask. + + Returns: + tuple: a tuple containing the following: + + - mask_feature (Tensor): shape [bs, c, h, w]. + - memory (Tensor): shape [bs, c, h, w]. + """ + feat_last = feats[-1] + bs, c, h, w = feat_last.shape + input_img_h, input_img_w = img_metas[0]['pad_shape'][:-1] + # input_img_h, input_img_w = img_metas[0]['batch_input_shape'] + padding_mask = feat_last.new_ones((bs, input_img_h, input_img_w), + dtype=torch.float32) + for i in range(bs): + img_h, img_w, _ = img_metas[i]['img_shape'] + padding_mask[i, :img_h, :img_w] = 0 + padding_mask = F.interpolate( + padding_mask.unsqueeze(1), + size=feat_last.shape[-2:], + mode='nearest').to(torch.bool).squeeze(1) + + pos_embed = self.positional_encoding(padding_mask) + feat_last = self.encoder_in_proj(feat_last) + # [bs, c, h, w] -> [nq, bs, dim] + feat_last = feat_last.flatten(2).permute(2, 0, 1) + pos_embed = pos_embed.flatten(2).permute(2, 0, 1) + padding_mask = padding_mask.flatten(1) # [bs, h, w] -> [bs, h*w] + memory = self.encoder( + query=feat_last, + key=None, + value=None, + query_pos=pos_embed, + query_key_padding_mask=padding_mask) + # [nq, bs, em] -> [bs, c, h, w] + memory = memory.permute(1, 2, 0).view(bs, self.encoder_embed_dims, h, + w) + y = self.encoder_out_proj(memory) + for i in range(self.num_inputs - 2, -1, -1): + x = feats[i] + cur_fpn = self.lateral_convs[i](x) + y = cur_fpn + \ + F.interpolate(y, size=cur_fpn.shape[-2:], mode='nearest') + y = self.output_convs[i](y) + + mask_feature = self.mask_feature(y) + return mask_feature, memory diff --git a/segmentation/mmseg_custom/models/segmentors/__init__.py b/segmentation/mmseg_custom/models/segmentors/__init__.py new file mode 100644 index 0000000..f380c9e --- /dev/null +++ b/segmentation/mmseg_custom/models/segmentors/__init__.py @@ -0,0 +1,5 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from .encoder_decoder_mask2former import EncoderDecoderMask2Former +from .encoder_decoder_mask2former_aug import EncoderDecoderMask2FormerAug + +__all__ = ['EncoderDecoderMask2Former', 'EncoderDecoderMask2FormerAug'] diff --git a/segmentation/mmseg_custom/models/segmentors/encoder_decoder_mask2former.py b/segmentation/mmseg_custom/models/segmentors/encoder_decoder_mask2former.py new file mode 100644 index 0000000..1906358 --- /dev/null +++ b/segmentation/mmseg_custom/models/segmentors/encoder_decoder_mask2former.py @@ -0,0 +1,285 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +import torch.nn as nn +import torch.nn.functional as F +from mmseg.core import add_prefix +from mmseg.models import builder +from mmseg.models.builder import SEGMENTORS +from mmseg.models.segmentors.base import BaseSegmentor +from mmseg.ops import resize + + +@SEGMENTORS.register_module() +class EncoderDecoderMask2Former(BaseSegmentor): + """Encoder Decoder segmentors. + + EncoderDecoder typically consists of backbone, decode_head, auxiliary_head. + Note that auxiliary_head is only used for deep supervision during training, + which could be dumped during inference. + """ + def __init__(self, + backbone, + decode_head, + neck=None, + auxiliary_head=None, + train_cfg=None, + test_cfg=None, + pretrained=None, + init_cfg=None): + super(EncoderDecoderMask2Former, self).__init__(init_cfg) + if pretrained is not None: + assert backbone.get('pretrained') is None, \ + 'both backbone and segmentor set pretrained weight' + backbone.pretrained = pretrained + self.backbone = builder.build_backbone(backbone) + if neck is not None: + self.neck = builder.build_neck(neck) + decode_head.update(train_cfg=train_cfg) + decode_head.update(test_cfg=test_cfg) + self._init_decode_head(decode_head) + self._init_auxiliary_head(auxiliary_head) + + self.train_cfg = train_cfg + self.test_cfg = test_cfg + + assert self.with_decode_head + + def _init_decode_head(self, decode_head): + """Initialize ``decode_head``""" + self.decode_head = builder.build_head(decode_head) + self.align_corners = self.decode_head.align_corners + self.num_classes = self.decode_head.num_classes + + def _init_auxiliary_head(self, auxiliary_head): + """Initialize ``auxiliary_head``""" + if auxiliary_head is not None: + if isinstance(auxiliary_head, list): + self.auxiliary_head = nn.ModuleList() + for head_cfg in auxiliary_head: + self.auxiliary_head.append(builder.build_head(head_cfg)) + else: + self.auxiliary_head = builder.build_head(auxiliary_head) + + def extract_feat(self, img): + """Extract features from images.""" + x = self.backbone(img) + if self.with_neck: + x = self.neck(x) + return x + + def encode_decode(self, img, img_metas): + """Encode images with backbone and decode into a semantic segmentation + map of the same size as input.""" + x = self.extract_feat(img) + out = self._decode_head_forward_test(x, img_metas) + out = resize( + input=out, + size=img.shape[2:], + mode='bilinear', + align_corners=self.align_corners) + return out + + def _decode_head_forward_train(self, x, img_metas, gt_semantic_seg, + **kwargs): + """Run forward function and calculate loss for decode head in + training.""" + losses = dict() + loss_decode = self.decode_head.forward_train(x, img_metas, + gt_semantic_seg, **kwargs) + + losses.update(add_prefix(loss_decode, 'decode')) + return losses + + def _decode_head_forward_test(self, x, img_metas): + """Run forward function and calculate loss for decode head in + inference.""" + seg_logits = self.decode_head.forward_test(x, img_metas, self.test_cfg) + return seg_logits + + def _auxiliary_head_forward_train(self, x, img_metas, gt_semantic_seg): + """Run forward function and calculate loss for auxiliary head in + training.""" + losses = dict() + if isinstance(self.auxiliary_head, nn.ModuleList): + for idx, aux_head in enumerate(self.auxiliary_head): + loss_aux = aux_head.forward_train(x, img_metas, + gt_semantic_seg, + self.train_cfg) + losses.update(add_prefix(loss_aux, f'aux_{idx}')) + else: + loss_aux = self.auxiliary_head.forward_train( + x, img_metas, gt_semantic_seg, self.train_cfg) + losses.update(add_prefix(loss_aux, 'aux')) + + return losses + + def forward_dummy(self, img): + """Dummy forward function.""" + seg_logit = self.encode_decode(img, None) + + return seg_logit + + def forward_train(self, img, img_metas, gt_semantic_seg, **kwargs): + """Forward function for training. + + Args: + img (Tensor): Input images. + img_metas (list[dict]): List of image info dict where each dict + has: 'img_shape', 'scale_factor', 'flip', and may also contain + 'filename', 'ori_shape', 'pad_shape', and 'img_norm_cfg'. + For details on the values of these keys see + `mmseg/datasets/pipelines/formatting.py:Collect`. + gt_semantic_seg (Tensor): Semantic segmentation masks + used if the architecture supports semantic segmentation task. + + Returns: + dict[str, Tensor]: a dictionary of loss components + """ + + x = self.extract_feat(img) + + losses = dict() + + loss_decode = self._decode_head_forward_train(x, img_metas, + gt_semantic_seg, + **kwargs) + losses.update(loss_decode) + + if self.with_auxiliary_head: + loss_aux = self._auxiliary_head_forward_train( + x, img_metas, gt_semantic_seg) + losses.update(loss_aux) + + return losses + + # TODO refactor + def slide_inference(self, img, img_meta, rescale): + """Inference by sliding-window with overlap. + + If h_crop > h_img or w_crop > w_img, the small patch will be used to + decode without padding. + """ + + h_stride, w_stride = self.test_cfg.stride + h_crop, w_crop = self.test_cfg.crop_size + batch_size, _, h_img, w_img = img.size() + num_classes = self.num_classes + h_grids = max(h_img - h_crop + h_stride - 1, 0) // h_stride + 1 + w_grids = max(w_img - w_crop + w_stride - 1, 0) // w_stride + 1 + preds = img.new_zeros((batch_size, num_classes, h_img, w_img)) + count_mat = img.new_zeros((batch_size, 1, h_img, w_img)) + for h_idx in range(h_grids): + for w_idx in range(w_grids): + y1 = h_idx * h_stride + x1 = w_idx * w_stride + y2 = min(y1 + h_crop, h_img) + x2 = min(x1 + w_crop, w_img) + y1 = max(y2 - h_crop, 0) + x1 = max(x2 - w_crop, 0) + crop_img = img[:, :, y1:y2, x1:x2] + crop_seg_logit = self.encode_decode(crop_img, img_meta) + preds += F.pad(crop_seg_logit, + (int(x1), int(preds.shape[3] - x2), int(y1), + int(preds.shape[2] - y2))) + + count_mat[:, :, y1:y2, x1:x2] += 1 + assert (count_mat == 0).sum() == 0 + if torch.onnx.is_in_onnx_export(): + # cast count_mat to constant while exporting to ONNX + count_mat = torch.from_numpy( + count_mat.cpu().detach().numpy()).to(device=img.device) + preds = preds / count_mat + if rescale: + preds = resize( + preds, + size=img_meta[0]['ori_shape'][:2], + mode='bilinear', + align_corners=self.align_corners, + warning=False) + return preds + + def whole_inference(self, img, img_meta, rescale): + """Inference with full image.""" + + seg_logit = self.encode_decode(img, img_meta) + if rescale: + # support dynamic shape for onnx + if torch.onnx.is_in_onnx_export(): + size = img.shape[2:] + else: + size = img_meta[0]['ori_shape'][:2] + seg_logit = resize( + seg_logit, + size=size, + mode='bilinear', + align_corners=self.align_corners, + warning=False) + + return seg_logit + + def inference(self, img, img_meta, rescale): + """Inference with slide/whole style. + + Args: + img (Tensor): The input image of shape (N, 3, H, W). + img_meta (dict): Image info dict where each dict has: 'img_shape', + 'scale_factor', 'flip', and may also contain + 'filename', 'ori_shape', 'pad_shape', and 'img_norm_cfg'. + For details on the values of these keys see + `mmseg/datasets/pipelines/formatting.py:Collect`. + rescale (bool): Whether rescale back to original shape. + + Returns: + Tensor: The output segmentation map. + """ + + assert self.test_cfg.mode in ['slide', 'whole'] + ori_shape = img_meta[0]['ori_shape'] + assert all(_['ori_shape'] == ori_shape for _ in img_meta) + if self.test_cfg.mode == 'slide': + seg_logit = self.slide_inference(img, img_meta, rescale) + else: + seg_logit = self.whole_inference(img, img_meta, rescale) + output = F.softmax(seg_logit, dim=1) + flip = img_meta[0]['flip'] + if flip: + flip_direction = img_meta[0]['flip_direction'] + assert flip_direction in ['horizontal', 'vertical'] + if flip_direction == 'horizontal': + output = output.flip(dims=(3,)) + elif flip_direction == 'vertical': + output = output.flip(dims=(2,)) + + return output + + def simple_test(self, img, img_meta, rescale=True): + """Simple test with single image.""" + seg_logit = self.inference(img, img_meta, rescale) + seg_pred = seg_logit.argmax(dim=1) + if torch.onnx.is_in_onnx_export(): + # our inference backend only support 4D output + seg_pred = seg_pred.unsqueeze(0) + return seg_pred + seg_pred = seg_pred.cpu().numpy() + # unravel batch dim + seg_pred = list(seg_pred) + return seg_pred + + def aug_test(self, imgs, img_metas, rescale=True): + """Test with augmentations. + + Only rescale=True is supported. + """ + # aug_test rescale all imgs back to ori_shape for now + assert rescale + # to save memory, we get augmented seg logit inplace + seg_logit = self.inference(imgs[0], img_metas[0], rescale) + for i in range(1, len(imgs)): + cur_seg_logit = self.inference(imgs[i], img_metas[i], rescale) + seg_logit += cur_seg_logit + seg_logit /= len(imgs) + seg_pred = seg_logit.argmax(dim=1) + seg_pred = seg_pred.cpu().numpy() + # unravel batch dim + seg_pred = list(seg_pred) + return seg_pred diff --git a/segmentation/mmseg_custom/models/segmentors/encoder_decoder_mask2former_aug.py b/segmentation/mmseg_custom/models/segmentors/encoder_decoder_mask2former_aug.py new file mode 100644 index 0000000..7dc54cf --- /dev/null +++ b/segmentation/mmseg_custom/models/segmentors/encoder_decoder_mask2former_aug.py @@ -0,0 +1,289 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +import torch.nn as nn +import torch.nn.functional as F +from mmseg.core import add_prefix +from mmseg.models import builder +from mmseg.models.builder import SEGMENTORS +from mmseg.models.segmentors.base import BaseSegmentor +from mmseg.ops import resize + + +@SEGMENTORS.register_module() +class EncoderDecoderMask2FormerAug(BaseSegmentor): + """Encoder Decoder segmentors. + + EncoderDecoder typically consists of backbone, decode_head, auxiliary_head. + Note that auxiliary_head is only used for deep supervision during training, + which could be dumped during inference. + """ + def __init__(self, + backbone, + decode_head, + neck=None, + auxiliary_head=None, + train_cfg=None, + test_cfg=None, + pretrained=None, + init_cfg=None): + super(EncoderDecoderMask2FormerAug, self).__init__(init_cfg) + if pretrained is not None: + assert backbone.get('pretrained') is None, \ + 'both backbone and segmentor set pretrained weight' + backbone.pretrained = pretrained + self.backbone = builder.build_backbone(backbone) + if neck is not None: + self.neck = builder.build_neck(neck) + decode_head.update(train_cfg=train_cfg) + decode_head.update(test_cfg=test_cfg) + self._init_decode_head(decode_head) + self._init_auxiliary_head(auxiliary_head) + + self.train_cfg = train_cfg + self.test_cfg = test_cfg + + assert self.with_decode_head + + def _init_decode_head(self, decode_head): + """Initialize ``decode_head``""" + self.decode_head = builder.build_head(decode_head) + self.align_corners = self.decode_head.align_corners + self.num_classes = self.decode_head.num_classes + + def _init_auxiliary_head(self, auxiliary_head): + """Initialize ``auxiliary_head``""" + if auxiliary_head is not None: + if isinstance(auxiliary_head, list): + self.auxiliary_head = nn.ModuleList() + for head_cfg in auxiliary_head: + self.auxiliary_head.append(builder.build_head(head_cfg)) + else: + self.auxiliary_head = builder.build_head(auxiliary_head) + + def extract_feat(self, img): + """Extract features from images.""" + x = self.backbone(img) + if self.with_neck: + x = self.neck(x) + return x + + def encode_decode(self, img, img_metas): + """Encode images with backbone and decode into a semantic segmentation + map of the same size as input.""" + x = self.extract_feat(img) + out = self._decode_head_forward_test(x, img_metas) + out = resize( + input=out, + size=img.shape[2:], + mode='bilinear', + align_corners=self.align_corners) + return out + + def _decode_head_forward_train(self, x, img_metas, gt_semantic_seg, + **kwargs): + """Run forward function and calculate loss for decode head in + training.""" + losses = dict() + loss_decode = self.decode_head.forward_train(x, img_metas, + gt_semantic_seg, **kwargs) + + losses.update(add_prefix(loss_decode, 'decode')) + return losses + + def _decode_head_forward_test(self, x, img_metas): + """Run forward function and calculate loss for decode head in + inference.""" + seg_logits = self.decode_head.forward_test(x, img_metas, self.test_cfg) + return seg_logits + + def _auxiliary_head_forward_train(self, x, img_metas, gt_semantic_seg): + """Run forward function and calculate loss for auxiliary head in + training.""" + losses = dict() + if isinstance(self.auxiliary_head, nn.ModuleList): + for idx, aux_head in enumerate(self.auxiliary_head): + loss_aux = aux_head.forward_train(x, img_metas, + gt_semantic_seg, + self.train_cfg) + losses.update(add_prefix(loss_aux, f'aux_{idx}')) + else: + loss_aux = self.auxiliary_head.forward_train( + x, img_metas, gt_semantic_seg, self.train_cfg) + losses.update(add_prefix(loss_aux, 'aux')) + + return losses + + def forward_dummy(self, img): + """Dummy forward function.""" + seg_logit = self.encode_decode(img, None) + + return seg_logit + + def forward_train(self, img, img_metas, gt_semantic_seg, **kwargs): + """Forward function for training. + + Args: + img (Tensor): Input images. + img_metas (list[dict]): List of image info dict where each dict + has: 'img_shape', 'scale_factor', 'flip', and may also contain + 'filename', 'ori_shape', 'pad_shape', and 'img_norm_cfg'. + For details on the values of these keys see + `mmseg/datasets/pipelines/formatting.py:Collect`. + gt_semantic_seg (Tensor): Semantic segmentation masks + used if the architecture supports semantic segmentation task. + + Returns: + dict[str, Tensor]: a dictionary of loss components + """ + + x = self.extract_feat(img) + + losses = dict() + + loss_decode = self._decode_head_forward_train(x, img_metas, + gt_semantic_seg, + **kwargs) + losses.update(loss_decode) + + if self.with_auxiliary_head: + loss_aux = self._auxiliary_head_forward_train( + x, img_metas, gt_semantic_seg) + losses.update(loss_aux) + + return losses + + # TODO refactor + def slide_inference(self, img, img_meta, rescale, unpad=True): + """Inference by sliding-window with overlap. + + If h_crop > h_img or w_crop > w_img, the small patch will be used to + decode without padding. + """ + + h_stride, w_stride = self.test_cfg.stride + h_crop, w_crop = self.test_cfg.crop_size + batch_size, _, h_img, w_img = img.size() + num_classes = self.num_classes + h_grids = max(h_img - h_crop + h_stride - 1, 0) // h_stride + 1 + w_grids = max(w_img - w_crop + w_stride - 1, 0) // w_stride + 1 + preds = img.new_zeros((batch_size, num_classes, h_img, w_img)) + count_mat = img.new_zeros((batch_size, 1, h_img, w_img)) + for h_idx in range(h_grids): + for w_idx in range(w_grids): + y1 = h_idx * h_stride + x1 = w_idx * w_stride + y2 = min(y1 + h_crop, h_img) + x2 = min(x1 + w_crop, w_img) + y1 = max(y2 - h_crop, 0) + x1 = max(x2 - w_crop, 0) + crop_img = img[:, :, y1:y2, x1:x2] + crop_seg_logit = self.encode_decode(crop_img, img_meta) + preds += F.pad(crop_seg_logit, + (int(x1), int(preds.shape[3] - x2), int(y1), + int(preds.shape[2] - y2))) + + count_mat[:, :, y1:y2, x1:x2] += 1 + assert (count_mat == 0).sum() == 0 + if torch.onnx.is_in_onnx_export(): + # cast count_mat to constant while exporting to ONNX + count_mat = torch.from_numpy( + count_mat.cpu().detach().numpy()).to(device=img.device) + preds = preds / count_mat + + if unpad: + unpad_h, unpad_w = img_meta[0]['img_shape'][:2] + # logging.info(preds.shape, img_meta[0]) + preds = preds[:, :, :unpad_h, :unpad_w] + if rescale: + preds = resize(preds, + size=img_meta[0]['ori_shape'][:2], + mode='bilinear', + align_corners=self.align_corners, + warning=False) + return preds + + def whole_inference(self, img, img_meta, rescale): + """Inference with full image.""" + + seg_logit = self.encode_decode(img, img_meta) + if rescale: + # support dynamic shape for onnx + if torch.onnx.is_in_onnx_export(): + size = img.shape[2:] + else: + size = img_meta[0]['ori_shape'][:2] + seg_logit = resize( + seg_logit, + size=size, + mode='bilinear', + align_corners=self.align_corners, + warning=False) + + return seg_logit + + def inference(self, img, img_meta, rescale): + """Inference with slide/whole style. + + Args: + img (Tensor): The input image of shape (N, 3, H, W). + img_meta (dict): Image info dict where each dict has: 'img_shape', + 'scale_factor', 'flip', and may also contain + 'filename', 'ori_shape', 'pad_shape', and 'img_norm_cfg'. + For details on the values of these keys see + `mmseg/datasets/pipelines/formatting.py:Collect`. + rescale (bool): Whether rescale back to original shape. + + Returns: + Tensor: The output segmentation map. + """ + + assert self.test_cfg.mode in ['slide', 'whole'] + ori_shape = img_meta[0]['ori_shape'] + assert all(_['ori_shape'] == ori_shape for _ in img_meta) + if self.test_cfg.mode == 'slide': + seg_logit = self.slide_inference(img, img_meta, rescale) + else: + seg_logit = self.whole_inference(img, img_meta, rescale) + output = F.softmax(seg_logit, dim=1) + flip = img_meta[0]['flip'] + if flip: + flip_direction = img_meta[0]['flip_direction'] + assert flip_direction in ['horizontal', 'vertical'] + if flip_direction == 'horizontal': + output = output.flip(dims=(3, )) + elif flip_direction == 'vertical': + output = output.flip(dims=(2, )) + + return output + + def simple_test(self, img, img_meta, rescale=True): + """Simple test with single image.""" + seg_logit = self.inference(img, img_meta, rescale) + seg_pred = seg_logit.argmax(dim=1) + if torch.onnx.is_in_onnx_export(): + # our inference backend only support 4D output + seg_pred = seg_pred.unsqueeze(0) + return seg_pred + seg_pred = seg_pred.cpu().numpy() + # unravel batch dim + seg_pred = list(seg_pred) + return seg_pred + + def aug_test(self, imgs, img_metas, rescale=True): + """Test with augmentations. + + Only rescale=True is supported. + """ + # aug_test rescale all imgs back to ori_shape for now + assert rescale + # to save memory, we get augmented seg logit inplace + seg_logit = self.inference(imgs[0], img_metas[0], rescale) + for i in range(1, len(imgs)): + cur_seg_logit = self.inference(imgs[i], img_metas[i], rescale) + seg_logit += cur_seg_logit + seg_logit /= len(imgs) + seg_pred = seg_logit.argmax(dim=1) + seg_pred = seg_pred.cpu().numpy() + # unravel batch dim + seg_pred = list(seg_pred) + return seg_pred diff --git a/segmentation/mmseg_custom/models/utils/__init__.py b/segmentation/mmseg_custom/models/utils/__init__.py new file mode 100644 index 0000000..ebc710f --- /dev/null +++ b/segmentation/mmseg_custom/models/utils/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) Shanghai AI Lab. All rights reserved. +from .assigner import MaskHungarianAssigner +from .point_sample import get_uncertain_point_coords_with_randomness +from .positional_encoding import (LearnedPositionalEncoding, + SinePositionalEncoding) +from .transformer import (DetrTransformerDecoder, DetrTransformerDecoderLayer, + DynamicConv, Transformer) + +__all__ = [ + 'DetrTransformerDecoderLayer', 'DetrTransformerDecoder', 'DynamicConv', + 'Transformer', 'LearnedPositionalEncoding', 'SinePositionalEncoding', + 'MaskHungarianAssigner', 'get_uncertain_point_coords_with_randomness' +] diff --git a/segmentation/mmseg_custom/models/utils/assigner.py b/segmentation/mmseg_custom/models/utils/assigner.py new file mode 100644 index 0000000..1d60289 --- /dev/null +++ b/segmentation/mmseg_custom/models/utils/assigner.py @@ -0,0 +1,165 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from abc import ABCMeta, abstractmethod + +import torch +import torch.nn.functional as F + +from ..builder import MASK_ASSIGNERS, build_match_cost + +try: + from scipy.optimize import linear_sum_assignment +except ImportError: + linear_sum_assignment = None + + +class AssignResult(metaclass=ABCMeta): + """Collection of assign results.""" + def __init__(self, num_gts, gt_inds, labels): + self.num_gts = num_gts + self.gt_inds = gt_inds + self.labels = labels + + @property + def info(self): + info = { + 'num_gts': self.num_gts, + 'gt_inds': self.gt_inds, + 'labels': self.labels, + } + return info + + +class BaseAssigner(metaclass=ABCMeta): + """Base assigner that assigns boxes to ground truth boxes.""" + @abstractmethod + def assign(self, masks, gt_masks, gt_masks_ignore=None, gt_labels=None): + """Assign boxes to either a ground truth boxes or a negative boxes.""" + pass + + +@MASK_ASSIGNERS.register_module() +class MaskHungarianAssigner(BaseAssigner): + """Computes one-to-one matching between predictions and ground truth for + mask. + + This class computes an assignment between the targets and the predictions + based on the costs. The costs are weighted sum of three components: + classification cost, regression L1 cost and regression iou cost. The + targets don't include the no_object, so generally there are more + predictions than targets. After the one-to-one matching, the un-matched + are treated as backgrounds. Thus each query prediction will be assigned + with `0` or a positive integer indicating the ground truth index: + + - 0: negative sample, no assigned gt + - positive integer: positive sample, index (1-based) of assigned gt + + Args: + cls_cost (obj:`mmcv.ConfigDict`|dict): Classification cost config. + mask_cost (obj:`mmcv.ConfigDict`|dict): Mask cost config. + dice_cost (obj:`mmcv.ConfigDict`|dict): Dice cost config. + """ + def __init__(self, + cls_cost=dict(type='ClassificationCost', weight=1.0), + dice_cost=dict(type='DiceCost', weight=1.0), + mask_cost=dict(type='MaskFocalCost', weight=1.0)): + self.cls_cost = build_match_cost(cls_cost) + self.dice_cost = build_match_cost(dice_cost) + self.mask_cost = build_match_cost(mask_cost) + + def assign(self, + cls_pred, + mask_pred, + gt_labels, + gt_masks, + img_meta, + gt_masks_ignore=None, + eps=1e-7): + """Computes one-to-one matching based on the weighted costs. + + This method assign each query prediction to a ground truth or + background. The `assigned_gt_inds` with -1 means don't care, + 0 means negative sample, and positive number is the index (1-based) + of assigned gt. + The assignment is done in the following steps, the order matters. + + 1. assign every prediction to -1 + 2. compute the weighted costs + 3. do Hungarian matching on CPU based on the costs + 4. assign all to 0 (background) first, then for each matched pair + between predictions and gts, treat this prediction as foreground + and assign the corresponding gt index (plus 1) to it. + + Args: + mask_pred (Tensor): Predicted mask, shape [num_query, h, w] + cls_pred (Tensor): Predicted classification logits, shape + [num_query, num_class]. + gt_masks (Tensor): Ground truth mask, shape [num_gt, h, w]. + gt_labels (Tensor): Label of `gt_masks`, shape (num_gt,). + img_meta (dict): Meta information for current image. + gt_masks_ignore (Tensor, optional): Ground truth masks that are + labelled as `ignored`. Default None. + eps (int | float, optional): A value added to the denominator for + numerical stability. Default 1e-7. + + Returns: + :obj:`AssignResult`: The assigned result. + """ + assert gt_masks_ignore is None, \ + 'Only case when gt_masks_ignore is None is supported.' + num_gts, num_queries = gt_labels.shape[0], cls_pred.shape[0] + + # 1. assign -1 by default + assigned_gt_inds = cls_pred.new_full((num_queries, ), + -1, + dtype=torch.long) + assigned_labels = cls_pred.new_full((num_queries, ), + -1, + dtype=torch.long) + if num_gts == 0 or num_queries == 0: + # No ground truth or boxes, return empty assignment + if num_gts == 0: + # No ground truth, assign all to background + assigned_gt_inds[:] = 0 + return AssignResult( + num_gts, assigned_gt_inds, labels=assigned_labels) + + # 2. compute the weighted costs + # classification and maskcost. + if self.cls_cost.weight != 0 and cls_pred is not None: + cls_cost = self.cls_cost(cls_pred, gt_labels) + else: + cls_cost = 0 + + if self.mask_cost.weight != 0: + # mask_pred shape = [nq, h, w] + # gt_mask shape = [ng, h, w] + # mask_cost shape = [nq, ng] + mask_cost = self.mask_cost(mask_pred, gt_masks) + else: + mask_cost = 0 + + if self.dice_cost.weight != 0: + dice_cost = self.dice_cost(mask_pred, gt_masks) + else: + dice_cost = 0 + cost = cls_cost + mask_cost + dice_cost + + # 3. do Hungarian matching on CPU using linear_sum_assignment + cost = cost.detach().cpu() + if linear_sum_assignment is None: + raise ImportError('Please run "pip install scipy" ' + 'to install scipy first.') + + matched_row_inds, matched_col_inds = linear_sum_assignment(cost) + matched_row_inds = torch.from_numpy(matched_row_inds).to( + cls_pred.device) + matched_col_inds = torch.from_numpy(matched_col_inds).to( + cls_pred.device) + + # 4. assign backgrounds and foregrounds + # assign all indices to backgrounds first + assigned_gt_inds[:] = 0 + # assign foregrounds based on matching results + assigned_gt_inds[matched_row_inds] = matched_col_inds + 1 + assigned_labels[matched_row_inds] = gt_labels[matched_col_inds] + return AssignResult(num_gts, assigned_gt_inds, labels=assigned_labels) diff --git a/segmentation/mmseg_custom/models/utils/point_sample.py b/segmentation/mmseg_custom/models/utils/point_sample.py new file mode 100644 index 0000000..ac4b2da --- /dev/null +++ b/segmentation/mmseg_custom/models/utils/point_sample.py @@ -0,0 +1,87 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +from mmcv.ops import point_sample + + +def get_uncertainty(mask_pred, labels): + """Estimate uncertainty based on pred logits. + + We estimate uncertainty as L1 distance between 0.0 and the logits + prediction in 'mask_pred' for the foreground class in `classes`. + + Args: + mask_pred (Tensor): mask predication logits, shape (num_rois, + num_classes, mask_height, mask_width). + + labels (list[Tensor]): Either predicted or ground truth label for + each predicted mask, of length num_rois. + + Returns: + scores (Tensor): Uncertainty scores with the most uncertain + locations having the highest uncertainty score, + shape (num_rois, 1, mask_height, mask_width) + """ + if mask_pred.shape[1] == 1: + gt_class_logits = mask_pred.clone() + else: + inds = torch.arange(mask_pred.shape[0], device=mask_pred.device) + gt_class_logits = mask_pred[inds, labels].unsqueeze(1) + return -torch.abs(gt_class_logits) + + +def get_uncertain_point_coords_with_randomness(mask_pred, labels, num_points, + oversample_ratio, + importance_sample_ratio): + """Get ``num_points`` most uncertain points with random points during + train. + + Sample points in [0, 1] x [0, 1] coordinate space based on their + uncertainty. The uncertainties are calculated for each point using + 'get_uncertainty()' function that takes point's logit prediction as + input. + + Args: + mask_pred (Tensor): A tensor of shape (num_rois, num_classes, + mask_height, mask_width) for class-specific or class-agnostic + prediction. + labels (list): The ground truth class for each instance. + num_points (int): The number of points to sample. + oversample_ratio (int): Oversampling parameter. + importance_sample_ratio (float): Ratio of points that are sampled + via importnace sampling. + + Returns: + point_coords (Tensor): A tensor of shape (num_rois, num_points, 2) + that contains the coordinates sampled points. + """ + assert oversample_ratio >= 1 + assert 0 <= importance_sample_ratio <= 1 + batch_size = mask_pred.shape[0] + num_sampled = int(num_points * oversample_ratio) + point_coords = torch.rand( + batch_size, num_sampled, 2, device=mask_pred.device) + point_logits = point_sample(mask_pred, point_coords) + # It is crucial to calculate uncertainty based on the sampled + # prediction value for the points. Calculating uncertainties of the + # coarse predictions first and sampling them for points leads to + # incorrect results. To illustrate this: assume uncertainty func( + # logits)=-abs(logits), a sampled point between two coarse + # predictions with -1 and 1 logits has 0 logits, and therefore 0 + # uncertainty value. However, if we calculate uncertainties for the + # coarse predictions first, both will have -1 uncertainty, + # and sampled point will get -1 uncertainty. + point_uncertainties = get_uncertainty(point_logits, labels) + num_uncertain_points = int(importance_sample_ratio * num_points) + num_random_points = num_points - num_uncertain_points + idx = torch.topk( + point_uncertainties[:, 0, :], k=num_uncertain_points, dim=1)[1] + shift = num_sampled * torch.arange( + batch_size, dtype=torch.long, device=mask_pred.device) + idx += shift[:, None] + point_coords = point_coords.view(-1, 2)[idx.view(-1), :].view( + batch_size, num_uncertain_points, 2) + if num_random_points > 0: + rand_roi_coords = torch.rand( + batch_size, num_random_points, 2, device=mask_pred.device) + point_coords = torch.cat((point_coords, rand_roi_coords), dim=1) + return point_coords \ No newline at end of file diff --git a/segmentation/mmseg_custom/models/utils/positional_encoding.py b/segmentation/mmseg_custom/models/utils/positional_encoding.py new file mode 100644 index 0000000..426e058 --- /dev/null +++ b/segmentation/mmseg_custom/models/utils/positional_encoding.py @@ -0,0 +1,161 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import math + +import torch +import torch.nn as nn +from mmcv.cnn.bricks.transformer import POSITIONAL_ENCODING +from mmcv.runner import BaseModule + + +@POSITIONAL_ENCODING.register_module() +class SinePositionalEncoding(BaseModule): + """Position encoding with sine and cosine functions. + + See `End-to-End Object Detection with Transformers + `_ for details. + + Args: + num_feats (int): The feature dimension for each position + along x-axis or y-axis. Note the final returned dimension + for each position is 2 times of this value. + temperature (int, optional): The temperature used for scaling + the position embedding. Defaults to 10000. + normalize (bool, optional): Whether to normalize the position + embedding. Defaults to False. + scale (float, optional): A scale factor that scales the position + embedding. The scale will be used only when `normalize` is True. + Defaults to 2*pi. + eps (float, optional): A value added to the denominator for + numerical stability. Defaults to 1e-6. + offset (float): offset add to embed when do the normalization. + Defaults to 0. + init_cfg (dict or list[dict], optional): Initialization config dict. + Default: None + """ + def __init__(self, + num_feats, + temperature=10000, + normalize=False, + scale=2 * math.pi, + eps=1e-6, + offset=0., + init_cfg=None): + super(SinePositionalEncoding, self).__init__(init_cfg) + if normalize: + assert isinstance(scale, (float, int)), 'when normalize is set,' \ + 'scale should be provided and in float or int type, ' \ + f'found {type(scale)}' + self.num_feats = num_feats + self.temperature = temperature + self.normalize = normalize + self.scale = scale + self.eps = eps + self.offset = offset + + def forward(self, mask): + """Forward function for `SinePositionalEncoding`. + + Args: + mask (Tensor): ByteTensor mask. Non-zero values representing + ignored positions, while zero values means valid positions + for this image. Shape [bs, h, w]. + + Returns: + pos (Tensor): Returned position embedding with shape + [bs, num_feats*2, h, w]. + """ + # For convenience of exporting to ONNX, it's required to convert + # `masks` from bool to int. + mask = mask.to(torch.int) + not_mask = 1 - mask # logical_not + y_embed = not_mask.cumsum(1, dtype=torch.float32) + x_embed = not_mask.cumsum(2, dtype=torch.float32) + if self.normalize: + y_embed = (y_embed + self.offset) / \ + (y_embed[:, -1:, :] + self.eps) * self.scale + x_embed = (x_embed + self.offset) / \ + (x_embed[:, :, -1:] + self.eps) * self.scale + dim_t = torch.arange( + self.num_feats, dtype=torch.float32, device=mask.device) + dim_t = self.temperature**(2 * (dim_t // 2) / self.num_feats) + pos_x = x_embed[:, :, :, None] / dim_t + pos_y = y_embed[:, :, :, None] / dim_t + # use `view` instead of `flatten` for dynamically exporting to ONNX + B, H, W = mask.size() + pos_x = torch.stack( + (pos_x[:, :, :, 0::2].sin(), pos_x[:, :, :, 1::2].cos()), + dim=4).view(B, H, W, -1) + pos_y = torch.stack( + (pos_y[:, :, :, 0::2].sin(), pos_y[:, :, :, 1::2].cos()), + dim=4).view(B, H, W, -1) + pos = torch.cat((pos_y, pos_x), dim=3).permute(0, 3, 1, 2) + return pos + + def __repr__(self): + """str: a string that describes the module""" + repr_str = self.__class__.__name__ + repr_str += f'(num_feats={self.num_feats}, ' + repr_str += f'temperature={self.temperature}, ' + repr_str += f'normalize={self.normalize}, ' + repr_str += f'scale={self.scale}, ' + repr_str += f'eps={self.eps})' + return repr_str + + +@POSITIONAL_ENCODING.register_module() +class LearnedPositionalEncoding(BaseModule): + """Position embedding with learnable embedding weights. + + Args: + num_feats (int): The feature dimension for each position + along x-axis or y-axis. The final returned dimension for + each position is 2 times of this value. + row_num_embed (int, optional): The dictionary size of row embeddings. + Default 50. + col_num_embed (int, optional): The dictionary size of col embeddings. + Default 50. + init_cfg (dict or list[dict], optional): Initialization config dict. + """ + def __init__(self, + num_feats, + row_num_embed=50, + col_num_embed=50, + init_cfg=dict(type='Uniform', layer='Embedding')): + super(LearnedPositionalEncoding, self).__init__(init_cfg) + self.row_embed = nn.Embedding(row_num_embed, num_feats) + self.col_embed = nn.Embedding(col_num_embed, num_feats) + self.num_feats = num_feats + self.row_num_embed = row_num_embed + self.col_num_embed = col_num_embed + + def forward(self, mask): + """Forward function for `LearnedPositionalEncoding`. + + Args: + mask (Tensor): ByteTensor mask. Non-zero values representing + ignored positions, while zero values means valid positions + for this image. Shape [bs, h, w]. + + Returns: + pos (Tensor): Returned position embedding with shape + [bs, num_feats*2, h, w]. + """ + h, w = mask.shape[-2:] + x = torch.arange(w, device=mask.device) + y = torch.arange(h, device=mask.device) + x_embed = self.col_embed(x) + y_embed = self.row_embed(y) + pos = torch.cat( + (x_embed.unsqueeze(0).repeat(h, 1, 1), y_embed.unsqueeze(1).repeat( + 1, w, 1)), + dim=-1).permute(2, 0, + 1).unsqueeze(0).repeat(mask.shape[0], 1, 1, 1) + return pos + + def __repr__(self): + """str: a string that describes the module""" + repr_str = self.__class__.__name__ + repr_str += f'(num_feats={self.num_feats}, ' + repr_str += f'row_num_embed={self.row_num_embed}, ' + repr_str += f'col_num_embed={self.col_num_embed})' + return repr_str diff --git a/segmentation/mmseg_custom/models/utils/transformer.py b/segmentation/mmseg_custom/models/utils/transformer.py new file mode 100644 index 0000000..3e9d768 --- /dev/null +++ b/segmentation/mmseg_custom/models/utils/transformer.py @@ -0,0 +1,1083 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import math +import warnings +from typing import Sequence + +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.utils.checkpoint as cp +from mmcv.cnn import (Linear, build_activation_layer, build_conv_layer, + build_norm_layer, xavier_init) +from mmcv.cnn.bricks.registry import (TRANSFORMER_LAYER, + TRANSFORMER_LAYER_SEQUENCE, + FEEDFORWARD_NETWORK) +from mmcv.cnn.bricks.drop import build_dropout +from mmcv.cnn.bricks.transformer import (BaseTransformerLayer, + TransformerLayerSequence, + build_transformer_layer_sequence, + build_attention, + build_feedforward_network) +from mmcv.runner.base_module import BaseModule, ModuleList, Sequential +from mmcv.utils import to_2tuple, ConfigDict, deprecated_api_warning +from torch.nn.init import normal_ + +from ..builder import TRANSFORMER + +try: + from mmcv.ops.multi_scale_deform_attn import MultiScaleDeformableAttention + +except ImportError: + warnings.warn( + '`MultiScaleDeformableAttention` in MMCV has been moved to ' + '`mmcv.ops.multi_scale_deform_attn`, please update your MMCV') + from mmcv.cnn.bricks.transformer import MultiScaleDeformableAttention + + +class AdaptivePadding(nn.Module): + """Applies padding to input (if needed) so that input can get fully covered + by filter you specified. It support two modes "same" and "corner". The + "same" mode is same with "SAME" padding mode in TensorFlow, pad zero around + input. The "corner" mode would pad zero to bottom right. + + Args: + kernel_size (int | tuple): Size of the kernel: + stride (int | tuple): Stride of the filter. Default: 1: + dilation (int | tuple): Spacing between kernel elements. + Default: 1 + padding (str): Support "same" and "corner", "corner" mode + would pad zero to bottom right, and "same" mode would + pad zero around input. Default: "corner". + Example: + >>> kernel_size = 16 + >>> stride = 16 + >>> dilation = 1 + >>> input = torch.rand(1, 1, 15, 17) + >>> adap_pad = AdaptivePadding( + >>> kernel_size=kernel_size, + >>> stride=stride, + >>> dilation=dilation, + >>> padding="corner") + >>> out = adap_pad(input) + >>> assert (out.shape[2], out.shape[3]) == (16, 32) + >>> input = torch.rand(1, 1, 16, 17) + >>> out = adap_pad(input) + >>> assert (out.shape[2], out.shape[3]) == (16, 32) + """ + def __init__(self, kernel_size=1, stride=1, dilation=1, padding='corner'): + + super(AdaptivePadding, self).__init__() + + assert padding in ('same', 'corner') + + kernel_size = to_2tuple(kernel_size) + stride = to_2tuple(stride) + padding = to_2tuple(padding) + dilation = to_2tuple(dilation) + + self.padding = padding + self.kernel_size = kernel_size + self.stride = stride + self.dilation = dilation + + def get_pad_shape(self, input_shape): + input_h, input_w = input_shape + kernel_h, kernel_w = self.kernel_size + stride_h, stride_w = self.stride + output_h = math.ceil(input_h / stride_h) + output_w = math.ceil(input_w / stride_w) + pad_h = max((output_h - 1) * stride_h + + (kernel_h - 1) * self.dilation[0] + 1 - input_h, 0) + pad_w = max((output_w - 1) * stride_w + + (kernel_w - 1) * self.dilation[1] + 1 - input_w, 0) + return pad_h, pad_w + + def forward(self, x): + pad_h, pad_w = self.get_pad_shape(x.size()[-2:]) + if pad_h > 0 or pad_w > 0: + if self.padding == 'corner': + x = F.pad(x, [0, pad_w, 0, pad_h]) + elif self.padding == 'same': + x = F.pad(x, [ + pad_w // 2, pad_w - pad_w // 2, pad_h // 2, + pad_h - pad_h // 2 + ]) + return x + + +class PatchMerging(BaseModule): + """Merge patch feature map. + + This layer groups feature map by kernel_size, and applies norm and linear + layers to the grouped feature map. Our implementation uses `nn.Unfold` to + merge patch, which is about 25% faster than original implementation. + Instead, we need to modify pretrained models for compatibility. + + Args: + in_channels (int): The num of input channels. + to gets fully covered by filter and stride you specified.. + Default: True. + out_channels (int): The num of output channels. + kernel_size (int | tuple, optional): the kernel size in the unfold + layer. Defaults to 2. + stride (int | tuple, optional): the stride of the sliding blocks in the + unfold layer. Default: None. (Would be set as `kernel_size`) + padding (int | tuple | string ): The padding length of + embedding conv. When it is a string, it means the mode + of adaptive padding, support "same" and "corner" now. + Default: "corner". + dilation (int | tuple, optional): dilation parameter in the unfold + layer. Default: 1. + bias (bool, optional): Whether to add bias in linear layer or not. + Defaults: False. + norm_cfg (dict, optional): Config dict for normalization layer. + Default: dict(type='LN'). + init_cfg (dict, optional): The extra config for initialization. + Default: None. + """ + def __init__(self, + in_channels, + out_channels, + kernel_size=2, + stride=None, + padding='corner', + dilation=1, + bias=False, + norm_cfg=dict(type='LN'), + init_cfg=None): + super().__init__(init_cfg=init_cfg) + self.in_channels = in_channels + self.out_channels = out_channels + if stride: + stride = stride + else: + stride = kernel_size + + kernel_size = to_2tuple(kernel_size) + stride = to_2tuple(stride) + dilation = to_2tuple(dilation) + + if isinstance(padding, str): + self.adap_padding = AdaptivePadding( + kernel_size=kernel_size, + stride=stride, + dilation=dilation, + padding=padding) + # disable the padding of unfold + padding = 0 + else: + self.adap_padding = None + + padding = to_2tuple(padding) + self.sampler = nn.Unfold( + kernel_size=kernel_size, + dilation=dilation, + padding=padding, + stride=stride) + + sample_dim = kernel_size[0] * kernel_size[1] * in_channels + + if norm_cfg is not None: + self.norm = build_norm_layer(norm_cfg, sample_dim)[1] + else: + self.norm = None + + self.reduction = nn.Linear(sample_dim, out_channels, bias=bias) + + def forward(self, x, input_size): + """ + Args: + x (Tensor): Has shape (B, H*W, C_in). + input_size (tuple[int]): The spatial shape of x, arrange as (H, W). + Default: None. + + Returns: + tuple: Contains merged results and its spatial shape. + + - x (Tensor): Has shape (B, Merged_H * Merged_W, C_out) + - out_size (tuple[int]): Spatial shape of x, arrange as + (Merged_H, Merged_W). + """ + B, L, C = x.shape + assert isinstance(input_size, Sequence), f'Expect ' \ + f'input_size is ' \ + f'`Sequence` ' \ + f'but get {input_size}' + + H, W = input_size + assert L == H * W, 'input feature has wrong size' + + x = x.view(B, H, W, C).permute([0, 3, 1, 2]) # B, C, H, W + # Use nn.Unfold to merge patch. About 25% faster than original method, + # but need to modify pretrained model for compatibility + + if self.adap_padding: + x = self.adap_padding(x) + H, W = x.shape[-2:] + + x = self.sampler(x) + # if kernel_size=2 and stride=2, x should has shape (B, 4*C, H/2*W/2) + + out_h = (H + 2 * self.sampler.padding[0] - self.sampler.dilation[0] * + (self.sampler.kernel_size[0] - 1) - + 1) // self.sampler.stride[0] + 1 + out_w = (W + 2 * self.sampler.padding[1] - self.sampler.dilation[1] * + (self.sampler.kernel_size[1] - 1) - + 1) // self.sampler.stride[1] + 1 + + output_size = (out_h, out_w) + x = x.transpose(1, 2) # B, H/2*W/2, 4*C + x = self.norm(x) if self.norm else x + x = self.reduction(x) + return x, output_size + + +def inverse_sigmoid(x, eps=1e-5): + """Inverse function of sigmoid. + + Args: + x (Tensor): The tensor to do the + inverse. + eps (float): EPS avoid numerical + overflow. Defaults 1e-5. + Returns: + Tensor: The x has passed the inverse + function of sigmoid, has same + shape with input. + """ + x = x.clamp(min=0, max=1) + x1 = x.clamp(min=eps) + x2 = (1 - x).clamp(min=eps) + return torch.log(x1 / x2) + + +@FEEDFORWARD_NETWORK.register_module(force=True) +class FFN(BaseModule): + """Implements feed-forward networks (FFNs) with identity connection. + Args: + embed_dims (int): The feature dimension. Same as + `MultiheadAttention`. Defaults: 256. + feedforward_channels (int): The hidden dimension of FFNs. + Defaults: 1024. + num_fcs (int, optional): The number of fully-connected layers in + FFNs. Default: 2. + act_cfg (dict, optional): The activation config for FFNs. + Default: dict(type='ReLU') + ffn_drop (float, optional): Probability of an element to be + zeroed in FFN. Default 0.0. + add_identity (bool, optional): Whether to add the + identity connection. Default: `True`. + dropout_layer (obj:`ConfigDict`): The dropout_layer used + when adding the shortcut. + init_cfg (obj:`mmcv.ConfigDict`): The Config for initialization. + Default: None. + """ + + @deprecated_api_warning( + { + 'dropout': 'ffn_drop', + 'add_residual': 'add_identity' + }, + cls_name='FFN') + def __init__(self, + embed_dims=256, + feedforward_channels=1024, + num_fcs=2, + act_cfg=dict(type='ReLU', inplace=True), + ffn_drop=0., + dropout_layer=None, + add_identity=True, + init_cfg=None, + with_cp=False, + **kwargs): + super().__init__(init_cfg) + assert num_fcs >= 2, 'num_fcs should be no less ' \ + f'than 2. got {num_fcs}.' + self.embed_dims = embed_dims + self.feedforward_channels = feedforward_channels + self.num_fcs = num_fcs + self.act_cfg = act_cfg + self.activate = build_activation_layer(act_cfg) + self.with_cp = with_cp + layers = [] + in_channels = embed_dims + for _ in range(num_fcs - 1): + layers.append( + Sequential( + Linear(in_channels, feedforward_channels), self.activate, + nn.Dropout(ffn_drop))) + in_channels = feedforward_channels + layers.append(Linear(feedforward_channels, embed_dims)) + layers.append(nn.Dropout(ffn_drop)) + self.layers = Sequential(*layers) + self.dropout_layer = build_dropout( + dropout_layer) if dropout_layer else torch.nn.Identity() + self.add_identity = add_identity + + @deprecated_api_warning({'residual': 'identity'}, cls_name='FFN') + def forward(self, x, identity=None): + """Forward function for `FFN`. + The function would add x to the output tensor if residue is None. + """ + + if self.with_cp and x.requires_grad: + out = cp.checkpoint(self.layers, x) + else: + out = self.layers(x) + + if not self.add_identity: + return self.dropout_layer(out) + if identity is None: + identity = x + return identity + self.dropout_layer(out) + + +@TRANSFORMER_LAYER.register_module() +class DetrTransformerDecoderLayer(BaseTransformerLayer): + """Implements decoder layer in DETR transformer. + + Args: + attn_cfgs (list[`mmcv.ConfigDict`] | list[dict] | dict )): + Configs for self_attention or cross_attention, the order + should be consistent with it in `operation_order`. If it is + a dict, it would be expand to the number of attention in + `operation_order`. + feedforward_channels (int): The hidden dimension for FFNs. + ffn_dropout (float): Probability of an element to be zeroed + in ffn. Default 0.0. + operation_order (tuple[str]): The execution order of operation + in transformer. Such as ('self_attn', 'norm', 'ffn', 'norm'). + Default：None + act_cfg (dict): The activation config for FFNs. Default: `LN` + norm_cfg (dict): Config dict for normalization layer. + Default: `LN`. + ffn_num_fcs (int): The number of fully-connected layers in FFNs. + Default：2. + """ + def __init__(self, + attn_cfgs, + feedforward_channels, + ffn_dropout=0.0, + operation_order=None, + act_cfg=dict(type='ReLU', inplace=True), + norm_cfg=dict(type='LN'), + ffn_num_fcs=2, + **kwargs): + super(DetrTransformerDecoderLayer, self).__init__( + attn_cfgs=attn_cfgs, + feedforward_channels=feedforward_channels, + ffn_dropout=ffn_dropout, + operation_order=operation_order, + act_cfg=act_cfg, + norm_cfg=norm_cfg, + ffn_num_fcs=ffn_num_fcs, + **kwargs) + assert len(operation_order) == 6 + assert set(operation_order) == set( + ['self_attn', 'norm', 'cross_attn', 'ffn']) + + +@TRANSFORMER_LAYER_SEQUENCE.register_module() +class DetrTransformerEncoder(TransformerLayerSequence): + """TransformerEncoder of DETR. + + Args: + post_norm_cfg (dict): Config of last normalization layer. Default： + `LN`. Only used when `self.pre_norm` is `True` + """ + def __init__(self, *args, post_norm_cfg=dict(type='LN'), **kwargs): + super(DetrTransformerEncoder, self).__init__(*args, **kwargs) + if post_norm_cfg is not None: + self.post_norm = build_norm_layer( + post_norm_cfg, self.embed_dims)[1] if self.pre_norm else None + else: + assert not self.pre_norm, f'Use prenorm in ' \ + f'{self.__class__.__name__},' \ + f'Please specify post_norm_cfg' + self.post_norm = None + + def forward(self, *args, **kwargs): + """Forward function for `TransformerCoder`. + + Returns: + Tensor: forwarded results with shape [num_query, bs, embed_dims]. + """ + x = super(DetrTransformerEncoder, self).forward(*args, **kwargs) + if self.post_norm is not None: + x = self.post_norm(x) + return x + + +@TRANSFORMER_LAYER_SEQUENCE.register_module() +class DetrTransformerDecoder(TransformerLayerSequence): + """Implements the decoder in DETR transformer. + + Args: + return_intermediate (bool): Whether to return intermediate outputs. + post_norm_cfg (dict): Config of last normalization layer. Default： + `LN`. + """ + def __init__(self, + *args, + post_norm_cfg=dict(type='LN'), + return_intermediate=False, + **kwargs): + + super(DetrTransformerDecoder, self).__init__(*args, **kwargs) + self.return_intermediate = return_intermediate + if post_norm_cfg is not None: + self.post_norm = build_norm_layer(post_norm_cfg, + self.embed_dims)[1] + else: + self.post_norm = None + + def forward(self, query, *args, **kwargs): + """Forward function for `TransformerDecoder`. + + Args: + query (Tensor): Input query with shape + `(num_query, bs, embed_dims)`. + + Returns: + Tensor: Results with shape [1, num_query, bs, embed_dims] when + return_intermediate is `False`, otherwise it has shape + [num_layers, num_query, bs, embed_dims]. + """ + if not self.return_intermediate: + x = super().forward(query, *args, **kwargs) + if self.post_norm: + x = self.post_norm(x)[None] + return x + + intermediate = [] + for layer in self.layers: + query = layer(query, *args, **kwargs) + if self.return_intermediate: + if self.post_norm is not None: + intermediate.append(self.post_norm(query)) + else: + intermediate.append(query) + return torch.stack(intermediate) + + +@TRANSFORMER.register_module() +class Transformer(BaseModule): + """Implements the DETR transformer. + + Following the official DETR implementation, this module copy-paste + from torch.nn.Transformer with modifications: + + * positional encodings are passed in MultiheadAttention + * extra LN at the end of encoder is removed + * decoder returns a stack of activations from all decoding layers + + See `paper: End-to-End Object Detection with Transformers + `_ for details. + + Args: + encoder (`mmcv.ConfigDict` | Dict): Config of + TransformerEncoder. Defaults to None. + decoder ((`mmcv.ConfigDict` | Dict)): Config of + TransformerDecoder. Defaults to None + init_cfg (obj:`mmcv.ConfigDict`): The Config for initialization. + Defaults to None. + """ + def __init__(self, encoder=None, decoder=None, init_cfg=None): + super(Transformer, self).__init__(init_cfg=init_cfg) + self.encoder = build_transformer_layer_sequence(encoder) + self.decoder = build_transformer_layer_sequence(decoder) + self.embed_dims = self.encoder.embed_dims + + def init_weights(self): + # follow the official DETR to init parameters + for m in self.modules(): + if hasattr(m, 'weight') and m.weight.dim() > 1: + xavier_init(m, distribution='uniform') + self._is_init = True + + def forward(self, x, mask, query_embed, pos_embed): + """Forward function for `Transformer`. + + Args: + x (Tensor): Input query with shape [bs, c, h, w] where + c = embed_dims. + mask (Tensor): The key_padding_mask used for encoder and decoder, + with shape [bs, h, w]. + query_embed (Tensor): The query embedding for decoder, with shape + [num_query, c]. + pos_embed (Tensor): The positional encoding for encoder and + decoder, with the same shape as `x`. + + Returns: + tuple[Tensor]: results of decoder containing the following tensor. + + - out_dec: Output from decoder. If return_intermediate_dec \ + is True output has shape [num_dec_layers, bs, + num_query, embed_dims], else has shape [1, bs, \ + num_query, embed_dims]. + - memory: Output results from encoder, with shape \ + [bs, embed_dims, h, w]. + """ + bs, c, h, w = x.shape + # use `view` instead of `flatten` for dynamically exporting to ONNX + x = x.view(bs, c, -1).permute(2, 0, 1) # [bs, c, h, w] -> [h*w, bs, c] + pos_embed = pos_embed.view(bs, c, -1).permute(2, 0, 1) + query_embed = query_embed.unsqueeze(1).repeat( + 1, bs, 1) # [num_query, dim] -> [num_query, bs, dim] + mask = mask.view(bs, -1) # [bs, h, w] -> [bs, h*w] + memory = self.encoder( + query=x, + key=None, + value=None, + query_pos=pos_embed, + query_key_padding_mask=mask) + target = torch.zeros_like(query_embed) + # out_dec: [num_layers, num_query, bs, dim] + out_dec = self.decoder( + query=target, + key=memory, + value=memory, + key_pos=pos_embed, + query_pos=query_embed, + key_padding_mask=mask) + out_dec = out_dec.transpose(1, 2) + memory = memory.permute(1, 2, 0).reshape(bs, c, h, w) + return out_dec, memory + + +@TRANSFORMER_LAYER_SEQUENCE.register_module() +class DeformableDetrTransformerDecoder(TransformerLayerSequence): + """Implements the decoder in DETR transformer. + + Args: + return_intermediate (bool): Whether to return intermediate outputs. + coder_norm_cfg (dict): Config of last normalization layer. Default： + `LN`. + """ + def __init__(self, *args, return_intermediate=False, **kwargs): + + super(DeformableDetrTransformerDecoder, self).__init__(*args, **kwargs) + self.return_intermediate = return_intermediate + + def forward(self, + query, + *args, + reference_points=None, + valid_ratios=None, + reg_branches=None, + **kwargs): + """Forward function for `TransformerDecoder`. + + Args: + query (Tensor): Input query with shape + `(num_query, bs, embed_dims)`. + reference_points (Tensor): The reference + points of offset. has shape + (bs, num_query, 4) when as_two_stage, + otherwise has shape ((bs, num_query, 2). + valid_ratios (Tensor): The radios of valid + points on the feature map, has shape + (bs, num_levels, 2) + reg_branch: (obj:`nn.ModuleList`): Used for + refining the regression results. Only would + be passed when with_box_refine is True, + otherwise would be passed a `None`. + + Returns: + Tensor: Results with shape [1, num_query, bs, embed_dims] when + return_intermediate is `False`, otherwise it has shape + [num_layers, num_query, bs, embed_dims]. + """ + output = query + intermediate = [] + intermediate_reference_points = [] + for lid, layer in enumerate(self.layers): + if reference_points.shape[-1] == 4: + reference_points_input = reference_points[:, :, None] * \ + torch.cat([valid_ratios, valid_ratios], + -1)[:, None] + else: + assert reference_points.shape[-1] == 2 + reference_points_input = reference_points[:, :, None] * \ + valid_ratios[:, None] + output = layer( + output, + *args, + reference_points=reference_points_input, + **kwargs) + output = output.permute(1, 0, 2) + + if reg_branches is not None: + tmp = reg_branches[lid](output) + if reference_points.shape[-1] == 4: + new_reference_points = tmp + inverse_sigmoid( + reference_points) + new_reference_points = new_reference_points.sigmoid() + else: + assert reference_points.shape[-1] == 2 + new_reference_points = tmp + new_reference_points[..., :2] = tmp[ + ..., :2] + inverse_sigmoid( + reference_points) + new_reference_points = new_reference_points.sigmoid() + reference_points = new_reference_points.detach() + + output = output.permute(1, 0, 2) + if self.return_intermediate: + intermediate.append(output) + intermediate_reference_points.append(reference_points) + + if self.return_intermediate: + return torch.stack(intermediate), torch.stack( + intermediate_reference_points) + + return output, reference_points + + +@TRANSFORMER.register_module() +class DeformableDetrTransformer(Transformer): + """Implements the DeformableDETR transformer. + + Args: + as_two_stage (bool): Generate query from encoder features. + Default: False. + num_feature_levels (int): Number of feature maps from FPN: + Default: 4. + two_stage_num_proposals (int): Number of proposals when set + `as_two_stage` as True. Default: 300. + """ + + def __init__(self, + as_two_stage=False, + num_feature_levels=4, + two_stage_num_proposals=300, + **kwargs): + super(DeformableDetrTransformer, self).__init__(**kwargs) + self.as_two_stage = as_two_stage + self.num_feature_levels = num_feature_levels + self.two_stage_num_proposals = two_stage_num_proposals + self.embed_dims = self.encoder.embed_dims + self.init_layers() + + def init_layers(self): + """Initialize layers of the DeformableDetrTransformer.""" + self.level_embeds = nn.Parameter( + torch.Tensor(self.num_feature_levels, self.embed_dims)) + + if self.as_two_stage: + self.enc_output = nn.Linear(self.embed_dims, self.embed_dims) + self.enc_output_norm = nn.LayerNorm(self.embed_dims) + self.pos_trans = nn.Linear(self.embed_dims * 2, + self.embed_dims * 2) + self.pos_trans_norm = nn.LayerNorm(self.embed_dims * 2) + else: + self.reference_points = nn.Linear(self.embed_dims, 2) + + def init_weights(self): + """Initialize the transformer weights.""" + for p in self.parameters(): + if p.dim() > 1: + nn.init.xavier_uniform_(p) + for m in self.modules(): + if isinstance(m, MultiScaleDeformableAttention): + m.init_weights() + if not self.as_two_stage: + xavier_init(self.reference_points, distribution='uniform', bias=0.) + normal_(self.level_embeds) + + def gen_encoder_output_proposals(self, memory, memory_padding_mask, + spatial_shapes): + """Generate proposals from encoded memory. + + Args: + memory (Tensor) : The output of encoder, + has shape (bs, num_key, embed_dim). num_key is + equal the number of points on feature map from + all level. + memory_padding_mask (Tensor): Padding mask for memory. + has shape (bs, num_key). + spatial_shapes (Tensor): The shape of all feature maps. + has shape (num_level, 2). + + Returns: + tuple: A tuple of feature map and bbox prediction. + + - output_memory (Tensor): The input of decoder, \ + has shape (bs, num_key, embed_dim). num_key is \ + equal the number of points on feature map from \ + all levels. + - output_proposals (Tensor): The normalized proposal \ + after a inverse sigmoid, has shape \ + (bs, num_keys, 4). + """ + + N, S, C = memory.shape + proposals = [] + _cur = 0 + for lvl, (H, W) in enumerate(spatial_shapes): + mask_flatten_ = memory_padding_mask[:, _cur:(_cur + H * W)].view( + N, H, W, 1) + valid_H = torch.sum(~mask_flatten_[:, :, 0, 0], 1) + valid_W = torch.sum(~mask_flatten_[:, 0, :, 0], 1) + + grid_y, grid_x = torch.meshgrid( + torch.linspace( + 0, H - 1, H, dtype=torch.float32, device=memory.device), + torch.linspace( + 0, W - 1, W, dtype=torch.float32, device=memory.device)) + grid = torch.cat([grid_x.unsqueeze(-1), grid_y.unsqueeze(-1)], -1) + + scale = torch.cat([valid_W.unsqueeze(-1), + valid_H.unsqueeze(-1)], 1).view(N, 1, 1, 2) + grid = (grid.unsqueeze(0).expand(N, -1, -1, -1) + 0.5) / scale + wh = torch.ones_like(grid) * 0.05 * (2.0 ** lvl) + proposal = torch.cat((grid, wh), -1).view(N, -1, 4) + proposals.append(proposal) + _cur += (H * W) + output_proposals = torch.cat(proposals, 1) + output_proposals_valid = ((output_proposals > 0.01) & + (output_proposals < 0.99)).all( + -1, keepdim=True) + output_proposals = torch.log(output_proposals / (1 - output_proposals)) + output_proposals = output_proposals.masked_fill( + memory_padding_mask.unsqueeze(-1), float('inf')) + output_proposals = output_proposals.masked_fill( + ~output_proposals_valid, float('inf')) + + output_memory = memory + output_memory = output_memory.masked_fill( + memory_padding_mask.unsqueeze(-1), float(0)) + output_memory = output_memory.masked_fill(~output_proposals_valid, + float(0)) + output_memory = self.enc_output_norm(self.enc_output(output_memory)) + return output_memory, output_proposals + + @staticmethod + def get_reference_points(spatial_shapes, valid_ratios, device): + """Get the reference points used in decoder. + + Args: + spatial_shapes (Tensor): The shape of all + feature maps, has shape (num_level, 2). + valid_ratios (Tensor): The radios of valid + points on the feature map, has shape + (bs, num_levels, 2) + device (obj:`device`): The device where + reference_points should be. + + Returns: + Tensor: reference points used in decoder, has \ + shape (bs, num_keys, num_levels, 2). + """ + reference_points_list = [] + for lvl, (H, W) in enumerate(spatial_shapes): + # TODO check this 0.5 + ref_y, ref_x = torch.meshgrid( + torch.linspace( + 0.5, H - 0.5, H, dtype=torch.float32, device=device), + torch.linspace( + 0.5, W - 0.5, W, dtype=torch.float32, device=device)) + ref_y = ref_y.reshape(-1)[None] / ( + valid_ratios[:, None, lvl, 1] * H) + ref_x = ref_x.reshape(-1)[None] / ( + valid_ratios[:, None, lvl, 0] * W) + ref = torch.stack((ref_x, ref_y), -1) + reference_points_list.append(ref) + reference_points = torch.cat(reference_points_list, 1) + reference_points = reference_points[:, :, None] * valid_ratios[:, None] + return reference_points + + def get_valid_ratio(self, mask): + """Get the valid radios of feature maps of all level.""" + _, H, W = mask.shape + valid_H = torch.sum(~mask[:, :, 0], 1) + valid_W = torch.sum(~mask[:, 0, :], 1) + valid_ratio_h = valid_H.float() / H + valid_ratio_w = valid_W.float() / W + valid_ratio = torch.stack([valid_ratio_w, valid_ratio_h], -1) + return valid_ratio + + def get_proposal_pos_embed(self, + proposals, + num_pos_feats=128, + temperature=10000): + """Get the position embedding of proposal.""" + scale = 2 * math.pi + dim_t = torch.arange( + num_pos_feats, dtype=torch.float32, device=proposals.device) + dim_t = temperature ** (2 * (dim_t // 2) / num_pos_feats) + # N, L, 4 + proposals = proposals.sigmoid() * scale + # N, L, 4, 128 + pos = proposals[:, :, :, None] / dim_t + # N, L, 4, 64, 2 + pos = torch.stack((pos[:, :, :, 0::2].sin(), pos[:, :, :, 1::2].cos()), + dim=4).flatten(2) + return pos + + def forward(self, + mlvl_feats, + mlvl_masks, + query_embed, + mlvl_pos_embeds, + reg_branches=None, + cls_branches=None, + **kwargs): + """Forward function for `Transformer`. + + Args: + mlvl_feats (list(Tensor)): Input queries from + different level. Each element has shape + [bs, embed_dims, h, w]. + mlvl_masks (list(Tensor)): The key_padding_mask from + different level used for encoder and decoder, + each element has shape [bs, h, w]. + query_embed (Tensor): The query embedding for decoder, + with shape [num_query, c]. + mlvl_pos_embeds (list(Tensor)): The positional encoding + of feats from different level, has the shape + [bs, embed_dims, h, w]. + reg_branches (obj:`nn.ModuleList`): Regression heads for + feature maps from each decoder layer. Only would + be passed when + `with_box_refine` is True. Default to None. + cls_branches (obj:`nn.ModuleList`): Classification heads + for feature maps from each decoder layer. Only would + be passed when `as_two_stage` + is True. Default to None. + + + Returns: + tuple[Tensor]: results of decoder containing the following tensor. + + - inter_states: Outputs from decoder. If + return_intermediate_dec is True output has shape \ + (num_dec_layers, bs, num_query, embed_dims), else has \ + shape (1, bs, num_query, embed_dims). + - init_reference_out: The initial value of reference \ + points, has shape (bs, num_queries, 4). + - inter_references_out: The internal value of reference \ + points in decoder, has shape \ + (num_dec_layers, bs,num_query, embed_dims) + - enc_outputs_class: The classification score of \ + proposals generated from \ + encoder's feature maps, has shape \ + (batch, h*w, num_classes). \ + Only would be returned when `as_two_stage` is True, \ + otherwise None. + - enc_outputs_coord_unact: The regression results \ + generated from encoder's feature maps., has shape \ + (batch, h*w, 4). Only would \ + be returned when `as_two_stage` is True, \ + otherwise None. + """ + assert self.as_two_stage or query_embed is not None + + feat_flatten = [] + mask_flatten = [] + lvl_pos_embed_flatten = [] + spatial_shapes = [] + for lvl, (feat, mask, pos_embed) in enumerate( + zip(mlvl_feats, mlvl_masks, mlvl_pos_embeds)): + bs, c, h, w = feat.shape + spatial_shape = (h, w) + spatial_shapes.append(spatial_shape) + feat = feat.flatten(2).transpose(1, 2) + mask = mask.flatten(1) + pos_embed = pos_embed.flatten(2).transpose(1, 2) + lvl_pos_embed = pos_embed + self.level_embeds[lvl].view(1, 1, -1) + lvl_pos_embed_flatten.append(lvl_pos_embed) + feat_flatten.append(feat) + mask_flatten.append(mask) + feat_flatten = torch.cat(feat_flatten, 1) + mask_flatten = torch.cat(mask_flatten, 1) + lvl_pos_embed_flatten = torch.cat(lvl_pos_embed_flatten, 1) + spatial_shapes = torch.as_tensor( + spatial_shapes, dtype=torch.long, device=feat_flatten.device) + level_start_index = torch.cat((spatial_shapes.new_zeros( + (1,)), spatial_shapes.prod(1).cumsum(0)[:-1])) + valid_ratios = torch.stack( + [self.get_valid_ratio(m) for m in mlvl_masks], 1) + + reference_points = \ + self.get_reference_points(spatial_shapes, + valid_ratios, + device=feat.device) + + feat_flatten = feat_flatten.permute(1, 0, 2) # (H*W, bs, embed_dims) + lvl_pos_embed_flatten = lvl_pos_embed_flatten.permute( + 1, 0, 2) # (H*W, bs, embed_dims) + memory = self.encoder( + query=feat_flatten, + key=None, + value=None, + query_pos=lvl_pos_embed_flatten, + query_key_padding_mask=mask_flatten, + spatial_shapes=spatial_shapes, + reference_points=reference_points, + level_start_index=level_start_index, + valid_ratios=valid_ratios, + **kwargs) + + memory = memory.permute(1, 0, 2) + bs, _, c = memory.shape + if self.as_two_stage: + output_memory, output_proposals = \ + self.gen_encoder_output_proposals( + memory, mask_flatten, spatial_shapes) + enc_outputs_class = cls_branches[self.decoder.num_layers]( + output_memory) + enc_outputs_coord_unact = \ + reg_branches[ + self.decoder.num_layers](output_memory) + output_proposals + + topk = self.two_stage_num_proposals + topk_proposals = torch.topk( + enc_outputs_class[..., 0], topk, dim=1)[1] + topk_coords_unact = torch.gather( + enc_outputs_coord_unact, 1, + topk_proposals.unsqueeze(-1).repeat(1, 1, 4)) + topk_coords_unact = topk_coords_unact.detach() + reference_points = topk_coords_unact.sigmoid() + init_reference_out = reference_points + pos_trans_out = self.pos_trans_norm( + self.pos_trans(self.get_proposal_pos_embed(topk_coords_unact))) + query_pos, query = torch.split(pos_trans_out, c, dim=2) + else: + query_pos, query = torch.split(query_embed, c, dim=1) + query_pos = query_pos.unsqueeze(0).expand(bs, -1, -1) + query = query.unsqueeze(0).expand(bs, -1, -1) + reference_points = self.reference_points(query_pos).sigmoid() + init_reference_out = reference_points + + # decoder + query = query.permute(1, 0, 2) + memory = memory.permute(1, 0, 2) + query_pos = query_pos.permute(1, 0, 2) + inter_states, inter_references = self.decoder( + query=query, + key=None, + value=memory, + query_pos=query_pos, + key_padding_mask=mask_flatten, + reference_points=reference_points, + spatial_shapes=spatial_shapes, + level_start_index=level_start_index, + valid_ratios=valid_ratios, + reg_branches=reg_branches, + **kwargs) + + inter_references_out = inter_references + if self.as_two_stage: + return inter_states, init_reference_out, \ + inter_references_out, enc_outputs_class, \ + enc_outputs_coord_unact + return inter_states, init_reference_out, \ + inter_references_out, None, None + + +@TRANSFORMER.register_module() +class DynamicConv(BaseModule): + """Implements Dynamic Convolution. + + This module generate parameters for each sample and + use bmm to implement 1*1 convolution. Code is modified + from the `official github repo `_ . + + Args: + in_channels (int): The input feature channel. + Defaults to 256. + feat_channels (int): The inner feature channel. + Defaults to 64. + out_channels (int, optional): The output feature channel. + When not specified, it will be set to `in_channels` + by default + input_feat_shape (int): The shape of input feature. + Defaults to 7. + with_proj (bool): Project two-dimentional feature to + one-dimentional feature. Default to True. + act_cfg (dict): The activation config for DynamicConv. + norm_cfg (dict): Config dict for normalization layer. Default + layer normalization. + init_cfg (obj:`mmcv.ConfigDict`): The Config for initialization. + Default: None. + """ + def __init__(self, + in_channels=256, + feat_channels=64, + out_channels=None, + input_feat_shape=7, + with_proj=True, + act_cfg=dict(type='ReLU', inplace=True), + norm_cfg=dict(type='LN'), + init_cfg=None): + super(DynamicConv, self).__init__(init_cfg) + self.in_channels = in_channels + self.feat_channels = feat_channels + self.out_channels_raw = out_channels + self.input_feat_shape = input_feat_shape + self.with_proj = with_proj + self.act_cfg = act_cfg + self.norm_cfg = norm_cfg + self.out_channels = out_channels if out_channels else in_channels + + self.num_params_in = self.in_channels * self.feat_channels + self.num_params_out = self.out_channels * self.feat_channels + self.dynamic_layer = nn.Linear( + self.in_channels, self.num_params_in + self.num_params_out) + + self.norm_in = build_norm_layer(norm_cfg, self.feat_channels)[1] + self.norm_out = build_norm_layer(norm_cfg, self.out_channels)[1] + + self.activation = build_activation_layer(act_cfg) + + num_output = self.out_channels * input_feat_shape ** 2 + if self.with_proj: + self.fc_layer = nn.Linear(num_output, self.out_channels) + self.fc_norm = build_norm_layer(norm_cfg, self.out_channels)[1] + + def forward(self, param_feature, input_feature): + """Forward function for `DynamicConv`. + + Args: + param_feature (Tensor): The feature can be used + to generate the parameter, has shape + (num_all_proposals, in_channels). + input_feature (Tensor): Feature that + interact with parameters, has shape + (num_all_proposals, in_channels, H, W). + + Returns: + Tensor: The output feature has shape + (num_all_proposals, out_channels). + """ + input_feature = input_feature.flatten(2).permute(2, 0, 1) + + input_feature = input_feature.permute(1, 0, 2) + parameters = self.dynamic_layer(param_feature) + + param_in = parameters[:, :self.num_params_in].view( + -1, self.in_channels, self.feat_channels) + param_out = parameters[:, -self.num_params_out:].view( + -1, self.feat_channels, self.out_channels) + + # input_feature has shape (num_all_proposals, H*W, in_channels) + # param_in has shape (num_all_proposals, in_channels, feat_channels) + # feature has shape (num_all_proposals, H*W, feat_channels) + features = torch.bmm(input_feature, param_in) + features = self.norm_in(features) + features = self.activation(features) + + # param_out has shape (batch_size, feat_channels, out_channels) + features = torch.bmm(features, param_out) + features = self.norm_out(features) + features = self.activation(features) + + if self.with_proj: + features = features.flatten(1) + features = self.fc_layer(features) + features = self.fc_norm(features) + features = self.activation(features) + + return features diff --git a/segmentation/ops_dcnv3/functions/__init__.py b/segmentation/ops_dcnv3/functions/__init__.py new file mode 100644 index 0000000..4ff82f3 --- /dev/null +++ b/segmentation/ops_dcnv3/functions/__init__.py @@ -0,0 +1,7 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + +from .dcnv3_func import DCNv3Function, dcnv3_core_pytorch diff --git a/segmentation/ops_dcnv3/functions/dcnv3_func.py b/segmentation/ops_dcnv3/functions/dcnv3_func.py new file mode 100644 index 0000000..8c97fc8 --- /dev/null +++ b/segmentation/ops_dcnv3/functions/dcnv3_func.py @@ -0,0 +1,188 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + +from __future__ import absolute_import +from __future__ import print_function +from __future__ import division + +import torch +import torch.nn.functional as F +from torch.autograd import Function +from torch.autograd.function import once_differentiable +from torch.cuda.amp import custom_bwd, custom_fwd +import DCNv3 + + +class DCNv3Function(Function): + @staticmethod + @custom_fwd + def forward( + ctx, input, offset, mask, + kernel_h, kernel_w, stride_h, stride_w, + pad_h, pad_w, dilation_h, dilation_w, + group, group_channels, offset_scale, im2col_step): + ctx.kernel_h = kernel_h + ctx.kernel_w = kernel_w + ctx.stride_h = stride_h + ctx.stride_w = stride_w + ctx.pad_h = pad_h + ctx.pad_w = pad_w + ctx.dilation_h = dilation_h + ctx.dilation_w = dilation_w + ctx.group = group + ctx.group_channels = group_channels + ctx.offset_scale = offset_scale + ctx.im2col_step = im2col_step + output = DCNv3.dcnv3_forward( + input, offset, mask, kernel_h, + kernel_w, stride_h, stride_w, pad_h, + pad_w, dilation_h, dilation_w, group, + group_channels, offset_scale, ctx.im2col_step) + ctx.save_for_backward(input, offset, mask) + + return output + + @staticmethod + @once_differentiable + @custom_bwd + def backward(ctx, grad_output): + input, offset, mask = ctx.saved_tensors + grad_input, grad_offset, grad_mask = \ + DCNv3.dcnv3_backward( + input, offset, mask, ctx.kernel_h, + ctx.kernel_w, ctx.stride_h, ctx.stride_w, ctx.pad_h, + ctx.pad_w, ctx.dilation_h, ctx.dilation_w, ctx.group, + ctx.group_channels, ctx.offset_scale, grad_output.contiguous(), ctx.im2col_step) + + return grad_input, grad_offset, grad_mask, \ + None, None, None, None, None, None, None, None, None, None, None, None + + @staticmethod + def symbolic(g, input, offset, mask, kernel_h, kernel_w, stride_h, + stride_w, pad_h, pad_w, dilation_h, dilation_w, group, + group_channels, offset_scale, im2col_step): + """Symbolic function for mmdeploy::DCNv3. + + Returns: + DCNv3 op for onnx. + """ + return g.op( + 'mmdeploy::TRTDCNv3', + input, + offset, + mask, + kernel_h_i=int(kernel_h), + kernel_w_i=int(kernel_w), + stride_h_i=int(stride_h), + stride_w_i=int(stride_w), + pad_h_i=int(pad_h), + pad_w_i=int(pad_w), + dilation_h_i=int(dilation_h), + dilation_w_i=int(dilation_w), + group_i=int(group), + group_channels_i=int(group_channels), + offset_scale_f=float(offset_scale), + im2col_step_i=int(im2col_step), + ) + +def _get_reference_points(spatial_shapes, device, kernel_h, kernel_w, dilation_h, dilation_w, pad_h=0, pad_w=0, stride_h=1, stride_w=1): + _, H_, W_, _ = spatial_shapes + H_out = (H_ - (dilation_h * (kernel_h - 1) + 1)) // stride_h + 1 + W_out = (W_ - (dilation_w * (kernel_w - 1) + 1)) // stride_w + 1 + + ref_y, ref_x = torch.meshgrid( + torch.linspace( + # pad_h + 0.5, + # H_ - pad_h - 0.5, + (dilation_h * (kernel_h - 1)) // 2 + 0.5, + (dilation_h * (kernel_h - 1)) // 2 + 0.5 + (H_out - 1) * stride_h, + H_out, + dtype=torch.float32, + device=device), + torch.linspace( + # pad_w + 0.5, + # W_ - pad_w - 0.5, + (dilation_w * (kernel_w - 1)) // 2 + 0.5, + (dilation_w * (kernel_w - 1)) // 2 + 0.5 + (W_out - 1) * stride_w, + W_out, + dtype=torch.float32, + device=device)) + ref_y = ref_y.reshape(-1)[None] / H_ + ref_x = ref_x.reshape(-1)[None] / W_ + + ref = torch.stack((ref_x, ref_y), -1).reshape( + 1, H_out, W_out, 1, 2) + + return ref + + +def _generate_dilation_grids(spatial_shapes, kernel_h, kernel_w, dilation_h, dilation_w, group, device): + _, H_, W_, _ = spatial_shapes + points_list = [] + x, y = torch.meshgrid( + torch.linspace( + -((dilation_w * (kernel_w - 1)) // 2), + -((dilation_w * (kernel_w - 1)) // 2) + + (kernel_w - 1) * dilation_w, kernel_w, + dtype=torch.float32, + device=device), + torch.linspace( + -((dilation_h * (kernel_h - 1)) // 2), + -((dilation_h * (kernel_h - 1)) // 2) + + (kernel_h - 1) * dilation_h, kernel_h, + dtype=torch.float32, + device=device)) + + points_list.extend([x / W_, y / H_]) + grid = torch.stack(points_list, -1).reshape(-1, 1, 2).\ + repeat(1, group, 1).permute(1, 0, 2) + grid = grid.reshape(1, 1, 1, group * kernel_h * kernel_w, 2) + + return grid + + +def dcnv3_core_pytorch( + input, offset, mask, kernel_h, + kernel_w, stride_h, stride_w, pad_h, + pad_w, dilation_h, dilation_w, group, + group_channels, offset_scale): + # for debug and test only, + # need to use cuda version instead + input = F.pad( + input, + [0, 0, pad_h, pad_h, pad_w, pad_w]) + N_, H_in, W_in, _ = input.shape + _, H_out, W_out, _ = offset.shape + + ref = _get_reference_points( + input.shape, input.device, kernel_h, kernel_w, dilation_h, dilation_w, pad_h, pad_w, stride_h, stride_w) + grid = _generate_dilation_grids( + input.shape, kernel_h, kernel_w, dilation_h, dilation_w, group, input.device) + spatial_norm = torch.tensor([W_in, H_in]).reshape(1, 1, 1, 2).\ + repeat(1, 1, 1, group*kernel_h*kernel_w).to(input.device) + + sampling_locations = (ref + grid * offset_scale).repeat(N_, 1, 1, 1, 1).flatten(3, 4) + \ + offset * offset_scale / spatial_norm + + P_ = kernel_h * kernel_w + sampling_grids = 2 * sampling_locations - 1 + # N_, H_in, W_in, group*group_channels -> N_, H_in*W_in, group*group_channels -> N_, group*group_channels, H_in*W_in -> N_*group, group_channels, H_in, W_in + input_ = input.view(N_, H_in*W_in, group*group_channels).transpose(1, 2).\ + reshape(N_*group, group_channels, H_in, W_in) + # N_, H_out, W_out, group*P_*2 -> N_, H_out*W_out, group, P_, 2 -> N_, group, H_out*W_out, P_, 2 -> N_*group, H_out*W_out, P_, 2 + sampling_grid_ = sampling_grids.view(N_, H_out*W_out, group, P_, 2).transpose(1, 2).\ + flatten(0, 1) + # N_*group, group_channels, H_out*W_out, P_ + sampling_input_ = F.grid_sample( + input_, sampling_grid_, mode='bilinear', padding_mode='zeros', align_corners=False) + + # (N_, H_out, W_out, group*P_) -> N_, H_out*W_out, group, P_ -> (N_, group, H_out*W_out, P_) -> (N_*group, 1, H_out*W_out, P_) + mask = mask.view(N_, H_out*W_out, group, P_).transpose(1, 2).\ + reshape(N_*group, 1, H_out*W_out, P_) + output = (sampling_input_ * mask).sum(-1).view(N_, + group*group_channels, H_out*W_out) + + return output.transpose(1, 2).reshape(N_, H_out, W_out, -1).contiguous() diff --git a/segmentation/ops_dcnv3/make.sh b/segmentation/ops_dcnv3/make.sh new file mode 100755 index 0000000..0240533 --- /dev/null +++ b/segmentation/ops_dcnv3/make.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + +python setup.py build install diff --git a/segmentation/ops_dcnv3/modules/__init__.py b/segmentation/ops_dcnv3/modules/__init__.py new file mode 100644 index 0000000..4bba45c --- /dev/null +++ b/segmentation/ops_dcnv3/modules/__init__.py @@ -0,0 +1,7 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + +from .dcnv3 import DCNv3, DCNv3_pytorch \ No newline at end of file diff --git a/segmentation/ops_dcnv3/modules/dcnv3.py b/segmentation/ops_dcnv3/modules/dcnv3.py new file mode 100644 index 0000000..ca5bcf8 --- /dev/null +++ b/segmentation/ops_dcnv3/modules/dcnv3.py @@ -0,0 +1,345 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + +from __future__ import absolute_import +from __future__ import print_function +from __future__ import division + +import warnings +import torch +from torch import nn +import torch.nn.functional as F +from torch.nn.init import xavier_uniform_, constant_ +from ..functions import DCNv3Function, dcnv3_core_pytorch + + +class to_channels_first(nn.Module): + + def __init__(self): + super().__init__() + + def forward(self, x): + return x.permute(0, 3, 1, 2) + + +class to_channels_last(nn.Module): + + def __init__(self): + super().__init__() + + def forward(self, x): + return x.permute(0, 2, 3, 1) + + +def build_norm_layer(dim, + norm_layer, + in_format='channels_last', + out_format='channels_last', + eps=1e-6): + layers = [] + if norm_layer == 'BN': + if in_format == 'channels_last': + layers.append(to_channels_first()) + layers.append(nn.BatchNorm2d(dim)) + if out_format == 'channels_last': + layers.append(to_channels_last()) + elif norm_layer == 'LN': + if in_format == 'channels_first': + layers.append(to_channels_last()) + layers.append(nn.LayerNorm(dim, eps=eps)) + if out_format == 'channels_first': + layers.append(to_channels_first()) + else: + raise NotImplementedError( + f'build_norm_layer does not support {norm_layer}') + return nn.Sequential(*layers) + + +def build_act_layer(act_layer): + if act_layer == 'ReLU': + return nn.ReLU(inplace=True) + elif act_layer == 'SiLU': + return nn.SiLU(inplace=True) + elif act_layer == 'GELU': + return nn.GELU() + + raise NotImplementedError(f'build_act_layer does not support {act_layer}') + + +def _is_power_of_2(n): + if (not isinstance(n, int)) or (n < 0): + raise ValueError( + "invalid input for _is_power_of_2: {} (type: {})".format(n, type(n))) + + return (n & (n - 1) == 0) and n != 0 + + +class CenterFeatureScaleModule(nn.Module): + def forward(self, + query, + center_feature_scale_proj_weight, + center_feature_scale_proj_bias): + center_feature_scale = F.linear(query, + weight=center_feature_scale_proj_weight, + bias=center_feature_scale_proj_bias).sigmoid() + return center_feature_scale + + +class DCNv3_pytorch(nn.Module): + def __init__( + self, + channels=64, + kernel_size=3, + dw_kernel_size=None, + stride=1, + pad=1, + dilation=1, + group=4, + offset_scale=1.0, + act_layer='GELU', + norm_layer='LN', + center_feature_scale=False): + """ + DCNv3 Module + :param channels + :param kernel_size + :param stride + :param pad + :param dilation + :param group + :param offset_scale + :param act_layer + :param norm_layer + """ + super().__init__() + if channels % group != 0: + raise ValueError( + f'channels must be divisible by group, but got {channels} and {group}') + _d_per_group = channels // group + dw_kernel_size = dw_kernel_size if dw_kernel_size is not None else kernel_size + # you'd better set _d_per_group to a power of 2 which is more efficient in our CUDA implementation + if not _is_power_of_2(_d_per_group): + warnings.warn( + "You'd better set channels in DCNv3 to make the dimension of each attention head a power of 2 " + "which is more efficient in our CUDA implementation.") + + self.offset_scale = offset_scale + self.channels = channels + self.kernel_size = kernel_size + self.dw_kernel_size = dw_kernel_size + self.stride = stride + self.dilation = dilation + self.pad = pad + self.group = group + self.group_channels = channels // group + self.offset_scale = offset_scale + self.center_feature_scale = center_feature_scale + + self.dw_conv = nn.Sequential( + nn.Conv2d( + channels, + channels, + kernel_size=dw_kernel_size, + stride=1, + padding=(dw_kernel_size - 1) // 2, + groups=channels), + build_norm_layer( + channels, + norm_layer, + 'channels_first', + 'channels_last'), + build_act_layer(act_layer)) + self.offset = nn.Linear( + channels, + group * kernel_size * kernel_size * 2) + self.mask = nn.Linear( + channels, + group * kernel_size * kernel_size) + self.input_proj = nn.Linear(channels, channels) + self.output_proj = nn.Linear(channels, channels) + self._reset_parameters() + + if center_feature_scale: + self.center_feature_scale_proj_weight = nn.Parameter( + torch.zeros((group, channels), dtype=torch.float)) + self.center_feature_scale_proj_bias = nn.Parameter( + torch.tensor(0.0, dtype=torch.float).view((1,)).repeat(group, )) + self.center_feature_scale_module = CenterFeatureScaleModule() + + def _reset_parameters(self): + constant_(self.offset.weight.data, 0.) + constant_(self.offset.bias.data, 0.) + constant_(self.mask.weight.data, 0.) + constant_(self.mask.bias.data, 0.) + xavier_uniform_(self.input_proj.weight.data) + constant_(self.input_proj.bias.data, 0.) + xavier_uniform_(self.output_proj.weight.data) + constant_(self.output_proj.bias.data, 0.) + + def forward(self, input): + """ + :param query (N, H, W, C) + :return output (N, H, W, C) + """ + N, H, W, _ = input.shape + + x = self.input_proj(input) + x_proj = x + + x1 = input.permute(0, 3, 1, 2) + x1 = self.dw_conv(x1) + offset = self.offset(x1) + mask = self.mask(x1).reshape(N, H, W, self.group, -1) + mask = F.softmax(mask, -1).reshape(N, H, W, -1) + + x = dcnv3_core_pytorch( + x, offset, mask, + self.kernel_size, self.kernel_size, + self.stride, self.stride, + self.pad, self.pad, + self.dilation, self.dilation, + self.group, self.group_channels, + self.offset_scale) + if self.center_feature_scale: + center_feature_scale = self.center_feature_scale_module( + x1, self.center_feature_scale_proj_weight, self.center_feature_scale_proj_bias) + # N, H, W, groups -> N, H, W, groups, 1 -> N, H, W, groups, _d_per_group -> N, H, W, channels + center_feature_scale = center_feature_scale[..., None].repeat( + 1, 1, 1, 1, self.channels // self.group).flatten(-2) + x = x * (1 - center_feature_scale) + x_proj * center_feature_scale + x = self.output_proj(x) + + return x + + +class DCNv3(nn.Module): + def __init__( + self, + channels=64, + kernel_size=3, + dw_kernel_size=None, + stride=1, + pad=1, + dilation=1, + group=4, + offset_scale=1.0, + act_layer='GELU', + norm_layer='LN', + center_feature_scale=False): + """ + DCNv3 Module + :param channels + :param kernel_size + :param stride + :param pad + :param dilation + :param group + :param offset_scale + :param act_layer + :param norm_layer + """ + super().__init__() + if channels % group != 0: + raise ValueError( + f'channels must be divisible by group, but got {channels} and {group}') + _d_per_group = channels // group + dw_kernel_size = dw_kernel_size if dw_kernel_size is not None else kernel_size + # you'd better set _d_per_group to a power of 2 which is more efficient in our CUDA implementation + if not _is_power_of_2(_d_per_group): + warnings.warn( + "You'd better set channels in DCNv3 to make the dimension of each attention head a power of 2 " + "which is more efficient in our CUDA implementation.") + + self.offset_scale = offset_scale + self.channels = channels + self.kernel_size = kernel_size + self.dw_kernel_size = dw_kernel_size + self.stride = stride + self.dilation = dilation + self.pad = pad + self.group = group + self.group_channels = channels // group + self.offset_scale = offset_scale + self.center_feature_scale = center_feature_scale + + self.dw_conv = nn.Sequential( + nn.Conv2d( + channels, + channels, + kernel_size=dw_kernel_size, + stride=1, + padding=(dw_kernel_size - 1) // 2, + groups=channels), + build_norm_layer( + channels, + norm_layer, + 'channels_first', + 'channels_last'), + build_act_layer(act_layer)) + self.offset = nn.Linear( + channels, + group * kernel_size * kernel_size * 2) + self.mask = nn.Linear( + channels, + group * kernel_size * kernel_size) + self.input_proj = nn.Linear(channels, channels) + self.output_proj = nn.Linear(channels, channels) + self._reset_parameters() + + if center_feature_scale: + self.center_feature_scale_proj_weight = nn.Parameter( + torch.zeros((group, channels), dtype=torch.float)) + self.center_feature_scale_proj_bias = nn.Parameter( + torch.tensor(0.0, dtype=torch.float).view((1,)).repeat(group, )) + self.center_feature_scale_module = CenterFeatureScaleModule() + + def _reset_parameters(self): + constant_(self.offset.weight.data, 0.) + constant_(self.offset.bias.data, 0.) + constant_(self.mask.weight.data, 0.) + constant_(self.mask.bias.data, 0.) + xavier_uniform_(self.input_proj.weight.data) + constant_(self.input_proj.bias.data, 0.) + xavier_uniform_(self.output_proj.weight.data) + constant_(self.output_proj.bias.data, 0.) + + def forward(self, input): + """ + :param query (N, H, W, C) + :return output (N, H, W, C) + """ + N, H, W, _ = input.shape + + x = self.input_proj(input) + x_proj = x + dtype = x.dtype + + x1 = input.permute(0, 3, 1, 2) + x1 = self.dw_conv(x1) + offset = self.offset(x1) + mask = self.mask(x1).reshape(N, H, W, self.group, -1) + mask = F.softmax(mask, -1).reshape(N, H, W, -1).type(dtype) + + x = DCNv3Function.apply( + x, offset, mask, + self.kernel_size, self.kernel_size, + self.stride, self.stride, + self.pad, self.pad, + self.dilation, self.dilation, + self.group, self.group_channels, + self.offset_scale, + 256) + + if self.center_feature_scale: + center_feature_scale = self.center_feature_scale_module( + x1, self.center_feature_scale_proj_weight, self.center_feature_scale_proj_bias) + # N, H, W, groups -> N, H, W, groups, 1 -> N, H, W, groups, _d_per_group -> N, H, W, channels + center_feature_scale = center_feature_scale[..., None].repeat( + 1, 1, 1, 1, self.channels // self.group).flatten(-2) + x = x * (1 - center_feature_scale) + x_proj * center_feature_scale + x = self.output_proj(x) + + return x diff --git a/segmentation/ops_dcnv3/setup.py b/segmentation/ops_dcnv3/setup.py new file mode 100644 index 0000000..196b0c7 --- /dev/null +++ b/segmentation/ops_dcnv3/setup.py @@ -0,0 +1,75 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + +import os +import glob + +import torch + +from torch.utils.cpp_extension import CUDA_HOME +from torch.utils.cpp_extension import CppExtension +from torch.utils.cpp_extension import CUDAExtension + +from setuptools import find_packages +from setuptools import setup + +requirements = ["torch", "torchvision"] + + +def get_extensions(): + this_dir = os.path.dirname(os.path.abspath(__file__)) + extensions_dir = os.path.join(this_dir, "src") + + main_file = glob.glob(os.path.join(extensions_dir, "*.cpp")) + source_cpu = glob.glob(os.path.join(extensions_dir, "cpu", "*.cpp")) + source_cuda = glob.glob(os.path.join(extensions_dir, "cuda", "*.cu")) + + sources = main_file + source_cpu + extension = CppExtension + extra_compile_args = {"cxx": []} + define_macros = [] + + if torch.cuda.is_available() and CUDA_HOME is not None: + extension = CUDAExtension + sources += source_cuda + define_macros += [("WITH_CUDA", None)] + extra_compile_args["nvcc"] = [ + # "-DCUDA_HAS_FP16=1", + # "-D__CUDA_NO_HALF_OPERATORS__", + # "-D__CUDA_NO_HALF_CONVERSIONS__", + # "-D__CUDA_NO_HALF2_OPERATORS__", + ] + else: + raise NotImplementedError('Cuda is not availabel') + + sources = [os.path.join(extensions_dir, s) for s in sources] + include_dirs = [extensions_dir] + ext_modules = [ + extension( + "DCNv3", + sources, + include_dirs=include_dirs, + define_macros=define_macros, + extra_compile_args=extra_compile_args, + ) + ] + return ext_modules + + +setup( + name="DCNv3", + version="1.0", + author="InternImage", + url="https://github.com/OpenGVLab/InternImage", + description= + "PyTorch Wrapper for CUDA Functions of DCNv3", + packages=find_packages(exclude=( + "configs", + "tests", + )), + ext_modules=get_extensions(), + cmdclass={"build_ext": torch.utils.cpp_extension.BuildExtension}, +) diff --git a/segmentation/ops_dcnv3/src/cpu/dcnv3_cpu.cpp b/segmentation/ops_dcnv3/src/cpu/dcnv3_cpu.cpp new file mode 100644 index 0000000..a3bddc1 --- /dev/null +++ b/segmentation/ops_dcnv3/src/cpu/dcnv3_cpu.cpp @@ -0,0 +1,37 @@ +/*! +************************************************************************************************** +* InternImage +* Copyright (c) 2022 OpenGVLab +* Licensed under The MIT License [see LICENSE for details] +************************************************************************************************** +* Modified from +*https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +************************************************************************************************** +*/ + +#include + +#include +#include + +at::Tensor dcnv3_cpu_forward(const at::Tensor &input, const at::Tensor &offset, + const at::Tensor &mask, const int kernel_h, + const int kernel_w, const int stride_h, + const int stride_w, const int pad_h, + const int pad_w, const int dilation_h, + const int dilation_w, const int group, + const int group_channels, const float offset_scale, + const int im2col_step) { + AT_ERROR("Not implement on cpu"); +} + +std::vector +dcnv3_cpu_backward(const at::Tensor &input, const at::Tensor &offset, + const at::Tensor &mask, const int kernel_h, + const int kernel_w, const int stride_h, const int stride_w, + const int pad_h, const int pad_w, const int dilation_h, + const int dilation_w, const int group, + const int group_channels, const float offset_scale, + const at::Tensor &grad_output, const int im2col_step) { + AT_ERROR("Not implement on cpu"); +} diff --git a/segmentation/ops_dcnv3/src/cpu/dcnv3_cpu.h b/segmentation/ops_dcnv3/src/cpu/dcnv3_cpu.h new file mode 100644 index 0000000..d457bcb --- /dev/null +++ b/segmentation/ops_dcnv3/src/cpu/dcnv3_cpu.h @@ -0,0 +1,31 @@ +/*! +************************************************************************************************** +* InternImage +* Copyright (c) 2022 OpenGVLab +* Licensed under The MIT License [see LICENSE for details] +************************************************************************************************** +* Modified from +*https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +************************************************************************************************** +*/ + +#pragma once +#include + +at::Tensor dcnv3_cpu_forward(const at::Tensor &input, const at::Tensor &offset, + const at::Tensor &mask, const int kernel_h, + const int kernel_w, const int stride_h, + const int stride_w, const int pad_h, + const int pad_w, const int dilation_h, + const int dilation_w, const int group, + const int group_channels, const float offset_scale, + const int im2col_step); + +std::vector +dcnv3_cpu_backward(const at::Tensor &input, const at::Tensor &offset, + const at::Tensor &mask, const int kernel_h, + const int kernel_w, const int stride_h, const int stride_w, + const int pad_h, const int pad_w, const int dilation_h, + const int dilation_w, const int group, + const int group_channels, const float offset_scale, + const at::Tensor &grad_output, const int im2col_step); diff --git a/segmentation/ops_dcnv3/src/cuda/dcnv3_cuda.cu b/segmentation/ops_dcnv3/src/cuda/dcnv3_cuda.cu new file mode 100644 index 0000000..5284095 --- /dev/null +++ b/segmentation/ops_dcnv3/src/cuda/dcnv3_cuda.cu @@ -0,0 +1,174 @@ +/*! +************************************************************************************************** +* InternImage +* Copyright (c) 2022 OpenGVLab +* Licensed under The MIT License [see LICENSE for details] +************************************************************************************************** +* Modified from +*https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +************************************************************************************************** +*/ + +#include "cuda/dcnv3_im2col_cuda.cuh" +#include + +#include +#include +#include +#include +#include + +at::Tensor dcnv3_cuda_forward(const at::Tensor &input, const at::Tensor &offset, + const at::Tensor &mask, const int kernel_h, + const int kernel_w, const int stride_h, + const int stride_w, const int pad_h, + const int pad_w, const int dilation_h, + const int dilation_w, const int group, + const int group_channels, + const float offset_scale, const int im2col_step) { + AT_ASSERTM(input.is_contiguous(), "input tensor has to be contiguous"); + AT_ASSERTM(offset.is_contiguous(), "offset tensor has to be contiguous"); + AT_ASSERTM(mask.is_contiguous(), "mask tensor has to be contiguous"); + AT_ASSERTM(input.type().is_cuda(), "input must be a CUDA tensor"); + AT_ASSERTM(offset.type().is_cuda(), "offset must be a CUDA tensor"); + AT_ASSERTM(mask.type().is_cuda(), "mask must be a CUDA tensor"); + + const int batch = input.size(0); + const int height_in = input.size(1); + const int width_in = input.size(2); + const int channels = input.size(3); + const int height_out = + (height_in + 2 * pad_h - (dilation_h * (kernel_h - 1) + 1)) / stride_h + + 1; + const int width_out = + (width_in + 2 * pad_w - (dilation_w * (kernel_w - 1) + 1)) / stride_w + + 1; + const int im2col_step_ = std::min(batch, im2col_step); + + AT_ASSERTM(batch % im2col_step_ == 0, + "batch(%d) must divide im2col_step(%d)", batch, im2col_step_); + AT_ASSERTM( + channels == (group * group_channels), + "Input channels and group times group channels wont match: (%d vs %d).", + channels, group * group_channels); + + auto output = + at::zeros({batch, height_out, width_out, group * group_channels}, + input.options()); + + const int batch_n = im2col_step_; + auto output_n = output.view({batch / batch_n, batch_n, height_out, + width_out, group * group_channels}); + auto per_input_size = height_in * width_in * group * group_channels; + auto per_offset_size = + height_out * width_out * group * kernel_h * kernel_w * 2; + auto per_mask_size = height_out * width_out * group * kernel_h * kernel_w; + for (int n = 0; n < batch / im2col_step_; ++n) { + auto columns = output_n.select(0, n); + // AT_DISPATCH_FLOATING_TYPES( + AT_DISPATCH_FLOATING_TYPES_AND_HALF( + input.type(), "ms_deform_attn_forward_cuda", ([&] { + dcnv3_im2col_cuda( + at::cuda::getCurrentCUDAStream(), + input.data() + n * im2col_step_ * per_input_size, + offset.data() + + n * im2col_step_ * per_offset_size, + mask.data() + n * im2col_step_ * per_mask_size, + columns.data(), kernel_h, kernel_w, stride_h, + stride_w, pad_h, pad_w, dilation_h, dilation_w, group, + group_channels, batch_n, height_in, width_in, height_out, + width_out, offset_scale); + })); + } + + return output; +} + +std::vector +dcnv3_cuda_backward(const at::Tensor &input, const at::Tensor &offset, + const at::Tensor &mask, const int kernel_h, + const int kernel_w, const int stride_h, const int stride_w, + const int pad_h, const int pad_w, const int dilation_h, + const int dilation_w, const int group, + const int group_channels, const float offset_scale, + const at::Tensor &grad_output, const int im2col_step) { + + AT_ASSERTM(input.is_contiguous(), "input tensor has to be contiguous"); + AT_ASSERTM(offset.is_contiguous(), "offset tensor has to be contiguous"); + AT_ASSERTM(mask.is_contiguous(), "mask tensor has to be contiguous"); + AT_ASSERTM(grad_output.is_contiguous(), + "grad_output tensor has to be contiguous"); + AT_ASSERTM(input.type().is_cuda(), "input must be a CUDA tensor"); + AT_ASSERTM(offset.type().is_cuda(), "offset must be a CUDA tensor"); + AT_ASSERTM(mask.type().is_cuda(), "mask must be a CUDA tensor"); + AT_ASSERTM(grad_output.type().is_cuda(), + "grad_output must be a CUDA tensor"); + + const int batch = input.size(0); + const int height_in = input.size(1); + const int width_in = input.size(2); + const int channels = input.size(3); + const int height_out = + (height_in + 2 * pad_h - (dilation_h * (kernel_h - 1) + 1)) / stride_h + + 1; + const int width_out = + (width_in + 2 * pad_w - (dilation_w * (kernel_w - 1) + 1)) / stride_w + + 1; + const int im2col_step_ = std::min(batch, im2col_step); + + AT_ASSERTM(batch % im2col_step_ == 0, + "batch(%d) must divide im2col_step(%d)", batch, im2col_step_); + AT_ASSERTM( + channels == (group * group_channels), + "Input channels and group times group channels wont match: (%d vs %d).", + channels, group * group_channels); + + auto dtype = input.dtype(); + if (dtype == at::kHalf) { + dtype = at::kFloat; + } + + auto grad_input = at::zeros_like(input, dtype); + auto grad_offset = at::zeros_like(offset, dtype); + auto grad_mask = at::zeros_like(mask, dtype); + + const int batch_n = im2col_step_; + auto per_input_size = height_in * width_in * group * group_channels; + auto per_offset_size = + height_out * width_out * group * kernel_h * kernel_w * 2; + auto per_mask_size = height_out * width_out * group * kernel_h * kernel_w; + auto grad_output_n = + grad_output.view({batch / im2col_step_, batch_n, height_out * width_out, + group, group_channels}); + + for (int n = 0; n < batch / im2col_step_; ++n) { + auto grad_output_g = grad_output_n.select(0, n); + // AT_DISPATCH_FLOATING_TYPES( + AT_DISPATCH_FLOATING_TYPES_AND_HALF( + input.type(), "ms_deform_attn_backward_cuda", ([&] { + dcnv3_col2im_cuda( + at::cuda::getCurrentCUDAStream(), + grad_output_g.data(), + input.data() + n * im2col_step_ * per_input_size, + offset.data() + + n * im2col_step_ * per_offset_size, + mask.data() + n * im2col_step_ * per_mask_size, + kernel_h, kernel_w, stride_h, stride_w, pad_h, pad_w, + dilation_h, dilation_w, group, group_channels, batch_n, + height_in, width_in, height_out, width_out, offset_scale, + grad_input.data() + + n * im2col_step_ * per_input_size, + grad_offset.data() + + n * im2col_step_ * per_offset_size, + grad_mask.data() + + n * im2col_step_ * per_mask_size); + })); + } + + if (input.dtype() == torch::kHalf) { + return {grad_input.to(torch::kHalf), grad_offset.to(torch::kHalf), + grad_mask.to(torch::kHalf)}; + } else { + return {grad_input, grad_offset, grad_mask}; + } +} \ No newline at end of file diff --git a/segmentation/ops_dcnv3/src/cuda/dcnv3_cuda.h b/segmentation/ops_dcnv3/src/cuda/dcnv3_cuda.h new file mode 100644 index 0000000..069f282 --- /dev/null +++ b/segmentation/ops_dcnv3/src/cuda/dcnv3_cuda.h @@ -0,0 +1,31 @@ +/*! +************************************************************************************************** +* InternImage +* Copyright (c) 2022 OpenGVLab +* Licensed under The MIT License [see LICENSE for details] +************************************************************************************************** +* Modified from +*https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +************************************************************************************************** +*/ + +#pragma once +#include + +at::Tensor dcnv3_cuda_forward(const at::Tensor &input, const at::Tensor &offset, + const at::Tensor &mask, const int kernel_h, + const int kernel_w, const int stride_h, + const int stride_w, const int pad_h, + const int pad_w, const int dilation_h, + const int dilation_w, const int group, + const int group_channels, + const float offset_scale, const int im2col_step); + +std::vector +dcnv3_cuda_backward(const at::Tensor &input, const at::Tensor &offset, + const at::Tensor &mask, const int kernel_h, + const int kernel_w, const int stride_h, const int stride_w, + const int pad_h, const int pad_w, const int dilation_h, + const int dilation_w, const int group, + const int group_channels, const float offset_scale, + const at::Tensor &grad_output, const int im2col_step); diff --git a/segmentation/ops_dcnv3/src/cuda/dcnv3_im2col_cuda.cuh b/segmentation/ops_dcnv3/src/cuda/dcnv3_im2col_cuda.cuh new file mode 100644 index 0000000..b551ba3 --- /dev/null +++ b/segmentation/ops_dcnv3/src/cuda/dcnv3_im2col_cuda.cuh @@ -0,0 +1,1045 @@ +/*! +************************************************************************************************** +* InternImage +* Copyright (c) 2022 OpenGVLab +* Licensed under The MIT License [see LICENSE for details] +************************************************************************************************** +* Modified from +*https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +************************************************************************************************** +*/ + +#include +#include +#include + +#include +#include +#include +#include + +#define CUDA_KERNEL_LOOP(i, n) \ + for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < (n); \ + i += blockDim.x * gridDim.x) + +const int CUDA_NUM_THREADS = 256; +inline int GET_BLOCKS(const int N, const int num_threads) { + return (N + num_threads - 1) / num_threads; +} + +#define opmath_t at::opmath_type + +template +__device__ opmath_t dcnv3_im2col_bilinear(const scalar_t *&bottom_data, + const int &height, const int &width, + const int &group, + const int &group_channels, + const opmath_t &h, const opmath_t &w, + const int &g, const int &c) { + const int h_low = floor(h); + const int w_low = floor(w); + const int h_high = h_low + 1; + const int w_high = w_low + 1; + + const opmath_t lh = h - h_low; + const opmath_t lw = w - w_low; + const opmath_t hh = 1 - lh, hw = 1 - lw; + + const int w_stride = group * group_channels; + const int h_stride = width * w_stride; + const int h_low_ptr_offset = h_low * h_stride; + const int h_high_ptr_offset = h_low_ptr_offset + h_stride; + const int w_low_ptr_offset = w_low * w_stride; + const int w_high_ptr_offset = w_low_ptr_offset + w_stride; + const int base_ptr = g * group_channels + c; + + opmath_t v1 = 0; + if (h_low >= 0 && w_low >= 0) { + const int ptr1 = h_low_ptr_offset + w_low_ptr_offset + base_ptr; + v1 = bottom_data[ptr1]; + } + opmath_t v2 = 0; + if (h_low >= 0 && w_high <= width - 1) { + const int ptr2 = h_low_ptr_offset + w_high_ptr_offset + base_ptr; + v2 = bottom_data[ptr2]; + } + opmath_t v3 = 0; + if (h_high <= height - 1 && w_low >= 0) { + const int ptr3 = h_high_ptr_offset + w_low_ptr_offset + base_ptr; + v3 = bottom_data[ptr3]; + } + opmath_t v4 = 0; + if (h_high <= height - 1 && w_high <= width - 1) { + const int ptr4 = h_high_ptr_offset + w_high_ptr_offset + base_ptr; + v4 = bottom_data[ptr4]; + } + const opmath_t w1 = hh * hw, w2 = hh * lw, w3 = lh * hw, w4 = lh * lw; + + const opmath_t val = (w1 * v1 + w2 * v2 + w3 * v3 + w4 * v4); + return val; +} + +template +__device__ void dcnv3_col2im_bilinear( + const scalar_t *&bottom_data, const int &height, const int &width, + const int &nheads, const int &group_channels, const opmath_t &h, + const opmath_t &w, const int &m, const int &c, const opmath_t offset_scale, + const opmath_t &top_grad, const opmath_t &mask, opmath_t *&grad_im, + opmath_t *grad_offset, opmath_t *grad_mask) { + const int h_low = floor(h); + const int w_low = floor(w); + const int h_high = h_low + 1; + const int w_high = w_low + 1; + + const opmath_t lh = h - h_low; + const opmath_t lw = w - w_low; + const opmath_t hh = 1 - lh, hw = 1 - lw; + + const int w_stride = nheads * group_channels; + const int h_stride = width * w_stride; + const int h_low_ptr_offset = h_low * h_stride; + const int h_high_ptr_offset = h_low_ptr_offset + h_stride; + const int w_low_ptr_offset = w_low * w_stride; + const int w_high_ptr_offset = w_low_ptr_offset + w_stride; + const int base_ptr = m * group_channels + c; + + const opmath_t w1 = hh * hw, w2 = hh * lw, w3 = lh * hw, w4 = lh * lw; + const opmath_t top_grad_im = top_grad * mask; + opmath_t grad_h_weight = 0, grad_w_weight = 0; + + opmath_t v1 = 0; + if (h_low >= 0 && w_low >= 0) { + const int ptr1 = h_low_ptr_offset + w_low_ptr_offset + base_ptr; + v1 = bottom_data[ptr1]; + grad_h_weight -= hw * v1; + grad_w_weight -= hh * v1; + atomicAdd(grad_im + ptr1, w1 * top_grad_im); + } + opmath_t v2 = 0; + if (h_low >= 0 && w_high <= width - 1) { + const int ptr2 = h_low_ptr_offset + w_high_ptr_offset + base_ptr; + v2 = bottom_data[ptr2]; + grad_h_weight -= lw * v2; + grad_w_weight += hh * v2; + atomicAdd(grad_im + ptr2, w2 * top_grad_im); + } + opmath_t v3 = 0; + if (h_high <= height - 1 && w_low >= 0) { + const int ptr3 = h_high_ptr_offset + w_low_ptr_offset + base_ptr; + v3 = bottom_data[ptr3]; + grad_h_weight += hw * v3; + grad_w_weight -= lh * v3; + atomicAdd(grad_im + ptr3, w3 * top_grad_im); + } + opmath_t v4 = 0; + if (h_high <= height - 1 && w_high <= width - 1) { + const int ptr4 = h_high_ptr_offset + w_high_ptr_offset + base_ptr; + v4 = bottom_data[ptr4]; + grad_h_weight += lw * v4; + grad_w_weight += lh * v4; + atomicAdd(grad_im + ptr4, w4 * top_grad_im); + } + + const opmath_t val = (w1 * v1 + w2 * v2 + w3 * v3 + w4 * v4); + *grad_mask = top_grad * val; + *grad_offset = offset_scale * grad_w_weight * top_grad_im; + *(grad_offset + 1) = offset_scale * grad_h_weight * top_grad_im; +} + +template +__device__ void dcnv3_col2im_bilinear_gm( + const scalar_t *&bottom_data, const int &height, const int &width, + const int &nheads, const int &group_channels, const opmath_t &h, + const opmath_t &w, const int &m, const int &c, const opmath_t offset_scale, + const opmath_t &top_grad, const opmath_t &mask, opmath_t *&grad_im, + opmath_t *grad_offset, opmath_t *grad_mask) { + const int h_low = floor(h); + const int w_low = floor(w); + const int h_high = h_low + 1; + const int w_high = w_low + 1; + + const opmath_t lh = h - h_low; + const opmath_t lw = w - w_low; + const opmath_t hh = 1 - lh, hw = 1 - lw; + + const int w_stride = nheads * group_channels; + const int h_stride = width * w_stride; + const int h_low_ptr_offset = h_low * h_stride; + const int h_high_ptr_offset = h_low_ptr_offset + h_stride; + const int w_low_ptr_offset = w_low * w_stride; + const int w_high_ptr_offset = w_low_ptr_offset + w_stride; + const int base_ptr = m * group_channels + c; + + const opmath_t w1 = hh * hw, w2 = hh * lw, w3 = lh * hw, w4 = lh * lw; + const opmath_t top_grad_im = top_grad * mask; + opmath_t grad_h_weight = 0, grad_w_weight = 0; + + opmath_t v1 = 0; + if (h_low >= 0 && w_low >= 0) { + const int ptr1 = h_low_ptr_offset + w_low_ptr_offset + base_ptr; + v1 = bottom_data[ptr1]; + grad_h_weight -= hw * v1; + grad_w_weight -= hh * v1; + atomicAdd(grad_im + ptr1, w1 * top_grad_im); + } + opmath_t v2 = 0; + if (h_low >= 0 && w_high <= width - 1) { + const int ptr2 = h_low_ptr_offset + w_high_ptr_offset + base_ptr; + v2 = bottom_data[ptr2]; + grad_h_weight -= lw * v2; + grad_w_weight += hh * v2; + atomicAdd(grad_im + ptr2, w2 * top_grad_im); + } + opmath_t v3 = 0; + if (h_high <= height - 1 && w_low >= 0) { + const int ptr3 = h_high_ptr_offset + w_low_ptr_offset + base_ptr; + v3 = bottom_data[ptr3]; + grad_h_weight += hw * v3; + grad_w_weight -= lh * v3; + atomicAdd(grad_im + ptr3, w3 * top_grad_im); + } + opmath_t v4 = 0; + if (h_high <= height - 1 && w_high <= width - 1) { + const int ptr4 = h_high_ptr_offset + w_high_ptr_offset + base_ptr; + v4 = bottom_data[ptr4]; + grad_h_weight += lw * v4; + grad_w_weight += lh * v4; + atomicAdd(grad_im + ptr4, w4 * top_grad_im); + } + + const opmath_t val = (w1 * v1 + w2 * v2 + w3 * v3 + w4 * v4); + atomicAdd(grad_mask, top_grad * val); + atomicAdd(grad_offset, offset_scale * grad_w_weight * top_grad_im); + atomicAdd(grad_offset + 1, offset_scale * grad_h_weight * top_grad_im); +} + +template +__global__ void dcnv3_im2col_gpu_kernel( + const int num_kernels, const scalar_t *data_im, const scalar_t *data_offset, + const scalar_t *data_mask, scalar_t *data_col, const int kernel_h, + const int kernel_w, const int stride_h, const int stride_w, const int pad_h, + const int pad_w, const int dilation_h, const int dilation_w, + const int group, const int group_channels, const int height_in, + const int width_in, const int height_out, const int width_out, + const opmath_t offset_scale) { + CUDA_KERNEL_LOOP(index, num_kernels) { + int _temp = index; + const int c_col = _temp % group_channels; + _temp /= group_channels; + const int sampling_index = _temp; + const int g_col = _temp % group; + _temp /= group; + const int p0_w = ((dilation_w * (kernel_w - 1)) >> 1) - pad_w + + (_temp % width_out) * stride_w; + _temp /= width_out; + const int p0_h = ((dilation_h * (kernel_h - 1)) >> 1) - pad_h + + (_temp % height_out) * stride_h; + _temp /= height_out; + const int b_col = _temp; + + const int input_size = height_in * width_in; + scalar_t *data_col_ptr = data_col + index; + const int kernel_size = kernel_h * kernel_w; + int data_weight_ptr = sampling_index * kernel_size; + int data_loc_w_ptr = data_weight_ptr << 1; + const int qid_stride = group * group_channels; + opmath_t col = 0; + const scalar_t *data_im_ptr = data_im + b_col * input_size * qid_stride; + // top-left + const opmath_t p0_w_ = + p0_w - ((dilation_w * (kernel_w - 1)) >> 1) * offset_scale; + const opmath_t p0_h_ = + p0_h - ((dilation_h * (kernel_h - 1)) >> 1) * offset_scale; + for (int i = 0; i < kernel_w; ++i) { + for (int j = 0; j < kernel_h; ++j) { + const opmath_t offset_w = data_offset[data_loc_w_ptr]; + const opmath_t offset_h = data_offset[data_loc_w_ptr + 1]; + const opmath_t loc_w = + p0_w_ + (i * dilation_w + offset_w) * offset_scale; + const opmath_t loc_h = + p0_h_ + (j * dilation_h + offset_h) * offset_scale; + const opmath_t weight = data_mask[data_weight_ptr]; + if (loc_h > -1 && loc_w > -1 && loc_h < height_in && + loc_w < width_in) { + col += dcnv3_im2col_bilinear( + data_im_ptr, height_in, width_in, group, + group_channels, loc_h, loc_w, g_col, c_col) * + weight; + } + data_weight_ptr += 1; + data_loc_w_ptr += 2; + } + } + *data_col_ptr = col; + } +} + +// debug +template +__global__ void dcnv3_col2im_gpu_kernel_shm_blocksize_aware_reduce_v1( + const int num_kernels, const scalar_t *grad_col, const scalar_t *data_im, + const scalar_t *data_offset, const scalar_t *data_mask, const int kernel_h, + const int kernel_w, const int stride_h, const int stride_w, const int pad_h, + const int pad_w, const int dilation_h, const int dilation_w, + const int group, const int group_channels, const int height_in, + const int width_in, const int height_out, const int width_out, + const opmath_t offset_scale, opmath_t *grad_im, opmath_t *grad_offset, + opmath_t *grad_mask) { + CUDA_KERNEL_LOOP(index, num_kernels) { + __shared__ opmath_t cache_grad_offset[blockSize * 2]; + __shared__ opmath_t cache_grad_mask[blockSize]; + unsigned int tid = threadIdx.x; + int _temp = index; + const int c_col = _temp % group_channels; + _temp /= group_channels; + const int sampling_index = _temp; + const int g_col = _temp % group; + _temp /= group; + const int p0_w = ((dilation_w * (kernel_w - 1)) >> 1) - pad_w + + (_temp % width_out) * stride_w; + _temp /= width_out; + const int p0_h = ((dilation_h * (kernel_h - 1)) >> 1) - pad_h + + (_temp % height_out) * stride_h; + _temp /= height_out; + const int b_col = _temp; + + const opmath_t top_grad = grad_col[index]; + const int input_size = height_in * width_in; + const int kernel_size = kernel_h * kernel_w; + int data_weight_ptr = sampling_index * kernel_size; + int data_loc_w_ptr = data_weight_ptr << 1; + const int grad_sampling_ptr = data_weight_ptr; + grad_offset += grad_sampling_ptr << 1; + grad_mask += grad_sampling_ptr; + const int qid_stride = group * group_channels; + const int im_ptr_offset = b_col * input_size * qid_stride; + const scalar_t *data_im_ptr = data_im + im_ptr_offset; + opmath_t *grad_im_ptr = grad_im + im_ptr_offset; + const opmath_t p0_w_ = + p0_w - ((dilation_w * (kernel_w - 1)) >> 1) * offset_scale; + const opmath_t p0_h_ = + p0_h - ((dilation_h * (kernel_h - 1)) >> 1) * offset_scale; + for (int i = 0; i < kernel_w; ++i) { + for (int j = 0; j < kernel_h; ++j) { + const opmath_t offset_w = data_offset[data_loc_w_ptr]; + const opmath_t offset_h = data_offset[data_loc_w_ptr + 1]; + const opmath_t loc_w = + p0_w_ + (i * dilation_w + offset_w) * offset_scale; + const opmath_t loc_h = + p0_h_ + (j * dilation_h + offset_h) * offset_scale; + const opmath_t weight = data_mask[data_weight_ptr]; + *(cache_grad_offset + (threadIdx.x << 1)) = 0; + *(cache_grad_offset + ((threadIdx.x << 1) + 1)) = 0; + *(cache_grad_mask + threadIdx.x) = 0; + if (loc_h > -1 && loc_w > -1 && loc_h < height_in && + loc_w < width_in) { + dcnv3_col2im_bilinear( + data_im_ptr, height_in, width_in, group, group_channels, + loc_h, loc_w, g_col, c_col, offset_scale, top_grad, + weight, grad_im_ptr, + cache_grad_offset + (threadIdx.x << 1), + cache_grad_mask + threadIdx.x); + } + + __syncthreads(); + if (tid == 0) { + opmath_t _grad_w = cache_grad_offset[0], + _grad_h = cache_grad_offset[1], + _grad_a = cache_grad_mask[0]; + int sid = 2; + for (unsigned int tid = 1; tid < blockSize; ++tid) { + _grad_w += cache_grad_offset[sid]; + _grad_h += cache_grad_offset[sid + 1]; + _grad_a += cache_grad_mask[tid]; + sid += 2; + } + + *grad_offset = _grad_w; + *(grad_offset + 1) = _grad_h; + *grad_mask = _grad_a; + } + __syncthreads(); + + data_weight_ptr += 1; + data_loc_w_ptr += 2; + grad_mask += 1; + grad_offset += 2; + } + } + } +} + +template +__global__ void dcnv3_col2im_gpu_kernel_shm_blocksize_aware_reduce_v2( + const int num_kernels, const scalar_t *grad_col, const scalar_t *data_im, + const scalar_t *data_offset, const scalar_t *data_mask, const int kernel_h, + const int kernel_w, const int stride_h, const int stride_w, const int pad_h, + const int pad_w, const int dilation_h, const int dilation_w, + const int group, const int group_channels, const int height_in, + const int width_in, const int height_out, const int width_out, + const opmath_t offset_scale, opmath_t *grad_im, opmath_t *grad_offset, + opmath_t *grad_mask) { + CUDA_KERNEL_LOOP(index, num_kernels) { + __shared__ opmath_t cache_grad_offset[blockSize * 2]; + __shared__ opmath_t cache_grad_mask[blockSize]; + unsigned int tid = threadIdx.x; + int _temp = index; + const int c_col = _temp % group_channels; + _temp /= group_channels; + const int sampling_index = _temp; + const int g_col = _temp % group; + _temp /= group; + const int p0_w = ((dilation_w * (kernel_w - 1)) >> 1) - pad_w + + (_temp % width_out) * stride_w; + _temp /= width_out; + const int p0_h = ((dilation_h * (kernel_h - 1)) >> 1) - pad_h + + (_temp % height_out) * stride_h; + _temp /= height_out; + const int b_col = _temp; + + const opmath_t top_grad = grad_col[index]; + const int input_size = height_in * width_in; + const int kernel_size = kernel_h * kernel_w; + int data_weight_ptr = sampling_index * kernel_size; + int data_loc_w_ptr = data_weight_ptr << 1; + const int grad_sampling_ptr = data_weight_ptr; + grad_offset += grad_sampling_ptr << 1; + grad_mask += grad_sampling_ptr; + const int qid_stride = group * group_channels; + const int im_ptr_offset = b_col * input_size * qid_stride; + const scalar_t *data_im_ptr = data_im + im_ptr_offset; + opmath_t *grad_im_ptr = grad_im + im_ptr_offset; + const opmath_t p0_w_ = + p0_w - ((dilation_w * (kernel_w - 1)) >> 1) * offset_scale; + const opmath_t p0_h_ = + p0_h - ((dilation_h * (kernel_h - 1)) >> 1) * offset_scale; + for (int i = 0; i < kernel_w; ++i) { + for (int j = 0; j < kernel_h; ++j) { + const opmath_t offset_w = data_offset[data_loc_w_ptr]; + const opmath_t offset_h = data_offset[data_loc_w_ptr + 1]; + const opmath_t loc_w = + p0_w_ + (i * dilation_w + offset_w) * offset_scale; + const opmath_t loc_h = + p0_h_ + (j * dilation_h + offset_h) * offset_scale; + const opmath_t weight = data_mask[data_weight_ptr]; + *(cache_grad_offset + (threadIdx.x << 1)) = 0; + *(cache_grad_offset + ((threadIdx.x << 1) + 1)) = 0; + *(cache_grad_mask + threadIdx.x) = 0; + if (loc_h > -1 && loc_w > -1 && loc_h < height_in && + loc_w < width_in) { + dcnv3_col2im_bilinear( + data_im_ptr, height_in, width_in, group, group_channels, + loc_h, loc_w, g_col, c_col, offset_scale, top_grad, + weight, grad_im_ptr, + cache_grad_offset + (threadIdx.x << 1), + cache_grad_mask + threadIdx.x); + } + + __syncthreads(); + + for (unsigned int s = blockSize / 2; s > 0; s >>= 1) { + if (tid < s) { + const unsigned int xid1 = tid << 1; + const unsigned int xid2 = (tid + s) << 1; + cache_grad_mask[tid] += cache_grad_mask[tid + s]; + cache_grad_offset[xid1] += cache_grad_offset[xid2]; + cache_grad_offset[xid1 + 1] += + cache_grad_offset[xid2 + 1]; + } + __syncthreads(); + } + + if (tid == 0) { + *grad_offset = cache_grad_offset[0]; + *(grad_offset + 1) = cache_grad_offset[1]; + *grad_mask = cache_grad_mask[0]; + } + __syncthreads(); + + data_weight_ptr += 1; + data_loc_w_ptr += 2; + grad_mask += 1; + grad_offset += 2; + } + } + } +} + +template +__global__ void dcnv3_col2im_gpu_kernel_shm_reduce_v1( + const int num_kernels, const scalar_t *grad_col, const scalar_t *data_im, + const scalar_t *data_offset, const scalar_t *data_mask, const int kernel_h, + const int kernel_w, const int stride_h, const int stride_w, const int pad_h, + const int pad_w, const int dilation_h, const int dilation_w, + const int group, const int group_channels, const int height_in, + const int width_in, const int height_out, const int width_out, + const opmath_t offset_scale, opmath_t *grad_im, opmath_t *grad_offset, + opmath_t *grad_mask) { + CUDA_KERNEL_LOOP(index, num_kernels) { + extern __shared__ int _s[]; + opmath_t *cache_grad_offset = (opmath_t *)_s; + opmath_t *cache_grad_mask = cache_grad_offset + 2 * blockDim.x; + unsigned int tid = threadIdx.x; + int _temp = index; + const int c_col = _temp % group_channels; + _temp /= group_channels; + const int sampling_index = _temp; + const int g_col = _temp % group; + _temp /= group; + const int p0_w = ((dilation_w * (kernel_w - 1)) >> 1) - pad_w + + (_temp % width_out) * stride_w; + _temp /= width_out; + const int p0_h = ((dilation_h * (kernel_h - 1)) >> 1) - pad_h + + (_temp % height_out) * stride_h; + _temp /= height_out; + const int b_col = _temp; + + const opmath_t top_grad = grad_col[index]; + const int input_size = height_in * width_in; + const int kernel_size = kernel_h * kernel_w; + int data_weight_ptr = sampling_index * kernel_size; + int data_loc_w_ptr = data_weight_ptr << 1; + const int grad_sampling_ptr = data_weight_ptr; + grad_offset += grad_sampling_ptr << 1; + grad_mask += grad_sampling_ptr; + const int qid_stride = group * group_channels; + const int im_ptr_offset = b_col * input_size * qid_stride; + const scalar_t *data_im_ptr = data_im + im_ptr_offset; + opmath_t *grad_im_ptr = grad_im + im_ptr_offset; + const opmath_t p0_w_ = + p0_w - ((dilation_w * (kernel_w - 1)) >> 1) * offset_scale; + const opmath_t p0_h_ = + p0_h - ((dilation_h * (kernel_h - 1)) >> 1) * offset_scale; + for (int i = 0; i < kernel_w; ++i) { + for (int j = 0; j < kernel_h; ++j) { + const opmath_t offset_w = data_offset[data_loc_w_ptr]; + const opmath_t offset_h = data_offset[data_loc_w_ptr + 1]; + const opmath_t loc_w = + p0_w_ + (i * dilation_w + offset_w) * offset_scale; + const opmath_t loc_h = + p0_h_ + (j * dilation_h + offset_h) * offset_scale; + const opmath_t weight = data_mask[data_weight_ptr]; + *(cache_grad_offset + (threadIdx.x << 1)) = 0; + *(cache_grad_offset + ((threadIdx.x << 1) + 1)) = 0; + *(cache_grad_mask + threadIdx.x) = 0; + if (loc_h > -1 && loc_w > -1 && loc_h < height_in && + loc_w < width_in) { + dcnv3_col2im_bilinear( + data_im_ptr, height_in, width_in, group, group_channels, + loc_h, loc_w, g_col, c_col, offset_scale, top_grad, + weight, grad_im_ptr, + cache_grad_offset + (threadIdx.x << 1), + cache_grad_mask + threadIdx.x); + } + + __syncthreads(); + if (tid == 0) { + opmath_t _grad_w = cache_grad_offset[0], + _grad_h = cache_grad_offset[1], + _grad_a = cache_grad_mask[0]; + int sid = 2; + for (unsigned int tid = 1; tid < blockDim.x; ++tid) { + _grad_w += cache_grad_offset[sid]; + _grad_h += cache_grad_offset[sid + 1]; + _grad_a += cache_grad_mask[tid]; + sid += 2; + } + + *grad_offset = _grad_w; + *(grad_offset + 1) = _grad_h; + *grad_mask = _grad_a; + } + __syncthreads(); + + data_weight_ptr += 1; + data_loc_w_ptr += 2; + grad_mask += 1; + grad_offset += 2; + } + } + } +} + +template +__global__ void dcnv3_col2im_gpu_kernel_shm_reduce_v2( + const int num_kernels, const scalar_t *grad_col, const scalar_t *data_im, + const scalar_t *data_offset, const scalar_t *data_mask, const int kernel_h, + const int kernel_w, const int stride_h, const int stride_w, const int pad_h, + const int pad_w, const int dilation_h, const int dilation_w, + const int group, const int group_channels, const int height_in, + const int width_in, const int height_out, const int width_out, + const opmath_t offset_scale, opmath_t *grad_im, opmath_t *grad_offset, + opmath_t *grad_mask) { + CUDA_KERNEL_LOOP(index, num_kernels) { + extern __shared__ int _s[]; + opmath_t *cache_grad_offset = (opmath_t *)_s; + opmath_t *cache_grad_mask = cache_grad_offset + 2 * blockDim.x; + unsigned int tid = threadIdx.x; + int _temp = index; + const int c_col = _temp % group_channels; + _temp /= group_channels; + const int sampling_index = _temp; + const int g_col = _temp % group; + _temp /= group; + const int p0_w = ((dilation_w * (kernel_w - 1)) >> 1) - pad_w + + (_temp % width_out) * stride_w; + _temp /= width_out; + const int p0_h = ((dilation_h * (kernel_h - 1)) >> 1) - pad_h + + (_temp % height_out) * stride_h; + _temp /= height_out; + const int b_col = _temp; + + const opmath_t top_grad = grad_col[index]; + const int input_size = height_in * width_in; + const int kernel_size = kernel_h * kernel_w; + int data_weight_ptr = sampling_index * kernel_size; + int data_loc_w_ptr = data_weight_ptr << 1; + const int grad_sampling_ptr = data_weight_ptr; + grad_offset += grad_sampling_ptr << 1; + grad_mask += grad_sampling_ptr; + const int qid_stride = group * group_channels; + const int im_ptr_offset = b_col * input_size * qid_stride; + const scalar_t *data_im_ptr = data_im + im_ptr_offset; + opmath_t *grad_im_ptr = grad_im + im_ptr_offset; + const opmath_t p0_w_ = + p0_w - ((dilation_w * (kernel_w - 1)) >> 1) * offset_scale; + const opmath_t p0_h_ = + p0_h - ((dilation_h * (kernel_h - 1)) >> 1) * offset_scale; + for (int i = 0; i < kernel_w; ++i) { + for (int j = 0; j < kernel_h; ++j) { + const opmath_t offset_w = data_offset[data_loc_w_ptr]; + const opmath_t offset_h = data_offset[data_loc_w_ptr + 1]; + const opmath_t loc_w = + p0_w_ + (i * dilation_w + offset_w) * offset_scale; + const opmath_t loc_h = + p0_h_ + (j * dilation_h + offset_h) * offset_scale; + const opmath_t weight = data_mask[data_weight_ptr]; + *(cache_grad_offset + (threadIdx.x << 1)) = 0; + *(cache_grad_offset + ((threadIdx.x << 1) + 1)) = 0; + *(cache_grad_mask + threadIdx.x) = 0; + if (loc_h > -1 && loc_w > -1 && loc_h < height_in && + loc_w < width_in) { + dcnv3_col2im_bilinear( + data_im_ptr, height_in, width_in, group, group_channels, + loc_h, loc_w, g_col, c_col, offset_scale, top_grad, + weight, grad_im_ptr, + cache_grad_offset + (threadIdx.x << 1), + cache_grad_mask + threadIdx.x); + } + + __syncthreads(); + + for (unsigned int s = blockDim.x / 2, spre = blockDim.x; s > 0; + s >>= 1, spre >>= 1) { + if (tid < s) { + const unsigned int xid1 = tid << 1; + const unsigned int xid2 = (tid + s) << 1; + cache_grad_mask[tid] += cache_grad_mask[tid + s]; + cache_grad_offset[xid1] += cache_grad_offset[xid2]; + cache_grad_offset[xid1 + 1] += + cache_grad_offset[xid2 + 1]; + if (tid + (s << 1) < spre) { + cache_grad_mask[tid] += + cache_grad_mask[tid + (s << 1)]; + cache_grad_offset[xid1] += + cache_grad_offset[xid2 + (s << 1)]; + cache_grad_offset[xid1 + 1] += + cache_grad_offset[xid2 + 1 + (s << 1)]; + } + } + __syncthreads(); + } + + if (tid == 0) { + *grad_offset = cache_grad_offset[0]; + *(grad_offset + 1) = cache_grad_offset[1]; + *grad_mask = cache_grad_mask[0]; + } + __syncthreads(); + + data_weight_ptr += 1; + data_loc_w_ptr += 2; + grad_mask += 1; + grad_offset += 2; + } + } + } +} + +template +__global__ void dcnv3_col2im_gpu_kernel_shm_reduce_v2_multi_blocks( + const int num_kernels, const scalar_t *grad_col, const scalar_t *data_im, + const scalar_t *data_offset, const scalar_t *data_mask, const int kernel_h, + const int kernel_w, const int stride_h, const int stride_w, const int pad_h, + const int pad_w, const int dilation_h, const int dilation_w, + const int group, const int group_channels, const int height_in, + const int width_in, const int height_out, const int width_out, + const opmath_t offset_scale, opmath_t *grad_im, opmath_t *grad_offset, + opmath_t *grad_mask) { + CUDA_KERNEL_LOOP(index, num_kernels) { + extern __shared__ int _s[]; + opmath_t *cache_grad_offset = (opmath_t *)_s; + opmath_t *cache_grad_mask = cache_grad_offset + 2 * blockDim.x; + unsigned int tid = threadIdx.x; + int _temp = index; + const int c_col = _temp % group_channels; + _temp /= group_channels; + const int sampling_index = _temp; + const int g_col = _temp % group; + _temp /= group; + const int p0_w = ((dilation_w * (kernel_w - 1)) >> 1) - pad_w + + (_temp % width_out) * stride_w; + _temp /= width_out; + const int p0_h = ((dilation_h * (kernel_h - 1)) >> 1) - pad_h + + (_temp % height_out) * stride_h; + _temp /= height_out; + const int b_col = _temp; + + const opmath_t top_grad = grad_col[index]; + const int input_size = height_in * width_in; + const int kernel_size = kernel_h * kernel_w; + int data_weight_ptr = sampling_index * kernel_size; + int data_loc_w_ptr = data_weight_ptr << 1; + const int grad_sampling_ptr = data_weight_ptr; + grad_offset += grad_sampling_ptr << 1; + grad_mask += grad_sampling_ptr; + const int qid_stride = group * group_channels; + const int im_ptr_offset = b_col * input_size * qid_stride; + const scalar_t *data_im_ptr = data_im + im_ptr_offset; + opmath_t *grad_im_ptr = grad_im + im_ptr_offset; + const opmath_t p0_w_ = + p0_w - ((dilation_w * (kernel_w - 1)) >> 1) * offset_scale; + const opmath_t p0_h_ = + p0_h - ((dilation_h * (kernel_h - 1)) >> 1) * offset_scale; + for (int i = 0; i < kernel_w; ++i) { + for (int j = 0; j < kernel_h; ++j) { + const opmath_t offset_w = data_offset[data_loc_w_ptr]; + const opmath_t offset_h = data_offset[data_loc_w_ptr + 1]; + const opmath_t loc_w = + p0_w_ + (i * dilation_w + offset_w) * offset_scale; + const opmath_t loc_h = + p0_h_ + (j * dilation_h + offset_h) * offset_scale; + const opmath_t weight = data_mask[data_weight_ptr]; + *(cache_grad_offset + (threadIdx.x << 1)) = 0; + *(cache_grad_offset + ((threadIdx.x << 1) + 1)) = 0; + *(cache_grad_mask + threadIdx.x) = 0; + if (loc_h > -1 && loc_w > -1 && loc_h < height_in && + loc_w < width_in) { + dcnv3_col2im_bilinear( + data_im_ptr, height_in, width_in, group, group_channels, + loc_h, loc_w, g_col, c_col, offset_scale, top_grad, + weight, grad_im_ptr, + cache_grad_offset + (threadIdx.x << 1), + cache_grad_mask + threadIdx.x); + } + + __syncthreads(); + + for (unsigned int s = blockDim.x / 2, spre = blockDim.x; s > 0; + s >>= 1, spre >>= 1) { + if (tid < s) { + const unsigned int xid1 = tid << 1; + const unsigned int xid2 = (tid + s) << 1; + cache_grad_mask[tid] += cache_grad_mask[tid + s]; + cache_grad_offset[xid1] += cache_grad_offset[xid2]; + cache_grad_offset[xid1 + 1] += + cache_grad_offset[xid2 + 1]; + if (tid + (s << 1) < spre) { + cache_grad_mask[tid] += + cache_grad_mask[tid + (s << 1)]; + cache_grad_offset[xid1] += + cache_grad_offset[xid2 + (s << 1)]; + cache_grad_offset[xid1 + 1] += + cache_grad_offset[xid2 + 1 + (s << 1)]; + } + } + __syncthreads(); + } + + if (tid == 0) { + atomicAdd(grad_offset, cache_grad_offset[0]); + atomicAdd(grad_offset + 1, cache_grad_offset[1]); + atomicAdd(grad_mask, cache_grad_mask[0]); + } + __syncthreads(); + + data_weight_ptr += 1; + data_loc_w_ptr += 2; + grad_mask += 1; + grad_offset += 2; + } + } + } +} + +template +__global__ void dcnv3_col2im_gpu_kernel_gm( + const int num_kernels, const scalar_t *grad_col, const scalar_t *data_im, + const scalar_t *data_offset, const scalar_t *data_mask, const int kernel_h, + const int kernel_w, const int stride_h, const int stride_w, const int pad_h, + const int pad_w, const int dilation_h, const int dilation_w, + const int group, const int group_channels, const int height_in, + const int width_in, const int height_out, const int width_out, + const opmath_t offset_scale, opmath_t *grad_im, opmath_t *grad_offset, + opmath_t *grad_mask) { + CUDA_KERNEL_LOOP(index, num_kernels) { + int _temp = index; + const int c_col = _temp % group_channels; + _temp /= group_channels; + const int sampling_index = _temp; + const int g_col = _temp % group; + _temp /= group; + const int p0_w = ((dilation_w * (kernel_w - 1)) >> 1) - pad_w + + (_temp % width_out) * stride_w; + _temp /= width_out; + const int p0_h = ((dilation_h * (kernel_h - 1)) >> 1) - pad_h + + (_temp % height_out) * stride_h; + _temp /= height_out; + const int b_col = _temp; + + const opmath_t top_grad = grad_col[index]; + const int input_size = height_in * width_in; + const int kernel_size = kernel_h * kernel_w; + int data_weight_ptr = sampling_index * kernel_size; + int data_loc_w_ptr = data_weight_ptr << 1; + const int grad_sampling_ptr = data_weight_ptr; + grad_offset += grad_sampling_ptr << 1; + grad_mask += grad_sampling_ptr; + const int qid_stride = group * group_channels; + const int im_ptr_offset = b_col * input_size * qid_stride; + const scalar_t *data_im_ptr = data_im + im_ptr_offset; + opmath_t *grad_im_ptr = grad_im + im_ptr_offset; + const opmath_t p0_w_ = + p0_w - ((dilation_w * (kernel_w - 1)) >> 1) * offset_scale; + const opmath_t p0_h_ = + p0_h - ((dilation_h * (kernel_h - 1)) >> 1) * offset_scale; + for (int i = 0; i < kernel_w; ++i) { + for (int j = 0; j < kernel_h; ++j) { + const opmath_t offset_w = data_offset[data_loc_w_ptr]; + const opmath_t offset_h = data_offset[data_loc_w_ptr + 1]; + const opmath_t loc_w = + p0_w_ + (i * dilation_w + offset_w) * offset_scale; + const opmath_t loc_h = + p0_h_ + (j * dilation_h + offset_h) * offset_scale; + const opmath_t weight = data_mask[data_weight_ptr]; + if (loc_h > -1 && loc_w > -1 && loc_h < height_in && + loc_w < width_in) { + dcnv3_col2im_bilinear_gm( + data_im_ptr, height_in, width_in, group, group_channels, + loc_h, loc_w, g_col, c_col, offset_scale, top_grad, + weight, grad_im_ptr, grad_offset, grad_mask); + } + data_weight_ptr += 1; + data_loc_w_ptr += 2; + grad_mask += 1; + grad_offset += 2; + } + } + } +} + +template +void dcnv3_im2col_cuda(cudaStream_t stream, const scalar_t *data_im, + const scalar_t *data_offset, const scalar_t *data_mask, + scalar_t *data_col, const int kernel_h, + const int kernel_w, const int stride_h, + const int stride_w, const int pad_h, const int pad_w, + const int dilation_h, const int dilation_w, + const int group, const int group_channels, + const int batch_n, const int height_in, + const int width_in, const int height_out, + const int width_out, const opmath_t offset_scale) { + const int num_kernels = + batch_n * height_out * width_out * group * group_channels; + const int num_actual_kernels = + batch_n * height_out * width_out * group * group_channels; + const int num_threads = CUDA_NUM_THREADS; + dcnv3_im2col_gpu_kernel + <<>>(num_kernels, data_im, data_offset, data_mask, data_col, + kernel_h, kernel_w, stride_h, stride_w, pad_h, pad_w, + dilation_h, dilation_w, group, group_channels, height_in, + width_in, height_out, width_out, offset_scale); + + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) { + printf("error in dcnv3_im2col_cuda: %s\n", cudaGetErrorString(err)); + } +} + +template +void dcnv3_col2im_cuda( + cudaStream_t stream, const scalar_t *grad_col, const scalar_t *data_im, + const scalar_t *data_offset, const scalar_t *data_mask, const int kernel_h, + const int kernel_w, const int stride_h, const int stride_w, const int pad_h, + const int pad_w, const int dilation_h, const int dilation_w, + const int group, const int group_channels, const int batch_n, + const int height_in, const int width_in, const int height_out, + const int width_out, const opmath_t offset_scale, opmath_t *grad_im, + opmath_t *grad_offset, opmath_t *grad_mask) { + const int num_threads = + (group_channels > CUDA_NUM_THREADS) ? CUDA_NUM_THREADS : group_channels; + const int num_kernels = + batch_n * height_out * width_out * group * group_channels; + const int num_actual_kernels = + batch_n * height_out * width_out * group * group_channels; + if (group_channels > 1024) { + if ((group_channels & 1023) == 0) { + dcnv3_col2im_gpu_kernel_shm_reduce_v2_multi_blocks + <<>>( + num_kernels, grad_col, data_im, data_offset, data_mask, + kernel_h, kernel_w, stride_h, stride_w, pad_h, pad_w, + dilation_h, dilation_w, group, group_channels, height_in, + width_in, height_out, width_out, offset_scale, grad_im, + grad_offset, grad_mask); + } else { + dcnv3_col2im_gpu_kernel_gm + <<>>(num_kernels, grad_col, data_im, data_offset, + data_mask, kernel_h, kernel_w, stride_h, stride_w, + pad_h, pad_w, dilation_h, dilation_w, group, + group_channels, height_in, width_in, height_out, + width_out, offset_scale, grad_im, grad_offset, + grad_mask); + } + } else { + switch (group_channels) { + case 1: + dcnv3_col2im_gpu_kernel_shm_blocksize_aware_reduce_v1 + <<>>(num_kernels, grad_col, data_im, data_offset, + data_mask, kernel_h, kernel_w, stride_h, stride_w, + pad_h, pad_w, dilation_h, dilation_w, group, + group_channels, height_in, width_in, height_out, + width_out, offset_scale, grad_im, grad_offset, + grad_mask); + break; + case 2: + dcnv3_col2im_gpu_kernel_shm_blocksize_aware_reduce_v1 + <<>>(num_kernels, grad_col, data_im, data_offset, + data_mask, kernel_h, kernel_w, stride_h, stride_w, + pad_h, pad_w, dilation_h, dilation_w, group, + group_channels, height_in, width_in, height_out, + width_out, offset_scale, grad_im, grad_offset, + grad_mask); + break; + case 4: + dcnv3_col2im_gpu_kernel_shm_blocksize_aware_reduce_v1 + <<>>(num_kernels, grad_col, data_im, data_offset, + data_mask, kernel_h, kernel_w, stride_h, stride_w, + pad_h, pad_w, dilation_h, dilation_w, group, + group_channels, height_in, width_in, height_out, + width_out, offset_scale, grad_im, grad_offset, + grad_mask); + break; + case 8: + dcnv3_col2im_gpu_kernel_shm_blocksize_aware_reduce_v1 + <<>>(num_kernels, grad_col, data_im, data_offset, + data_mask, kernel_h, kernel_w, stride_h, stride_w, + pad_h, pad_w, dilation_h, dilation_w, group, + group_channels, height_in, width_in, height_out, + width_out, offset_scale, grad_im, grad_offset, + grad_mask); + break; + case 16: + dcnv3_col2im_gpu_kernel_shm_blocksize_aware_reduce_v1 + <<>>(num_kernels, grad_col, data_im, data_offset, + data_mask, kernel_h, kernel_w, stride_h, stride_w, + pad_h, pad_w, dilation_h, dilation_w, group, + group_channels, height_in, width_in, height_out, + width_out, offset_scale, grad_im, grad_offset, + grad_mask); + break; + case 32: + dcnv3_col2im_gpu_kernel_shm_blocksize_aware_reduce_v1 + <<>>(num_kernels, grad_col, data_im, data_offset, + data_mask, kernel_h, kernel_w, stride_h, stride_w, + pad_h, pad_w, dilation_h, dilation_w, group, + group_channels, height_in, width_in, height_out, + width_out, offset_scale, grad_im, grad_offset, + grad_mask); + break; + case 64: + dcnv3_col2im_gpu_kernel_shm_blocksize_aware_reduce_v2 + <<>>(num_kernels, grad_col, data_im, data_offset, + data_mask, kernel_h, kernel_w, stride_h, stride_w, + pad_h, pad_w, dilation_h, dilation_w, group, + group_channels, height_in, width_in, height_out, + width_out, offset_scale, grad_im, grad_offset, + grad_mask); + break; + case 128: + dcnv3_col2im_gpu_kernel_shm_blocksize_aware_reduce_v2 + <<>>(num_kernels, grad_col, data_im, data_offset, + data_mask, kernel_h, kernel_w, stride_h, stride_w, + pad_h, pad_w, dilation_h, dilation_w, group, + group_channels, height_in, width_in, height_out, + width_out, offset_scale, grad_im, grad_offset, + grad_mask); + break; + case 256: + dcnv3_col2im_gpu_kernel_shm_blocksize_aware_reduce_v2 + <<>>(num_kernels, grad_col, data_im, data_offset, + data_mask, kernel_h, kernel_w, stride_h, stride_w, + pad_h, pad_w, dilation_h, dilation_w, group, + group_channels, height_in, width_in, height_out, + width_out, offset_scale, grad_im, grad_offset, + grad_mask); + break; + case 512: + dcnv3_col2im_gpu_kernel_shm_blocksize_aware_reduce_v2 + <<>>(num_kernels, grad_col, data_im, data_offset, + data_mask, kernel_h, kernel_w, stride_h, stride_w, + pad_h, pad_w, dilation_h, dilation_w, group, + group_channels, height_in, width_in, height_out, + width_out, offset_scale, grad_im, grad_offset, + grad_mask); + break; + case 1024: + dcnv3_col2im_gpu_kernel_shm_blocksize_aware_reduce_v2 + <<>>(num_kernels, grad_col, data_im, data_offset, + data_mask, kernel_h, kernel_w, stride_h, stride_w, + pad_h, pad_w, dilation_h, dilation_w, group, + group_channels, height_in, width_in, height_out, + width_out, offset_scale, grad_im, grad_offset, + grad_mask); + break; + default: + if (group_channels < 64) { + dcnv3_col2im_gpu_kernel_shm_reduce_v1 + <<>>( + num_kernels, grad_col, data_im, data_offset, data_mask, + kernel_h, kernel_w, stride_h, stride_w, pad_h, pad_w, + dilation_h, dilation_w, group, group_channels, + height_in, width_in, height_out, width_out, + offset_scale, grad_im, grad_offset, grad_mask); + } else { + dcnv3_col2im_gpu_kernel_shm_reduce_v2 + <<>>( + num_kernels, grad_col, data_im, data_offset, data_mask, + kernel_h, kernel_w, stride_h, stride_w, pad_h, pad_w, + dilation_h, dilation_w, group, group_channels, + height_in, width_in, height_out, width_out, + offset_scale, grad_im, grad_offset, grad_mask); + } + } + } + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) { + printf("error in dcnv3_col2im_cuda: %s\n", cudaGetErrorString(err)); + } +} \ No newline at end of file diff --git a/segmentation/ops_dcnv3/src/dcnv3.h b/segmentation/ops_dcnv3/src/dcnv3.h new file mode 100644 index 0000000..029648e --- /dev/null +++ b/segmentation/ops_dcnv3/src/dcnv3.h @@ -0,0 +1,59 @@ +/*! +************************************************************************************************** +* InternImage +* Copyright (c) 2022 OpenGVLab +* Licensed under The MIT License [see LICENSE for details] +************************************************************************************************** +* Modified from +*https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +************************************************************************************************** +*/ + +#pragma once + +#include "cpu/dcnv3_cpu.h" + +#ifdef WITH_CUDA +#include "cuda/dcnv3_cuda.h" +#endif + +at::Tensor dcnv3_forward(const at::Tensor &input, const at::Tensor &offset, + const at::Tensor &mask, const int kernel_h, + const int kernel_w, const int stride_h, + const int stride_w, const int pad_h, const int pad_w, + const int dilation_h, const int dilation_w, + const int group, const int group_channels, + const float offset_scale, const int im2col_step) { + if (input.type().is_cuda()) { +#ifdef WITH_CUDA + return dcnv3_cuda_forward(input, offset, mask, kernel_h, kernel_w, + stride_h, stride_w, pad_h, pad_w, dilation_h, + dilation_w, group, group_channels, + offset_scale, im2col_step); +#else + AT_ERROR("Not compiled with GPU support"); +#endif + } + AT_ERROR("Not implemented on the CPU"); +} + +std::vector +dcnv3_backward(const at::Tensor &input, const at::Tensor &offset, + const at::Tensor &mask, const int kernel_h, const int kernel_w, + const int stride_h, const int stride_w, const int pad_h, + const int pad_w, const int dilation_h, const int dilation_w, + const int group, const int group_channels, + const float offset_scale, const at::Tensor &grad_output, + const int im2col_step) { + if (input.type().is_cuda()) { +#ifdef WITH_CUDA + return dcnv3_cuda_backward(input, offset, mask, kernel_h, kernel_w, + stride_h, stride_w, pad_h, pad_w, dilation_h, + dilation_w, group, group_channels, + offset_scale, grad_output, im2col_step); +#else + AT_ERROR("Not compiled with GPU support"); +#endif + } + AT_ERROR("Not implemented on the CPU"); +} diff --git a/segmentation/ops_dcnv3/src/vision.cpp b/segmentation/ops_dcnv3/src/vision.cpp new file mode 100644 index 0000000..1f7a908 --- /dev/null +++ b/segmentation/ops_dcnv3/src/vision.cpp @@ -0,0 +1,17 @@ +/*! +************************************************************************************************** +* InternImage +* Copyright (c) 2022 OpenGVLab +* Licensed under The MIT License [see LICENSE for details] +************************************************************************************************** +* Modified from +*https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +************************************************************************************************** +*/ + +#include "dcnv3.h" + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("dcnv3_forward", &dcnv3_forward, "dcnv3_forward"); + m.def("dcnv3_backward", &dcnv3_backward, "dcnv3_backward"); +} diff --git a/segmentation/ops_dcnv3/test.py b/segmentation/ops_dcnv3/test.py new file mode 100644 index 0000000..467f952 --- /dev/null +++ b/segmentation/ops_dcnv3/test.py @@ -0,0 +1,263 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + +from __future__ import absolute_import +from __future__ import print_function +from __future__ import division + +import time +import torch +import torch.nn as nn +import math +from torch.autograd import gradcheck + +from functions.dcnv3_func import DCNv3Function, dcnv3_core_pytorch + +H_in, W_in = 8, 8 +N, M, D = 2, 4, 16 +Kh, Kw = 3, 3 +P = Kh * Kw +offset_scale = 2.0 +pad = 1 +dilation = 1 +stride = 1 +H_out = (H_in + 2 * pad - (dilation * (Kh - 1) + 1)) // stride + 1 +W_out = (W_in + 2 * pad - (dilation * (Kw - 1) + 1)) // stride + 1 + +torch.manual_seed(3) + + +@torch.no_grad() +def check_forward_equal_with_pytorch_double(): + input = torch.rand(N, H_in, W_in, M*D).cuda() * 0.01 + offset = torch.rand(N, H_out, W_out, M*P*2).cuda() * 10 + mask = torch.rand(N, H_out, W_out, M, P).cuda() + 1e-5 + mask /= mask.sum(-1, keepdim=True) + mask = mask.reshape(N, H_out, W_out, M*P) + + output_pytorch = dcnv3_core_pytorch( + input.double(), + offset.double(), + mask.double(), + Kh, Kw, stride, stride, Kh // 2, Kw // 2, dilation, dilation, M, D, offset_scale).detach().cpu() + + im2col_step = 2 + output_cuda = DCNv3Function.apply( + input.double(), + offset.double(), + mask.double(), + Kh, Kw, stride, stride, Kh // 2, Kw // 2, dilation, dilation, M, D, offset_scale, + im2col_step).detach().cpu() + + fwdok = torch.allclose(output_cuda, output_pytorch) + max_abs_err = (output_cuda - output_pytorch).abs().max() + max_rel_err = ((output_cuda - output_pytorch).abs() / + output_pytorch.abs()).max() + print('>>> forward double') + print(f'* {fwdok} check_forward_equal_with_pytorch_double: max_abs_err {max_abs_err:.2e} max_rel_err {max_rel_err:.2e}') + + +@torch.no_grad() +def check_forward_equal_with_pytorch_float(): + input = torch.rand(N, H_in, W_in, M*D).cuda() * 0.01 + offset = torch.rand(N, H_out, W_out, M*P*2).cuda() * 10 + mask = torch.rand(N, H_out, W_out, M, P).cuda() + 1e-5 + mask /= mask.sum(-1, keepdim=True) + mask = mask.reshape(N, H_out, W_out, M*P) + + output_pytorch = dcnv3_core_pytorch( + input, + offset, + mask, + Kh, Kw, stride, stride, Kh // 2, Kw // 2, dilation, dilation, M, D, offset_scale).detach().cpu() + + im2col_step = 2 + output_cuda = DCNv3Function.apply( + input, + offset, + mask, + Kh, Kw, stride, stride, Kh // 2, Kw // 2, dilation, dilation, M, D, offset_scale, + im2col_step).detach().cpu() + + fwdok = torch.allclose(output_cuda, output_pytorch, rtol=1e-2, atol=1e-3) + max_abs_err = (output_cuda - output_pytorch).abs().max() + max_rel_err = ((output_cuda - output_pytorch).abs() / + output_pytorch.abs()).max() + print('>>> forward float') + print(f'* {fwdok} check_forward_equal_with_pytorch_float: max_abs_err {max_abs_err:.2e} max_rel_err {max_rel_err:.2e}') + + +def check_backward_equal_with_pytorch_double(channels=4, grad_input=True, grad_offset=True, grad_mask=True): + # H_in, W_in = 4, 4 + N = 2 + M = 2 + H_out = (H_in + 2 * pad - (dilation * (Kh - 1) + 1)) // stride + 1 + W_out = (W_in + 2 * pad - (dilation * (Kw - 1) + 1)) // stride + 1 + + D = channels + input0 = torch.rand(N, H_in, W_in, M*D).cuda() * 0.01 + offset0 = torch.rand(N, H_out, W_out, M*P*2).cuda() * 10 + mask0 = torch.rand(N, H_out, W_out, M, P).cuda() + 1e-5 + mask0 /= mask0.sum(-1, keepdim=True) + mask0 = mask0.reshape(N, H_out, W_out, M*P) + input0.requires_grad = grad_input + offset0.requires_grad = grad_offset + mask0.requires_grad = grad_mask + + output_pytorch = dcnv3_core_pytorch( + input0.double(), + offset0.double(), + mask0.double(), + Kh, Kw, stride, stride, Kh // 2, Kw // 2, dilation, dilation, M, D, offset_scale) + output_pytorch.sum().backward() + + input1 = input0.detach() + offset1 = offset0.detach() + mask1 = mask0.detach() + input1.requires_grad = grad_input + offset1.requires_grad = grad_offset + mask1.requires_grad = grad_mask + + im2col_step = 2 + output_cuda = DCNv3Function.apply( + input1.double(), + offset1.double(), + mask1.double(), + Kh, Kw, stride, stride, Kh // 2, Kw // 2, dilation, dilation, M, D, offset_scale, + im2col_step) + output_cuda.sum().backward() + + print(f'>>> backward double: channels {D}') + bwdok = torch.allclose(input0.grad, input1.grad, rtol=1e-2, atol=1e-3) + max_abs_err = (input0.grad - input1.grad).abs().max() + max_rel_err = ((input0.grad - input1.grad).abs() / + input0.grad.abs()).max() + print( + f'* {bwdok} input_grad check_backward_equal_with_pytorch_double: max_abs_err {max_abs_err:.2e} max_rel_err {max_rel_err:.2e}') + + bwdok = torch.allclose(offset0.grad, offset1.grad, rtol=1e-2, atol=1e-3) + max_abs_err = (offset0.grad - offset1.grad).abs().max() + max_rel_err = ((offset0.grad - offset1.grad).abs() / + offset0.grad.abs()).max() + print( + f'* {bwdok} offset_grad check_backward_equal_with_pytorch_double: max_abs_err {max_abs_err:.2e} max_rel_err {max_rel_err:.2e}') + + bwdok = torch.allclose(mask0.grad, mask1.grad, rtol=1e-2, atol=1e-3) + max_abs_err = (mask0.grad - mask1.grad).abs().max() + max_rel_err = ((mask0.grad - mask1.grad).abs() / + mask0.grad.abs()).max() + print( + f'* {bwdok} mask_grad check_backward_equal_with_pytorch_double: max_abs_err {max_abs_err:.2e} max_rel_err {max_rel_err:.2e}') + + +def check_backward_equal_with_pytorch_float(channels=4, grad_input=True, grad_offset=True, grad_mask=True): + # H_in, W_in = 4, 4 + N = 2 + M = 2 + H_out = (H_in + 2 * pad - (dilation * (Kh - 1) + 1)) // stride + 1 + W_out = (W_in + 2 * pad - (dilation * (Kw - 1) + 1)) // stride + 1 + + D = channels + input0 = torch.rand(N, H_in, W_in, M*D).cuda() * 0.01 + offset0 = torch.rand(N, H_out, W_out, M*P*2).cuda() * 10 + mask0 = torch.rand(N, H_out, W_out, M, P).cuda() + 1e-5 + mask0 /= mask0.sum(-1, keepdim=True) + mask0 = mask0.reshape(N, H_out, W_out, M*P) + input0.requires_grad = grad_input + offset0.requires_grad = grad_offset + mask0.requires_grad = grad_mask + + output_pytorch = dcnv3_core_pytorch( + input0, + offset0, + mask0, + Kh, Kw, stride, stride, Kh // 2, Kw // 2, dilation, dilation, M, D, offset_scale) + output_pytorch.sum().backward() + + input1 = input0.detach() + offset1 = offset0.detach() + mask1 = mask0.detach() + input1.requires_grad = grad_input + offset1.requires_grad = grad_offset + mask1.requires_grad = grad_mask + + im2col_step = 2 + output_cuda = DCNv3Function.apply( + input1, + offset1, + mask1, + Kh, Kw, stride, stride, Kh // 2, Kw // 2, dilation, dilation, M, D, offset_scale, + im2col_step) + output_cuda.sum().backward() + + print(f'>>> backward float: channels {D}') + bwdok = torch.allclose(input0.grad, input1.grad, rtol=1e-2, atol=1e-3) + max_abs_err = (input0.grad - input1.grad).abs().max() + max_rel_err = ((input0.grad - input1.grad).abs() / + input0.grad.abs()).max() + print( + f'* {bwdok} input_grad check_backward_equal_with_pytorch_float: max_abs_err {max_abs_err:.2e} max_rel_err {max_rel_err:.2e}') + + bwdok = torch.allclose(offset0.grad, offset1.grad, rtol=1e-2, atol=1e-3) + max_abs_err = (offset0.grad - offset1.grad).abs().max() + max_rel_err = ((offset0.grad - offset1.grad).abs() / + offset0.grad.abs()).max() + print( + f'* {bwdok} offset_grad check_backward_equal_with_pytorch_float: max_abs_err {max_abs_err:.2e} max_rel_err {max_rel_err:.2e}') + + bwdok = torch.allclose(mask0.grad, mask1.grad, rtol=1e-2, atol=1e-3) + max_abs_err = (mask0.grad - mask1.grad).abs().max() + max_rel_err = ((mask0.grad - mask1.grad).abs() / + mask0.grad.abs()).max() + print( + f'* {bwdok} mask_grad check_backward_equal_with_pytorch_float: max_abs_err {max_abs_err:.2e} max_rel_err {max_rel_err:.2e}') + + +@torch.no_grad() +def check_time_cost(im2col_step=128): + N = 512 + H_in, W_in = 64, 64 + H_out = (H_in + 2 * pad - (dilation * (Kh - 1) + 1)) // stride + 1 + W_out = (W_in + 2 * pad - (dilation * (Kw - 1) + 1)) // stride + 1 + + input = torch.rand(N, H_in, W_in, M*D).cuda() * 0.01 + offset = torch.rand(N, H_out, W_out, M*P*2).cuda() * 10 + mask = torch.rand(N, H_out, W_out, M, P).cuda() + 1e-5 + mask /= mask.sum(-1, keepdim=True) + mask = mask.reshape(N, H_out, W_out, M*P) + print( + f'>>> time cost: im2col_step {im2col_step}; input {input.shape}; points {P} ') + repeat = 100 + for i in range(repeat): + output_cuda = DCNv3Function.apply( + input, + offset, + mask, + Kh, Kw, stride, stride, Kh // 2, Kw // 2, dilation, dilation, M, D, 1.0, + im2col_step) + torch.cuda.synchronize() + start = time.time() + for i in range(repeat): + output_cuda = DCNv3Function.apply( + input, + offset, + mask, + Kh, Kw, stride, stride, Kh // 2, Kw // 2, dilation, dilation, M, D, 1.0, + im2col_step) + torch.cuda.synchronize() + print(f'foward time cost: {(time.time() - start) / repeat}') + + +if __name__ == '__main__': + check_forward_equal_with_pytorch_double() + check_forward_equal_with_pytorch_float() + for channels in [1, 16, 30, 32, 64, 71, 1025]: + check_backward_equal_with_pytorch_double(channels, True, True, True) + for channels in [1, 16, 30, 32, 64, 71, 1025]: + check_backward_equal_with_pytorch_float(channels, True, True, True) + for i in range(3): + im2col_step = 128 * (2 ** i) + check_time_cost(im2col_step) diff --git a/segmentation/slurm_test.sh b/segmentation/slurm_test.sh new file mode 100644 index 0000000..e200c97 --- /dev/null +++ b/segmentation/slurm_test.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +set -x + +PARTITION=$1 +JOB_NAME=$2 +CONFIG=$3 +CHECKPOINT=$4 +GPUS=${GPUS:-8} +GPUS_PER_NODE=${GPUS_PER_NODE:-8} +CPUS_PER_TASK=${CPUS_PER_TASK:-5} +PY_ARGS=${@:5} +SRUN_ARGS=${SRUN_ARGS:-""} + +PYTHONPATH="$(dirname $0)/..":$PYTHONPATH \ +srun -p ${PARTITION} \ + --job-name=${JOB_NAME} \ + --gres=gpu:${GPUS_PER_NODE} \ + --ntasks=${GPUS} \ + --ntasks-per-node=${GPUS_PER_NODE} \ + --cpus-per-task=${CPUS_PER_TASK} \ + --kill-on-bad-exit=1 \ + --quotatype=auto \ + ${SRUN_ARGS} \ + python -u test.py ${CONFIG} ${CHECKPOINT} --launcher="slurm" ${PY_ARGS} diff --git a/segmentation/slurm_train.sh b/segmentation/slurm_train.sh new file mode 100755 index 0000000..233cbea --- /dev/null +++ b/segmentation/slurm_train.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +set -x + +PARTITION=$1 +JOB_NAME=$2 +CONFIG=$3 +GPUS=${GPUS:-8} +GPUS_PER_NODE=${GPUS_PER_NODE:-8} +CPUS_PER_TASK=${CPUS_PER_TASK:-5} +SRUN_ARGS=${SRUN_ARGS:-""} +PY_ARGS=${@:4} + +PYTHONPATH="$(dirname $0)/..":$PYTHONPATH \ +srun -p ${PARTITION} \ + --job-name=${JOB_NAME} \ + --gres=gpu:${GPUS_PER_NODE} \ + --ntasks=${GPUS} \ + --ntasks-per-node=${GPUS_PER_NODE} \ + --cpus-per-task=${CPUS_PER_TASK} \ + --quotatype=reserved \ + --kill-on-bad-exit=1 \ + ${SRUN_ARGS} \ + python -u train.py ${CONFIG} --launcher="slurm" ${PY_ARGS} diff --git a/segmentation/test.py b/segmentation/test.py new file mode 100644 index 0000000..aa22a25 --- /dev/null +++ b/segmentation/test.py @@ -0,0 +1,294 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + +import argparse +import os +import os.path as osp +import shutil +import time +import warnings + +import mmcv +import torch +from mmcv.parallel import MMDataParallel, MMDistributedDataParallel +from mmcv.runner import (get_dist_info, init_dist, load_checkpoint, + wrap_fp16_model, load_state_dict) +from mmcv.utils import DictAction +from mmseg.apis import multi_gpu_test, single_gpu_test +from mmseg.datasets import build_dataloader, build_dataset +from mmseg.models import build_segmentor + +import mmcv_custom # noqa: F401,F403 +import mmseg_custom # noqa: F401,F403 + + +def parse_args(): + parser = argparse.ArgumentParser( + description='mmseg test (and eval) a model') + parser.add_argument('config', help='test config file path') + parser.add_argument('checkpoint', help='checkpoint file') + parser.add_argument( + '--work-dir', + help=('if specified, the evaluation metric results will be dumped' + 'into the directory as json')) + parser.add_argument( + '--aug-test', action='store_true', help='Use Flip and Multi scale aug') + parser.add_argument('--out', help='output result file in pickle format') + parser.add_argument('--dir-name', help='dir name') + parser.add_argument( + '--format-only', + action='store_true', + help='Format the output results without perform evaluation. It is' + 'useful when you want to format the result to a specific format and ' + 'submit it to the test server') + parser.add_argument( + '--eval', + type=str, + nargs='+', + help='evaluation metrics, which depends on the dataset, e.g., "mIoU"' + ' for generic datasets, and "cityscapes" for Cityscapes') + parser.add_argument('--show', action='store_true', help='show results') + parser.add_argument( + '--show-dir', help='directory where painted images will be saved') + parser.add_argument( + '--gpu-collect', + action='store_true', + help='whether to use gpu to collect results.') + parser.add_argument( + '--tmpdir', + help='tmp directory used for collecting results from multiple ' + 'workers, available when gpu_collect is not specified') + parser.add_argument( + '--options', + nargs='+', + action=DictAction, + help="--options is deprecated in favor of --cfg_options' and it will " + 'not be supported in version v0.22.0. Override some settings in the ' + 'used config, the key-value pair in xxx=yyy format will be merged ' + 'into config file. If the value to be overwritten is a list, it ' + 'should be like key="[a,b]" or key=a,b It also allows nested ' + 'list/tuple values, e.g. key="[(a,b),(c,d)]" Note that the quotation ' + 'marks are necessary and that no white space is allowed.') + parser.add_argument( + '--cfg-options', + nargs='+', + action=DictAction, + help='override some settings in the used config, the key-value pair ' + 'in xxx=yyy format will be merged into config file. If the value to ' + 'be overwritten is a list, it should be like key="[a,b]" or key=a,b ' + 'It also allows nested list/tuple values, e.g. key="[(a,b),(c,d)]" ' + 'Note that the quotation marks are necessary and that no white space ' + 'is allowed.') + parser.add_argument( + '--eval-options', + nargs='+', + action=DictAction, + help='custom options for evaluation') + parser.add_argument( + '--launcher', + choices=['none', 'pytorch', 'slurm', 'mpi'], + default='none', + help='job launcher') + parser.add_argument( + '--opacity', + type=float, + default=0.5, + help='Opacity of painted segmentation map. In (0, 1] range.') + parser.add_argument('--local_rank', type=int, default=0) + args = parser.parse_args() + if 'LOCAL_RANK' not in os.environ: + os.environ['LOCAL_RANK'] = str(args.local_rank) + + if args.options and args.cfg_options: + raise ValueError( + '--options and --cfg-options cannot be both ' + 'specified, --options is deprecated in favor of --cfg-options. ' + '--options will not be supported in version v0.22.0.') + if args.options: + warnings.warn('--options is deprecated in favor of --cfg-options. ' + '--options will not be supported in version v0.22.0.') + args.cfg_options = args.options + + return args + + +def main(): + args = parse_args() + assert args.out or args.eval or args.format_only or args.show \ + or args.show_dir, \ + ('Please specify at least one operation (save/eval/format/show the ' + 'results / save the results) with the argument "--out", "--eval"' + ', "--format-only", "--show" or "--show-dir"') + + if args.eval and args.format_only: + raise ValueError('--eval and --format_only cannot be both specified') + + if args.out is not None and not args.out.endswith(('.pkl', '.pickle')): + raise ValueError('The output file must be a pkl file.') + + cfg = mmcv.Config.fromfile(args.config) + if args.cfg_options is not None: + cfg.merge_from_dict(args.cfg_options) + # set cudnn_benchmark + if cfg.get('cudnn_benchmark', False): + torch.backends.cudnn.benchmark = True + if args.aug_test: + # hard code index + cfg.data.test.pipeline[1].img_ratios = [ + 0.5, 0.75, 1.0, 1.25, 1.5, 1.75 + ] + cfg.data.test.pipeline[1].flip = True + cfg.model.pretrained = None + cfg.data.test.test_mode = True + + # init distributed env first, since logger depends on the dist info. + if args.launcher == 'none': + distributed = False + else: + distributed = True + init_dist(args.launcher, **cfg.dist_params) + + rank, _ = get_dist_info() + # allows not to create + if args.work_dir is not None and rank == 0: + mmcv.mkdir_or_exist(osp.abspath(args.work_dir)) + timestamp = time.strftime('%Y%m%d_%H%M%S', time.localtime()) + if args.aug_test: + json_file = osp.join(args.work_dir, + f'eval_multi_scale_{timestamp}.json') + else: + json_file = osp.join(args.work_dir, + f'eval_single_scale_{timestamp}.json') + elif rank == 0: + work_dir = osp.join('./work_dirs', + osp.splitext(osp.basename(args.config))[0]) + mmcv.mkdir_or_exist(osp.abspath(work_dir)) + timestamp = time.strftime('%Y%m%d_%H%M%S', time.localtime()) + if args.aug_test: + json_file = osp.join(work_dir, + f'eval_multi_scale_{timestamp}.json') + else: + json_file = osp.join(work_dir, + f'eval_single_scale_{timestamp}.json') + + # build the dataloader + # TODO: support multiple images per gpu (only minor changes are needed) + dataset = build_dataset(cfg.data.test) + data_loader = build_dataloader( + dataset, + samples_per_gpu=1, + workers_per_gpu=cfg.data.workers_per_gpu, + dist=distributed, + shuffle=False) + + # build the model and load checkpoint + cfg.model.train_cfg = None + model = build_segmentor(cfg.model, test_cfg=cfg.get('test_cfg')) + + model_without_ddp = model + n_parameters = sum(p.numel() for p in model.parameters() + if p.requires_grad) + print(f"number of params: {n_parameters}") + + fp16_cfg = cfg.get('fp16', None) + if fp16_cfg is not None: + wrap_fp16_model(model) + checkpoint = load_checkpoint(model, args.checkpoint, map_location='cpu') + + if hasattr(model, 'module'): + load_state_dict(model.module, checkpoint['state_dict'], strict=False) + else: + load_state_dict(model, checkpoint['state_dict'], strict=False) + + if 'CLASSES' in checkpoint.get('meta', {}): + model.CLASSES = checkpoint['meta']['CLASSES'] + else: + print('"CLASSES" not found in meta, use dataset.CLASSES instead') + model.CLASSES = dataset.CLASSES + if 'PALETTE' in checkpoint.get('meta', {}): + model.PALETTE = checkpoint['meta']['PALETTE'] + else: + print('"PALETTE" not found in meta, use dataset.PALETTE instead') + model.PALETTE = dataset.PALETTE + + # clean gpu memory when starting a new evaluation. + torch.cuda.empty_cache() + eval_kwargs = {} if args.eval_options is None else args.eval_options + + # Deprecated + efficient_test = eval_kwargs.get('efficient_test', False) + if efficient_test: + warnings.warn( + '``efficient_test=True`` does not have effect in tools/test.py, ' + 'the evaluation and format results are CPU memory efficient by ' + 'default') + + eval_on_format_results = ( + args.eval is not None and 'cityscapes' in args.eval) + if eval_on_format_results: + assert len(args.eval) == 1, 'eval on format results is not ' \ + 'applicable for metrics other than ' \ + 'cityscapes' + if args.format_only or eval_on_format_results: + if 'imgfile_prefix' in eval_kwargs: + tmpdir = eval_kwargs['imgfile_prefix'] + else: + tmpdir = '.format_cityscapes' + eval_kwargs.setdefault('imgfile_prefix', tmpdir) + mmcv.mkdir_or_exist(tmpdir) + else: + tmpdir = None + + if not distributed: + model = MMDataParallel(model, device_ids=[0]) + results = single_gpu_test( + model, + data_loader, + args.show, + args.show_dir, + False, + args.opacity, + pre_eval=args.eval is not None and not eval_on_format_results, + format_only=args.format_only or eval_on_format_results, + format_args=eval_kwargs) + else: + model = MMDistributedDataParallel( + model.cuda(), + device_ids=[torch.cuda.current_device()], + broadcast_buffers=False) + results = multi_gpu_test( + model, + data_loader, + args.tmpdir, + args.gpu_collect, + False, + pre_eval=args.eval is not None and not eval_on_format_results, + format_only=args.format_only or eval_on_format_results, + format_args=eval_kwargs) + + + rank, _ = get_dist_info() + if rank == 0: + if args.out: + warnings.warn( + 'The behavior of ``args.out`` has been changed since MMSeg ' + 'v0.16, the pickled outputs could be seg map as type of ' + 'np.array, pre-eval results or file paths for ' + '``dataset.format_results()``.') + print(f'\nwriting results to {args.out}') + mmcv.dump(results, args.out) + if args.eval: + eval_kwargs.update(metric=args.eval) + metric = dataset.evaluate(results, **eval_kwargs) + metric_dict = dict(config=args.config, metric=metric) + mmcv.dump(metric_dict, json_file, indent=4) + if tmpdir is not None and eval_on_format_results: + # remove tmp dir when cityscapes evaluation + shutil.rmtree(tmpdir) + + +if __name__ == '__main__': + main() diff --git a/segmentation/train.py b/segmentation/train.py new file mode 100644 index 0000000..637704a --- /dev/null +++ b/segmentation/train.py @@ -0,0 +1,251 @@ +# -------------------------------------------------------- +# DCNv4 +# Copyright (c) 2024 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- + + +import argparse +import copy +import os +import os.path as osp +import time +import warnings + +import mmcv +import torch +import torch.distributed as dist +from mmcv.cnn.utils import revert_sync_batchnorm +from mmcv.runner import get_dist_info, init_dist +from mmcv.utils import Config, DictAction, get_git_hash + +from mmseg import __version__ +from mmseg.apis import init_random_seed, set_random_seed, train_segmentor +from mmseg.datasets import build_dataset +from mmseg.models import build_segmentor +from mmseg.utils import (collect_env, get_device, get_root_logger, + setup_multi_processes) +import mmcv_custom # noqa: F401,F403 +import mmseg_custom # noqa: F401,F403 + + +def parse_args(): + parser = argparse.ArgumentParser(description='Train a segmentor') + parser.add_argument('config', help='train config file path') + parser.add_argument('--work-dir', help='the dir to save logs and models') + parser.add_argument('--load-from', + help='the checkpoint file to load weights from') + parser.add_argument('--resume-from', + help='the checkpoint file to resume from') + parser.add_argument( + '--no-validate', + action='store_true', + help='whether not to evaluate the checkpoint during training') + group_gpus = parser.add_mutually_exclusive_group() + group_gpus.add_argument( + '--gpus', + type=int, + help='(Deprecated, please use --gpu-id) number of gpus to use ' + '(only applicable to non-distributed training)') + group_gpus.add_argument( + '--gpu-ids', + type=int, + nargs='+', + help='(Deprecated, please use --gpu-id) ids of gpus to use ' + '(only applicable to non-distributed training)') + group_gpus.add_argument('--gpu-id', + type=int, + default=0, + help='id of gpu to use ' + '(only applicable to non-distributed training)') + parser.add_argument('--seed', type=int, default=None, help='random seed') + parser.add_argument( + '--diff_seed', + action='store_true', + help='Whether or not set different seeds for different ranks') + parser.add_argument( + '--deterministic', + action='store_true', + help='whether to set deterministic options for CUDNN backend.') + parser.add_argument( + '--options', + nargs='+', + action=DictAction, + help="--options is deprecated in favor of --cfg_options' and it will " + 'not be supported in version v0.22.0. Override some settings in the ' + 'used config, the key-value pair in xxx=yyy format will be merged ' + 'into config file. If the value to be overwritten is a list, it ' + 'should be like key="[a,b]" or key=a,b It also allows nested ' + 'list/tuple values, e.g. key="[(a,b),(c,d)]" Note that the quotation ' + 'marks are necessary and that no white space is allowed.') + parser.add_argument( + '--cfg-options', + nargs='+', + action=DictAction, + help='override some settings in the used config, the key-value pair ' + 'in xxx=yyy format will be merged into config file. If the value to ' + 'be overwritten is a list, it should be like key="[a,b]" or key=a,b ' + 'It also allows nested list/tuple values, e.g. key="[(a,b),(c,d)]" ' + 'Note that the quotation marks are necessary and that no white space ' + 'is allowed.') + parser.add_argument('--launcher', + choices=['none', 'pytorch', 'slurm', 'mpi'], + default='none', + help='job launcher') + parser.add_argument('--local_rank', type=int, default=0) + parser.add_argument( + '--auto-resume', + action='store_true', + help='resume from the latest checkpoint automatically.') + args = parser.parse_args() + if 'LOCAL_RANK' not in os.environ: + os.environ['LOCAL_RANK'] = str(args.local_rank) + + if args.options and args.cfg_options: + raise ValueError( + '--options and --cfg-options cannot be both ' + 'specified, --options is deprecated in favor of --cfg-options. ' + '--options will not be supported in version v0.22.0.') + if args.options: + warnings.warn('--options is deprecated in favor of --cfg-options. ' + '--options will not be supported in version v0.22.0.') + args.cfg_options = args.options + + return args + + +def main(): + # os.environ['CN2DCN'] = True + args = parse_args() + + cfg = Config.fromfile(args.config) + if args.cfg_options is not None: + cfg.merge_from_dict(args.cfg_options) + # set cudnn_benchmark + if cfg.get('cudnn_benchmark', False): + torch.backends.cudnn.benchmark = True + + # work_dir is determined in this priority: CLI > segment in file > filename + if args.work_dir is not None: + # update configs according to CLI args if args.work_dir is not None + cfg.work_dir = args.work_dir + elif cfg.get('work_dir', None) is None: + # use config filename as default work_dir if cfg.work_dir is None + cfg.work_dir = osp.join('./work_dirs', + osp.splitext(osp.basename(args.config))[0]) + if args.load_from is not None: + cfg.load_from = args.load_from + if args.resume_from is not None: + cfg.resume_from = args.resume_from + if args.gpus is not None: + cfg.gpu_ids = range(1) + warnings.warn('`--gpus` is deprecated because we only support ' + 'single GPU mode in non-distributed training. ' + 'Use `gpus=1` now.') + if args.gpu_ids is not None: + cfg.gpu_ids = args.gpu_ids[0:1] + warnings.warn('`--gpu-ids` is deprecated, please use `--gpu-id`. ' + 'Because we only support single GPU mode in ' + 'non-distributed training. Use the first GPU ' + 'in `gpu_ids` now.') + if args.gpus is None and args.gpu_ids is None: + cfg.gpu_ids = [args.gpu_id] + + cfg.auto_resume = args.auto_resume + + # init distributed env first, since logger depends on the dist info. + if args.launcher == 'none': + distributed = False + else: + distributed = True + init_dist(args.launcher, **cfg.dist_params) + # gpu_ids is used to calculate iter when resuming checkpoint + _, world_size = get_dist_info() + cfg.gpu_ids = range(world_size) + + # create work_dir + mmcv.mkdir_or_exist(osp.abspath(cfg.work_dir)) + # dump config + cfg.dump(osp.join(cfg.work_dir, osp.basename(args.config))) + # init the logger before other steps + timestamp = time.strftime('%Y%m%d_%H%M%S', time.localtime()) + log_file = osp.join(cfg.work_dir, f'{timestamp}.log') + logger = get_root_logger(log_file=log_file, log_level=cfg.log_level) + + # set multi-process settings + setup_multi_processes(cfg) + + # init the meta dict to record some important information such as + # environment info and seed, which will be logged + meta = dict() + # log env info + env_info_dict = collect_env() + env_info = '\n'.join([f'{k}: {v}' for k, v in env_info_dict.items()]) + dash_line = '-' * 60 + '\n' + logger.info('Environment info:\n' + dash_line + env_info + '\n' + + dash_line) + meta['env_info'] = env_info + + # log some basic info + logger.info(f'Distributed training: {distributed}') + logger.info(f'Config:\n{cfg.pretty_text}') + + # set random seeds + cfg.device = get_device() + seed = init_random_seed(args.seed, device=cfg.device) + seed = seed + dist.get_rank() if args.diff_seed else seed + logger.info(f'Set random seed to {seed}, ' + f'deterministic: {args.deterministic}') + set_random_seed(seed, deterministic=args.deterministic) + cfg.seed = seed + meta['seed'] = seed + meta['exp_name'] = osp.basename(args.config) + + model = build_segmentor(cfg.model, + train_cfg=cfg.get('train_cfg'), + test_cfg=cfg.get('test_cfg')) + print(model) + model.init_weights() + + # SyncBN is not support for DP + if not distributed: + warnings.warn( + 'SyncBN is only supported with DDP. To be compatible with DP, ' + 'we convert SyncBN to BN. Please use dist_train.sh which can ' + 'avoid this error.') + model = revert_sync_batchnorm(model) + + # logger.info(model) + + datasets = [build_dataset(cfg.data.train)] + if len(cfg.workflow) == 2: + val_dataset = copy.deepcopy(cfg.data.val) + val_dataset.pipeline = cfg.data.train.pipeline + datasets.append(build_dataset(val_dataset)) + if cfg.checkpoint_config is not None: + # save mmseg version, config file content and class names in + # checkpoints as meta data + cfg.checkpoint_config.meta = dict( + mmseg_version=f'{__version__}+{get_git_hash()[:7]}', + config=cfg.pretty_text, + CLASSES=datasets[0].CLASSES, + PALETTE=datasets[0].PALETTE) + # add an attribute for visualization convenience + model.CLASSES = datasets[0].CLASSES + # passing checkpoint meta for saving best checkpoint + meta.update(cfg.checkpoint_config.meta) + + torch.backends.cuda.matmul.allow_tf32 = True + torch.backends.cudnn.allow_tf32 = True + + train_segmentor(model, + datasets, + cfg, + distributed=distributed, + validate=(not args.no_validate), + timestamp=timestamp, + meta=meta) + + +if __name__ == '__main__': + main()