mirror of
https://github.com/facebookresearch/fairseq.git
synced 2024-09-11 09:15:43 +03:00
Merge branch 'main' into master
This commit is contained in:
commit
9f703949f5
159
.circleci/config.yml
Normal file
159
.circleci/config.yml
Normal file
@ -0,0 +1,159 @@
|
||||
# Use 2.1 for orbs
|
||||
version: 2.1
|
||||
|
||||
# -------------------------------------------------------------------------------------
|
||||
# Environments to run the jobs in
|
||||
# -------------------------------------------------------------------------------------
|
||||
gpu: &gpu
|
||||
environment:
|
||||
CUDA_VERSION: "11.1"
|
||||
machine:
|
||||
image: ubuntu-1604-cuda-11.1:202012-01
|
||||
resource_class: gpu.nvidia.medium.multi
|
||||
|
||||
|
||||
# -------------------------------------------------------------------------------------
|
||||
# Re-usable commands
|
||||
# -------------------------------------------------------------------------------------
|
||||
cache_key: &cache_key cache-key-{{ .Environment.CIRCLE_JOB }}-{{ checksum ".circleci/config.yml" }}-{{ checksum "setup.py"}}
|
||||
|
||||
install_dep_common: &install_dep_common
|
||||
- run:
|
||||
name: Install Common Dependencies
|
||||
command: |
|
||||
source activate fairseq
|
||||
pip install --upgrade setuptools
|
||||
pip install bitarray boto3 deepspeed editdistance fastBPE iopath ipdb ipython pyarrow pytest sacremoses sentencepiece subword-nmt hydra-core==1.0.7 omegaconf==2.0.6
|
||||
pip install --progress-bar off pytest
|
||||
pip install --progress-bar off fairscale
|
||||
pip install -i https://test.pypi.org/simple/ bitsandbytes-cuda111 -U
|
||||
python -c 'import torch; print("Torch version:", torch.__version__)'
|
||||
python -m torch.utils.collect_env
|
||||
|
||||
install_dep_fused_ops: &install_dep_fused_ops
|
||||
- run:
|
||||
name: Install Megatron/Apex Dependencies
|
||||
working_directory: ~/
|
||||
command: |
|
||||
source activate fairseq
|
||||
git clone https://github.com/NVIDIA/apex
|
||||
cd apex
|
||||
git checkout e2083df5eb96643c61613b9df48dd4eea6b07690
|
||||
pip install -v --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext" --global-option="--deprecated_fused_adam" --global-option="--xentropy" --global-option="--fast_multihead_attn" ./
|
||||
cd ~/
|
||||
git clone --depth=1 --branch v2.4 https://github.com/NVIDIA/Megatron-LM.git
|
||||
cd Megatron-LM
|
||||
pip install -e .
|
||||
|
||||
|
||||
install_dep_pt19: &install_dep_pt19
|
||||
- run:
|
||||
name: Install Pytorch Dependencies
|
||||
command: |
|
||||
source activate fairseq
|
||||
pip install --upgrade setuptools
|
||||
pip install torch==1.9.1+cu111 torchvision==0.10.1+cu111 torchaudio==0.9.1 -f https://download.pytorch.org/whl/torch_stable.html
|
||||
python -c 'import torch; print("Torch version:", torch.__version__)'
|
||||
|
||||
install_dep_pt18: &install_dep_pt18
|
||||
- run:
|
||||
name: Install Pytorch Dependencies
|
||||
command: |
|
||||
source activate fairseq
|
||||
pip install --upgrade setuptools
|
||||
pip install torch==1.8.1+cu111 torchvision==0.9.1+cu111 torchaudio==0.8.1 -f https://download.pytorch.org/whl/torch_stable.html
|
||||
python -c 'import torch; print("Torch version:", torch.__version__)'
|
||||
|
||||
install_repo: &install_repo
|
||||
- run:
|
||||
name: Install Repository
|
||||
command: |
|
||||
source activate fairseq
|
||||
pip install .
|
||||
python setup.py build_ext --inplace
|
||||
|
||||
run_unittests: &run_unittests
|
||||
- run:
|
||||
name: Run Unit Tests
|
||||
command: |
|
||||
source activate fairseq
|
||||
pytest tests/gpu/test_binaries_gpu.py
|
||||
|
||||
check_nvidia_driver: &check_nvidia_driver
|
||||
- run:
|
||||
name: Check NVIDIA Driver
|
||||
working_directory: ~/
|
||||
command: |
|
||||
pyenv versions
|
||||
nvidia-smi
|
||||
|
||||
create_conda_env: &create_conda_env
|
||||
- run:
|
||||
name: Install and Create Conda Environment
|
||||
command: |
|
||||
curl -o ~/miniconda.sh -O https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh
|
||||
chmod +x ~/miniconda.sh
|
||||
~/miniconda.sh -b -p $HOME/miniconda
|
||||
rm ~/miniconda.sh
|
||||
echo 'export PATH=$HOME/miniconda/bin:$PATH' >> $BASH_ENV
|
||||
source $BASH_ENV
|
||||
if [ ! -d ~/miniconda/envs/fairseq ]
|
||||
then
|
||||
conda create -y -n fairseq python=3.8
|
||||
fi
|
||||
source activate fairseq
|
||||
python --version
|
||||
pip install --upgrade pip
|
||||
# -------------------------------------------------------------------------------------
|
||||
# Jobs to run
|
||||
# -------------------------------------------------------------------------------------
|
||||
|
||||
jobs:
|
||||
gpu_tests_pt19:
|
||||
<<: *gpu
|
||||
|
||||
working_directory: ~/fairseq-py
|
||||
|
||||
steps:
|
||||
- checkout
|
||||
- <<: *check_nvidia_driver
|
||||
- <<: *create_conda_env
|
||||
- restore_cache:
|
||||
key: *cache_key
|
||||
- <<: *install_dep_pt19
|
||||
- <<: *install_dep_common
|
||||
- <<: *install_dep_fused_ops
|
||||
- save_cache:
|
||||
paths:
|
||||
- ~/miniconda/
|
||||
key: *cache_key
|
||||
- <<: *install_repo
|
||||
- <<: *run_unittests
|
||||
|
||||
gpu_tests_pt18:
|
||||
<<: *gpu
|
||||
|
||||
working_directory: ~/fairseq-py
|
||||
|
||||
steps:
|
||||
- checkout
|
||||
- <<: *check_nvidia_driver
|
||||
- <<: *create_conda_env
|
||||
- restore_cache:
|
||||
key: *cache_key
|
||||
- <<: *install_dep_pt18
|
||||
- <<: *install_dep_common
|
||||
- <<: *install_dep_fused_ops
|
||||
- save_cache:
|
||||
paths:
|
||||
- ~/miniconda/
|
||||
key: *cache_key
|
||||
- <<: *install_repo
|
||||
- <<: *run_unittests
|
||||
|
||||
workflows:
|
||||
version: 2
|
||||
build:
|
||||
jobs:
|
||||
- gpu_tests_pt18
|
||||
- gpu_tests_pt19
|
9
.github/workflows/build.yml
vendored
9
.github/workflows/build.yml
vendored
@ -14,7 +14,7 @@ jobs:
|
||||
max-parallel: 4
|
||||
matrix:
|
||||
platform: [ubuntu-latest, macos-latest]
|
||||
python-version: [3.6, 3.7]
|
||||
python-version: [3.8, 3.9]
|
||||
|
||||
runs-on: ${{ matrix.platform }}
|
||||
|
||||
@ -40,7 +40,7 @@ jobs:
|
||||
- name: Install optional test requirements
|
||||
run: |
|
||||
python -m pip install iopath transformers pyarrow
|
||||
python -m pip install git+https://github.com/facebookresearch/fairscale.git@master
|
||||
python -m pip install git+https://github.com/facebookresearch/fairscale.git@main
|
||||
|
||||
- name: Lint with flake8
|
||||
run: |
|
||||
@ -53,3 +53,8 @@ jobs:
|
||||
- name: Run tests
|
||||
run: |
|
||||
python setup.py test
|
||||
|
||||
- name: Lint with black
|
||||
run: |
|
||||
pip install black
|
||||
black --check . --extend-exclude 'examples|fairseq\/model_parallel\/megatron'
|
||||
|
5
.gitignore
vendored
5
.gitignore
vendored
@ -134,3 +134,8 @@ experimental/*
|
||||
|
||||
# Weights and Biases logs
|
||||
wandb/
|
||||
|
||||
# Hydra artifacts
|
||||
nohup.out
|
||||
multirun
|
||||
outputs
|
||||
|
2
.isort.cfg
Normal file
2
.isort.cfg
Normal file
@ -0,0 +1,2 @@
|
||||
[settings]
|
||||
known_third_party = _cffi_backend,agg_results,aml,bitarray,boto3,botocore,dump_hubert_feature,dynamicconv_cuda,editdistance,faiss,fasttext,feature_utils,ffmpeg,g2p_en,h5py,hydra,hypothesis,indicnlp,inflect,iopath,joblib,kaldi_io,kenlm,libfb,librosa,lightconv_cuda,matplotlib,misc,mmpt,mmpt_cli,model,nltk,npy_append_array,numpy,omegaconf,pandas,pathbuilder,preprocessing,progressbar,pythainlp,random_sequence_shuffler,regex,sacrebleu,sacremoses,scipy,sentencepiece,setuptools,six,sklearn,soundfile,sweep,sweep_wmt_en2de_transformer_big_common,tabulate,torch,torchaudio,tqdm,unidecode,utils,videoreader,wav2vec_cluster_faiss,wget,yaml
|
40
.pre-commit-config.yaml
Normal file
40
.pre-commit-config.yaml
Normal file
@ -0,0 +1,40 @@
|
||||
exclude: 'build|stubs'
|
||||
|
||||
default_language_version:
|
||||
python: python3
|
||||
|
||||
repos:
|
||||
- repo: https://github.com/pre-commit/pre-commit-hooks
|
||||
rev: v4.1.0
|
||||
hooks:
|
||||
- id: trailing-whitespace
|
||||
- id: check-ast
|
||||
- id: check-merge-conflict
|
||||
- id: no-commit-to-branch
|
||||
args: ['--branch=master']
|
||||
- id: check-added-large-files
|
||||
args: ['--maxkb=500']
|
||||
- id: end-of-file-fixer
|
||||
|
||||
- repo: https://github.com/ambv/black
|
||||
rev: 22.1.0
|
||||
hooks:
|
||||
- id: black
|
||||
language_version: python3.8
|
||||
|
||||
- repo: https://gitlab.com/pycqa/flake8
|
||||
rev: 3.9.2
|
||||
hooks:
|
||||
- id: flake8
|
||||
args: [
|
||||
# only error for syntax errors and undefined names
|
||||
"--select=E9,F63,F7,F82",
|
||||
]
|
||||
|
||||
- repo: https://github.com/pycqa/isort
|
||||
rev: 5.10.1
|
||||
hooks:
|
||||
- id: isort
|
||||
exclude: README.md
|
||||
additional_dependencies: [toml]
|
||||
args: ["--profile", "black"]
|
@ -26,3 +26,57 @@ clear and has sufficient instructions to be able to reproduce the issue.
|
||||
By contributing to Facebook AI Research Sequence-to-Sequence Toolkit (fairseq),
|
||||
you agree that your contributions will be licensed under the LICENSE file in
|
||||
the root directory of this source tree.
|
||||
|
||||
## Pre-commit hooks
|
||||
In order to ensure your code lints, there are pre-commit hooks configured in the repository which you can install.
|
||||
After installation, they will automatically run each time you commit.
|
||||
An abbreviated guide is given below; for more information, refer to [the offical pre-commit documentation](https://pre-commit.com/).
|
||||
|
||||
### Installation
|
||||
```
|
||||
pip install pre-commit
|
||||
pre-commit install
|
||||
```
|
||||
|
||||
### Usage
|
||||
Just commit your changes:
|
||||
```
|
||||
git commit -m "My informative commit message"
|
||||
```
|
||||
|
||||
If there was a failure, you will get feedback
|
||||
```
|
||||
[INFO] Initializing environment for https://github.com/PyCQA/flake8.
|
||||
[INFO] Installing environment for https://github.com/pre-commit/pre-commit-hooks.
|
||||
[INFO] Once installed this environment will be reused.
|
||||
[INFO] This may take a few minutes...
|
||||
[INFO] Installing environment for https://github.com/PyCQA/flake8.
|
||||
[INFO] Once installed this environment will be reused.
|
||||
[INFO] This may take a few minutes...
|
||||
Trim Trailing Whitespace.................................................Failed
|
||||
- hook id: trailing-whitespace
|
||||
- exit code: 1
|
||||
- files were modified by this hook
|
||||
Fixing examples/nllb/modeling/wmt15_benchmark/eval_langs2.sh
|
||||
Fix End of Files.........................................................Failed
|
||||
- hook id: end-of-file-fixer
|
||||
- exit code: 1
|
||||
- files were modified by this hook
|
||||
Fixing examples/few_shot/scripts/schedule_jobs_few_shot.py
|
||||
flake8...................................................................Passed
|
||||
```
|
||||
|
||||
Certain hooks modify your files to comply.
|
||||
To include these modifications, you will need to add them (i.e. `git add ...`) and commit again.
|
||||
|
||||
If all is well, you should see something like:
|
||||
```
|
||||
Trim Trailing Whitespace.................................................Passed
|
||||
Fix End of Files.........................................................Passed
|
||||
flake8...................................................................Passed
|
||||
[gshard-fix-ci 8698644e1] Fix lint, add pre-commit hooks
|
||||
10 files changed, 148 insertions(+), 110 deletions(-)
|
||||
create mode 100644 .flake8
|
||||
create mode 100644 .pre-commit-config.yaml
|
||||
rename examples/nllb/modeling/wmt15_benchmark/{eval_langs2.py => eval_langs2.sh} (99%)
|
||||
```
|
||||
|
11
README.md
11
README.md
@ -2,6 +2,7 @@
|
||||
<img src="docs/fairseq_logo.png" width="150">
|
||||
<br />
|
||||
<br />
|
||||
<a href="https://opensource.fb.com/support-ukraine"><img alt="Support Ukraine" src="https://img.shields.io/badge/Support-Ukraine-FFD500?style=flat&labelColor=005BBB" /></a>
|
||||
<a href="https://github.com/pytorch/fairseq/blob/main/LICENSE"><img alt="MIT License" src="https://img.shields.io/badge/license-MIT-blue.svg" /></a>
|
||||
<a href="https://github.com/pytorch/fairseq/releases"><img alt="Latest Release" src="https://img.shields.io/github/release/pytorch/fairseq.svg" /></a>
|
||||
<a href="https://github.com/pytorch/fairseq/actions?query=workflow:build"><img alt="Build Status" src="https://github.com/pytorch/fairseq/workflows/build/badge.svg" /></a>
|
||||
@ -49,8 +50,13 @@ We provide reference implementations of various sequence modeling papers:
|
||||
+ [Cross-lingual Retrieval for Iterative Self-Supervised Training (Tran et al., 2020)](examples/criss/README.md)
|
||||
+ [Deep Transformers with Latent Depth (Li et al., 2020)](examples/latent_depth/README.md)
|
||||
+ [Unsupervised Cross-lingual Representation Learning for Speech Recognition (Conneau et al., 2020)](https://arxiv.org/abs/2006.13979)
|
||||
+ [Self-training and Pre-training are Complementary for Speech Recognition (Xu et al., 2020)](https://arxiv.org/abs/2010.11430)
|
||||
+ [Robust wav2vec 2.0: Analyzing Domain Shift in Self-Supervised Pre-Training (Hsu, et al., 2021)](https://arxiv.org/abs/2104.01027)
|
||||
+ [Unsupervised Speech Recognition (Baevski, et al., 2021)](https://arxiv.org/abs/2105.11084)
|
||||
+ [Simple and Effective Zero-shot Cross-lingual Phoneme Recognition (Xu et al., 2021)](https://arxiv.org/abs/2109.11680)
|
||||
+ [VideoCLIP: Contrastive Pre-training for Zero-shot Video-Text Understanding (Xu et. al., 2021)](https://arxiv.org/pdf/2109.14084.pdf)
|
||||
+ [VLM: Task-agnostic Video-Language Model Pre-training for Video Understanding (Xu et. al., 2021)](https://aclanthology.org/2021.findings-acl.370.pdf)
|
||||
+ [NormFormer: Improved Transformer Pretraining with Extra Normalization (Shleifer et. al, 2021)](examples/normformer/README.md)
|
||||
* **Non-autoregressive Transformers**
|
||||
+ Non-Autoregressive Neural Machine Translation (Gu et al., 2017)
|
||||
+ Deterministic Non-Autoregressive Neural Sequence Modeling by Iterative Refinement (Lee et al. 2018)
|
||||
@ -63,7 +69,9 @@ We provide reference implementations of various sequence modeling papers:
|
||||
</p></details>
|
||||
|
||||
### What's New:
|
||||
|
||||
* December 2021 [Released Direct speech-to-speech translation code](examples/speech_to_speech/README.md)
|
||||
* October 2021 [Released VideoCLIP and VLM models](examples/MMPT/README.md)
|
||||
* October 2021 [Released multilingual finetuned XLSR-53 model](examples/wav2vec/README.md)
|
||||
* September 2021 [`master` branch renamed to `main`](https://github.com/github/renaming).
|
||||
* July 2021 [Released DrNMT code](examples/discriminative_reranking_nmt/README.md)
|
||||
* July 2021 [Released Robust wav2vec 2.0 model](examples/wav2vec/README.md)
|
||||
@ -183,6 +191,7 @@ as well as example training and evaluation commands.
|
||||
|
||||
We also have more detailed READMEs to reproduce results from specific papers:
|
||||
|
||||
* [XLS-R: Self-supervised Cross-lingual Speech Representation Learning at Scale (Babu et al., 2021)](examples/wav2vec/xlsr/README.md)
|
||||
* [Cross-lingual Retrieval for Iterative Self-Supervised Training (Tran et al., 2020)](examples/criss/README.md)
|
||||
* [wav2vec 2.0: A Framework for Self-Supervised Learning of Speech Representations (Baevski et al., 2020)](examples/wav2vec/README.md)
|
||||
* [Unsupervised Quality Estimation for Neural Machine Translation (Fomicheva et al., 2020)](examples/unsupervised_quality_estimation/README.md)
|
||||
|
139
examples/MMPT/.gitignore
vendored
Normal file
139
examples/MMPT/.gitignore
vendored
Normal file
@ -0,0 +1,139 @@
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
|
||||
# C extensions
|
||||
*.so
|
||||
|
||||
# Distribution / packaging
|
||||
.Python
|
||||
build/
|
||||
develop-eggs/
|
||||
dist/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
var/
|
||||
wheels/
|
||||
pip-wheel-metadata/
|
||||
share/python-wheels/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
MANIFEST
|
||||
|
||||
# PyInstaller
|
||||
# Usually these files are written by a python script from a template
|
||||
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||
*.manifest
|
||||
*.spec
|
||||
|
||||
# Installer logs
|
||||
pip-log.txt
|
||||
pip-delete-this-directory.txt
|
||||
|
||||
# Unit test / coverage reports
|
||||
htmlcov/
|
||||
.tox/
|
||||
.nox/
|
||||
.coverage
|
||||
.coverage.*
|
||||
.cache
|
||||
nosetests.xml
|
||||
coverage.xml
|
||||
*.cover
|
||||
*.py,cover
|
||||
.hypothesis/
|
||||
.pytest_cache/
|
||||
|
||||
# Translations
|
||||
*.mo
|
||||
*.pot
|
||||
|
||||
# Django stuff:
|
||||
*.log
|
||||
local_settings.py
|
||||
db.sqlite3
|
||||
db.sqlite3-journal
|
||||
|
||||
# Flask stuff:
|
||||
instance/
|
||||
.webassets-cache
|
||||
|
||||
# Scrapy stuff:
|
||||
.scrapy
|
||||
|
||||
# Sphinx documentation
|
||||
docs/_build/
|
||||
|
||||
# PyBuilder
|
||||
target/
|
||||
|
||||
# Jupyter Notebook
|
||||
.ipynb_checkpoints
|
||||
|
||||
# IPython
|
||||
profile_default/
|
||||
ipython_config.py
|
||||
|
||||
# pyenv
|
||||
.python-version
|
||||
|
||||
# pipenv
|
||||
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
||||
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
||||
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
||||
# install all needed dependencies.
|
||||
#Pipfile.lock
|
||||
|
||||
# PEP 582; used by e.g. github.com/David-OConnor/pyflow
|
||||
__pypackages__/
|
||||
|
||||
# Celery stuff
|
||||
celerybeat-schedule
|
||||
celerybeat.pid
|
||||
|
||||
# SageMath parsed files
|
||||
*.sage.py
|
||||
|
||||
# Environments
|
||||
.env
|
||||
.venv
|
||||
env/
|
||||
venv/
|
||||
ENV/
|
||||
env.bak/
|
||||
venv.bak/
|
||||
|
||||
# Spyder project settings
|
||||
.spyderproject
|
||||
.spyproject
|
||||
|
||||
# Rope project settings
|
||||
.ropeproject
|
||||
|
||||
# mkdocs documentation
|
||||
/site
|
||||
|
||||
# mypy
|
||||
.mypy_cache/
|
||||
.dmypy.json
|
||||
dmypy.json
|
||||
|
||||
# Pyre type checker
|
||||
.pyre/
|
||||
runs
|
||||
data
|
||||
pretrained_models
|
||||
projects/mmfusion_*
|
||||
log_test
|
||||
third-party
|
||||
python_log
|
||||
slurm_snapshot_code
|
||||
lightning_logs
|
||||
demos
|
41
examples/MMPT/CONFIG.md
Normal file
41
examples/MMPT/CONFIG.md
Normal file
@ -0,0 +1,41 @@
|
||||
### Config Files Explained
|
||||
|
||||
Taking `projects/mfmmlm.yaml` for example, which run pretraining using masked frame model (MFM) and masked language model (MLM) on a single BERT:
|
||||
|
||||
```yaml
|
||||
project_dir: mfmmlm # specify the project dir for this baseline.
|
||||
run_task:
|
||||
- how2.yaml # run pretraining on how2 when launching `projects/taskmfmmlm.yaml`
|
||||
- [vtt.yaml, vttcap.yaml, vttqa.yaml, youcook.yaml, youcookcap.yaml, crosstask.yaml, coin.yaml] # run fine-tuning tasks.
|
||||
base_dir: task # a global template folder to specify each training task.
|
||||
task_group:
|
||||
pretrain: # section for pretraining. Most baselines differs in this section.
|
||||
task_list:
|
||||
- how2.yaml # reconfig `projects/task/how2.yaml`
|
||||
dataset:
|
||||
aligner: MFMMLMAligner # overwrite the aligner for MFMMLM training task.
|
||||
model:
|
||||
model_cls: MMFusionMFMMLM # overwrite the model, which constructs negative examples for MFM on-the-fly.
|
||||
loss:
|
||||
loss_cls: MFMMLM # overwrite the loss as MFMMLM, which combines MFM and MLM together.
|
||||
fairseq: # all fairseq args can be expecified under this name.
|
||||
dataset:
|
||||
batch_size: 128
|
||||
finetune: # section for fine-tuning tasks, we don't need to change anything here mostly since we want to see how pretraining can contribute to finetuning.
|
||||
task_list: # specify the list of downstream tasks, e.g., copy `projects/task/vtt.yaml` to `projects/mfmmlm`.
|
||||
- vtt.yaml
|
||||
- vttqa.yaml
|
||||
- youcook.yaml
|
||||
- youcookcap.yaml
|
||||
- crosstask.yaml
|
||||
- coin.yaml
|
||||
test: # section for testing.
|
||||
task_list:
|
||||
- test_vtt.yaml
|
||||
- test_vttqa.yaml
|
||||
- test_youcook.yaml
|
||||
- test_youcookcap.yaml
|
||||
- test_crosstask.yaml
|
||||
- test_crosstask_zs.yaml
|
||||
- test_coin.yaml
|
||||
```
|
34
examples/MMPT/DATASET.md
Normal file
34
examples/MMPT/DATASET.md
Normal file
@ -0,0 +1,34 @@
|
||||
# Dataset
|
||||
|
||||
We understand video data are challenging to download and process. For videos, we provide our preprocessing scripts under `scripts/video_feature_extractor` (deeply adapted from `https://github.com/antoine77340/video_feature_extractor`); for text, we pre-tokenizing scripts under `scripts/text_token_extractor`.
|
||||
|
||||
### S3D Feature Extraction
|
||||
We use pre-trained [S3D](https://github.com/antoine77340/S3D_HowTo100M) for video feature extraction. Please place the models as `pretrained_models/s3d_dict.npy` and `pretrained_models/s3d_howto100m.pth`.
|
||||
|
||||
We implement a `PathBuilder` to automatically track video ids, source video paths to their feature locations (you may need `conda install -c anaconda pandas`). Decoding may need `pip install ffmpeg-python`.
|
||||
|
||||
### Howto100M
|
||||
[Howto100M](https://www.di.ens.fr/willow/research/howto100m/) is a large-scale video pre-training datasets. You may download videos by yourself and run preprocessing of our scripts.
|
||||
|
||||
Several key differences of our preprocessing from existing papers: (1) we use `raw_caption.json` instead of `caption.json` to have pure self-supervision on text (`caption.json` has manual removal of stop words); (2) we remove partially duplicated texts that are originally designed for real-time readability (see `mmpt/processors/dedupprocessor.py`); (3) then we shard video/text features using `SharedTensor` in `mmpt/utils/shardedtensor.py` for fast loading during training (faster than `h5py`).
|
||||
|
||||
#### Steps
|
||||
##### video
|
||||
To extract video features: edit and run `bash scripts/video_feature_extractor/how2/s3d.sh`. (consider to run this on multiple machines; by default, we store features in fp16 to save space and also for faster training).
|
||||
|
||||
Split available video ids as `data/how2/how2_s3d_train.lst` and `data/how2/how2_s3d_val.lst`.
|
||||
|
||||
Lastly, pack video features into `ShardedTensor` using `python scripts/video_feature_extractor/shard_feature.py`.
|
||||
|
||||
##### text
|
||||
Clean captions using `python -m mmpt.processors.dedupprocessor`.
|
||||
|
||||
Tokenize dedupped captions `data/how2/raw_caption_dedup.pkl` into sharded numpy arrays:
|
||||
```
|
||||
python scripts/text_token_extractor/pretokenization.py scripts/text_token_extractor/configs/bert-base-uncased.yaml
|
||||
```
|
||||
|
||||
### Youcook, MSRVTT etc.
|
||||
We use the version of Youcook and MSRVTT come with Howto100M and MILNCE. Please download the data to `data/youcook` and `data/msrvtt` accordingly, you can also check `projects/task/youcook.yaml` and `projects/task/vtt.yaml` etc. in details.
|
||||
We extract features for Youcook, MSRVTT similar to the first step of Howto100M but we read text from meta data directly and perform on-the-fly tokenization.
|
||||
|
166
examples/MMPT/README.md
Normal file
166
examples/MMPT/README.md
Normal file
@ -0,0 +1,166 @@
|
||||
# VideoCLIP and VLM
|
||||
|
||||
You just find this toolkit for multimodal video understanding! It contains implementation of two recent multi-modal video understanding papers [VideoCLIP](https://arxiv.org/pdf/2109.14084.pdf) (EMNLP, 2021) and [VLM](https://aclanthology.org/2021.findings-acl.370.pdf) (ACL Findings, 2021), along with high-performance toolkits that are typically lacking in existing codebase. The toolkit is desigend to contain generic performance-tuned components that can be potentially adapted to other frameworks (we initially use fairseq).
|
||||
|
||||
VideoCLIP is a contrastive learning model for zero-shot transfer to retrieval/classification/sequence labeling style tasks.
|
||||
|
||||
<img src="videoclip.png" width="350" class="center">
|
||||
|
||||
VLM is a masked language model style pre-training using only one encoder with masked modality model (MMM) for retrieval/generation/sequence labeling style tasks.
|
||||
|
||||
<img src="vlm.png" width="350" class="center">
|
||||
|
||||
### News
|
||||
[Oct. 2021] Initial release of implementation for the following papers:
|
||||
[VideoCLIP: Contrastive Pre-training for Zero-shot Video-Text Understanding](https://arxiv.org/pdf/2109.14084.pdf) (Xu et. al., EMNLP 2021)
|
||||
[VLM: Task-agnostic Video-Language Model Pre-training for Video Understanding](https://aclanthology.org/2021.findings-acl.370.pdf) (Xu et. al., ACL Findings 2021)
|
||||
|
||||
|
||||
### Installation
|
||||
We aim to minimize the dependency of this repo on other packages.
|
||||
We use fairseq as the main trainer (no models/datasets dependency on fairseq. We will support other trainer in future):
|
||||
```
|
||||
git clone https://github.com/pytorch/fairseq
|
||||
cd fairseq
|
||||
pip install -e . # also optionally follow fairseq README for apex installation for fp16 training.
|
||||
export MKL_THREADING_LAYER=GNU # fairseq may need this for numpy.
|
||||
```
|
||||
|
||||
Then install this toolkit:
|
||||
```
|
||||
cd examples/MMPT # MMPT can be in any folder, not necessarily under fairseq/examples.
|
||||
pip install -e .
|
||||
```
|
||||
|
||||
The code is developed under Python=3.8.8, Pytorch=1.8, cuda=11.0 with fairseq=1.0.0a0+af0389f and tested under Python=3.8.8 pytorch=1.9 cuda=11.0 fairseq=1.0.0a0+8e7bc73 during code release.
|
||||
Most models require `transformers==3.4` for API compatibility `pip install transformers==3.4`.
|
||||
In addition, some downstream tasks may need `conda install pandas`.
|
||||
|
||||
|
||||
### Usage
|
||||
#### Download Checkpoints
|
||||
We use pre-trained [S3D](https://github.com/antoine77340/S3D_HowTo100M) for video feature extraction. Please place the models as `pretrained_models/s3d_dict.npy` and `pretrained_models/s3d_howto100m.pth`.
|
||||
|
||||
Download VideoCLIP checkpoint `https://dl.fbaipublicfiles.com/MMPT/retri/videoclip/checkpoint_best.pt` to `runs/retri/videoclip` or VLM checkpoint `https://dl.fbaipublicfiles.com/MMPT/mtm/vlm/checkpoint_best.pt` to `runs/mtm/vlm`.
|
||||
|
||||
#### Demo of Inference
|
||||
run `python locallaunch.py projects/retri/videoclip.yaml --dryrun` to get all `.yaml`s for VideoCLIP.
|
||||
|
||||
```python
|
||||
import torch
|
||||
|
||||
from mmpt.models import MMPTModel
|
||||
|
||||
|
||||
model, tokenizer, aligner = MMPTModel.from_pretrained(
|
||||
"projects/retri/videoclip/how2.yaml")
|
||||
|
||||
model.eval()
|
||||
|
||||
|
||||
# B, T, FPS, H, W, C (VideoCLIP is trained on 30 fps of s3d)
|
||||
video_frames = torch.randn(1, 2, 30, 224, 224, 3)
|
||||
caps, cmasks = aligner._build_text_seq(
|
||||
tokenizer("some text", add_special_tokens=False)["input_ids"]
|
||||
)
|
||||
|
||||
caps, cmasks = caps[None, :], cmasks[None, :] # bsz=1
|
||||
|
||||
with torch.no_grad():
|
||||
output = model(video_frames, caps, cmasks, return_score=True)
|
||||
print(output["score"]) # dot-product
|
||||
```
|
||||
|
||||
#### Data Preparation
|
||||
See [dataset](DATASET.md) for each dataset.
|
||||
|
||||
#### Global Config for Training Pipeline
|
||||
We organize a global config file for a training/testing pipeline under projects (see a detailed [explanation](CONFIG.md)). For example, VideoCLIP in `projects/retri/videoclip.yaml` and VLM is in `projects/mtm/vlm.yaml`.
|
||||
|
||||
We wrap all cmds into `locallaunch.py` and `mmpt_cli/localjob.py`. You can check concrete cmds by `--dryrun` and then drop it for actual run.
|
||||
|
||||
First, run `python locallaunch.py projects/retri/videoclip.yaml --dryrun` will generate configs for all configs of pre-training, zero-shot evaluation, fine-tuning and testing, for VideoCLIP under `projects/retri/videoclip`.
|
||||
|
||||
Then each (either training or evaluation) process will be configed by a concrete config file (we save all complex arguments into the concrete config file for reproducibility, including fairseq args). For example, run zero-shot evaluation on youcook,
|
||||
```
|
||||
python locallaunch.py projects/retri/videoclip/test_youcook_zs.yaml --jobtype local_predict # zero-shot evaluation.
|
||||
python locallaunch.py projects/retri/videoclip/youcook_videoclip.yaml --jobtype local_single --dryrun # fine-tuning: use --dryrun to check cmds and drop it to make an actual run; local_small will run on two gpus (as in paper).
|
||||
python locallaunch.py projects/retri/videoclip/test_youcook_videoclip.yaml --jobtype local_predict # testing on fine-tuned model.
|
||||
```
|
||||
|
||||
Pretraining can be run as:
|
||||
```
|
||||
python locallaunch.py projects/retri/videoclip/how2.yaml --jobtype local_single --dryrun # check then drop dryrun; paper is ran on local_big as 8 gpus.
|
||||
```
|
||||
You may need to change `--jobtype`, check/extend `LocalJob` in `mmpt_cli/localjob.py` for multi-gpu/multi-node pre-training.
|
||||
|
||||
The detailed instructions of pretraining and fine-tuning can be found at [pretraining instruction](pretraining.md) and [finetuning instruction](endtask.md).
|
||||
|
||||
|
||||
### Development
|
||||
Several components of this toolkit can be re-used for future research (and also our ongoing research).
|
||||
|
||||
#### Framework Wrapper
|
||||
We currently only support fairseq, but most components can be easily fit into other frameworks like huggingface. This repo is a `--user-dir` of fairseq with fairseq wrapper. For example, `mmpt/tasks` includes a `FairseqMMTTask`, which manages `mmpt/datasets` with `FairseqDataset`, `mmpt/models` with `FairseqModel`, `mmpt/losses` with `FairseqCriterion`.
|
||||
|
||||
#### Processors
|
||||
**Multi**modal research introduces the complexity on modality alignment from different input sources to losses. Inspired by [MMF](https://github.com/facebookresearch/mmf), this toolkit leverages `mmpt/processors` to handle various needs of data preprocessing and loading, **alleviating** the needs of multiple `torch.data.utils.Dataset` (that can be tricky for ablation study).
|
||||
Processors can also be decoupled from `torch.data.utils.Dataset` for offline preprocessing instead of on-the-fly data preprocessing.
|
||||
|
||||
We decouple a `mmpt.MMDataset` as 3 types of processors: `MetaProcessor`, `VideoProcessor`, `TextProcessor` and `Aligner`. They can be configed in `dataset` field of a config file (e.g., see `projects/task/how2.yaml`).
|
||||
`MetaProcessor` is used to load the meta data about a dataset, aka, all video_ids of how2 dataset.
|
||||
`VideoProcessor` is used to load the video features about a dataset. For example, S3D features for each second of a video.
|
||||
`TextProcessor` is used to load the text (feature). For example, BERT pre-tokenized text clips for how2 dataset (with `start`s, `end`s of timestamps and `cap` for `token_ids`).
|
||||
`Aligner` is the core class for different baselines that prepares the training data. For example, sampling a clip, masking tokens for MLM, etc.
|
||||
|
||||
#### Performance-tuned Components
|
||||
To speed up pre-training, this toolkit uses sharded features stored in mmaped numpy, backed by `ShardedTensor` in `mmpt/utils/shardedtensor.py` (adopted from MARGE paper). This reduces the loads of IO for multi-GPU training without loading all features for a video into the memory each time and `ShardedTensor` ensure features are stored in continuous disk space for near random access. This is used for both How2 video features and texts in `mmpt/processors/how2processor.py`.
|
||||
|
||||
|
||||
### Citation
|
||||
If this codebase is useful for your work, please cite the following papers:
|
||||
|
||||
```BibTeX
|
||||
@inproceedings{xu-etal-2021-videoclip,
|
||||
title = "{VideoCLIP}: Contrastive Pre-training for\\Zero-shot Video-Text Understanding",
|
||||
author = "Xu, Hu and
|
||||
Ghosh, Gargi and
|
||||
Huang, Po-Yao and
|
||||
Okhonko, Dmytro and
|
||||
Aghajanyan, Armen and
|
||||
Metze, Florian and
|
||||
Zettlemoyer, Luke and
|
||||
Feichtenhofer, Christoph",
|
||||
booktitle = "Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing (EMNLP)",
|
||||
month = nov,
|
||||
year = "2021",
|
||||
address = "Online",
|
||||
publisher = "Association for Computational Linguistics",
|
||||
}
|
||||
|
||||
@inproceedings{xu-etal-2021-vlm,
|
||||
title = "{VLM}: Task-agnostic Video-Language Model Pre-training for Video Understanding",
|
||||
author = "Xu, Hu and
|
||||
Ghosh, Gargi and
|
||||
Huang, Po-Yao and
|
||||
Arora, Prahal and
|
||||
Aminzadeh, Masoumeh and
|
||||
Feichtenhofer, Christoph and
|
||||
Metze, Florian and
|
||||
Zettlemoyer, Luke",
|
||||
booktitle = "Findings of the Association for Computational Linguistics: ACL-IJCNLP 2021",
|
||||
month = aug,
|
||||
year = "2021",
|
||||
address = "Online",
|
||||
publisher = "Association for Computational Linguistics",
|
||||
url = "https://aclanthology.org/2021.findings-acl.370",
|
||||
doi = "10.18653/v1/2021.findings-acl.370",
|
||||
pages = "4227--4239",
|
||||
}
|
||||
```
|
||||
|
||||
### Bug Reports
|
||||
This repo is in its initial stage, welcome bug reports to huxu@fb.com
|
||||
|
||||
### Copyright
|
||||
The majority of Multimodal Pre-training (MMPT) is licensed under CC-BY-NC, however portions of the project are available under separate license terms: Evaluation Codes/Models: Howto100M and HuggingFace Transformers are licensed under the Apache2.0 license; COIN and NLG-eval are licensed under the MIT license; CrossTask is licensed under the BSD-3; DiDeMo is licensed under the BSD-2 license.
|
41
examples/MMPT/endtask.md
Normal file
41
examples/MMPT/endtask.md
Normal file
@ -0,0 +1,41 @@
|
||||
# Zero-shot Transfer and Finetuning
|
||||
|
||||
(If you are new to the ideas of `mmpt.processors`, see [README](README.md) first.)
|
||||
All finetuning datasets (specifically `processors`) are defined in `mmpt.processors.dsprocessor`.
|
||||
Given the complexity of different types of finetuning tasks, each task may have their own meta/video/text/aligner processors and `mmpt/evaluators/{Predictor,Metric}`.
|
||||
|
||||
### Tasks
|
||||
|
||||
Currently, we support 5 end datasets: `MSRVTT`, `Youcook`, `COIN`, `Crosstask` and `DiDeMo` with the following tasks:
|
||||
text-video retrieval: `MSRVTT`, `Youcook`, `DiDeMo`;
|
||||
video captioning: `Youcook`;
|
||||
Video Question and Answering: `MSRVTT-QA`.
|
||||
|
||||
To add your own dataset, you can specify the corresponding processors and config them in the `dataset` field of a config file, such as `projects/task/vtt.yaml`.
|
||||
|
||||
### Zero-shot Transfer (no Training)
|
||||
Zero-shot transfer will run the pre-trained model (e.g., VideoCLIP) directly on testing data. Configs with pattern: `projects/task/*_zs_*.yaml` are dedicated for zero-shot transfer.
|
||||
|
||||
### Fine-tuning
|
||||
|
||||
The training of a downstream task is similar to pretraining, execept you may need to specify the `restore_file` in `fairseq.checkpoint` and reset optimizers, see `projects/task/ft.yaml` that is included by `projects/task/vtt.yaml`.
|
||||
|
||||
We typically do finetuning on 2 gpus (`local_small`).
|
||||
|
||||
### Testing
|
||||
For each finetuning dataset, you may need to specify a testing config, similar to `projects/task/test_vtt.yaml`.
|
||||
|
||||
We define `mmpt.evaluators.Predictor` for different types of prediction. For example, `MSRVTT` and `Youcook` are video-retrieval tasks and expecting to use `RetrievalPredictor`. You may need to define your new type of predictors and specify that in `predictor` field of a testing config.
|
||||
|
||||
Each task may also have their own metric for evaluation. This can be created in `mmpt.evaluators.Metric` and specified in the `metric` field of a testing config.
|
||||
|
||||
Launching a testing is as simple as training by specifying the path of a testing config:
|
||||
```python locallaunch.py projects/mfmmlm/test_vtt.yaml```
|
||||
Testing will be launched locally by default since prediction is computationally less expensive.
|
||||
|
||||
### Third-party Libraries
|
||||
We list the following finetuning tasks that require third-party libraries.
|
||||
|
||||
Youcook captioning: `https://github.com/Maluuba/nlg-eval`
|
||||
|
||||
CrossTask: `https://github.com/DmZhukov/CrossTask`'s `dp` under `third-party/CrossTask` (`python setup.py build_ext --inplace`)
|
148
examples/MMPT/locallaunch.py
Normal file
148
examples/MMPT/locallaunch.py
Normal file
@ -0,0 +1,148 @@
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
#
|
||||
# This source code is licensed under the MIT license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
import argparse
|
||||
import os
|
||||
|
||||
from omegaconf import OmegaConf
|
||||
|
||||
from mmpt.utils import recursive_config, overwrite_dir
|
||||
from mmpt_cli.localjob import LocalJob
|
||||
|
||||
|
||||
class JobLauncher(object):
|
||||
JOB_CONFIG = {
|
||||
"local": LocalJob,
|
||||
}
|
||||
|
||||
def __init__(self, yaml_file):
|
||||
self.yaml_file = yaml_file
|
||||
job_key = "local"
|
||||
|
||||
if yaml_file.endswith(".yaml"):
|
||||
config = recursive_config(yaml_file)
|
||||
if config.task_type is not None:
|
||||
job_key = config.task_type.split("_")[0]
|
||||
else:
|
||||
raise ValueError("unknown extension of job file:", yaml_file)
|
||||
self.job_key = job_key
|
||||
|
||||
def __call__(self, job_type=None, dryrun=False):
|
||||
if job_type is not None:
|
||||
self.job_key = job_type.split("_")[0]
|
||||
print("[JobLauncher] job_key", self.job_key)
|
||||
job = JobLauncher.JOB_CONFIG[self.job_key](
|
||||
self.yaml_file, job_type=job_type, dryrun=dryrun)
|
||||
return job.submit()
|
||||
|
||||
|
||||
class Pipeline(object):
|
||||
"""a job that loads yaml config."""
|
||||
|
||||
def __init__(self, fn):
|
||||
"""
|
||||
load a yaml config of a job and save generated configs as yaml for each task.
|
||||
return: a list of files to run as specified by `run_task`.
|
||||
"""
|
||||
if fn.endswith(".py"):
|
||||
# a python command.
|
||||
self.backend = "python"
|
||||
self.run_yamls = [fn]
|
||||
return
|
||||
|
||||
job_config = recursive_config(fn)
|
||||
if job_config.base_dir is None: # single file job config.
|
||||
self.run_yamls = [fn]
|
||||
return
|
||||
|
||||
self.project_dir = os.path.join("projects", job_config.project_dir)
|
||||
self.run_dir = os.path.join("runs", job_config.project_dir)
|
||||
|
||||
if job_config.run_task is not None:
|
||||
run_yamls = []
|
||||
for stage in job_config.run_task:
|
||||
# each stage can have multiple tasks running in parallel.
|
||||
if OmegaConf.is_list(stage):
|
||||
stage_yamls = []
|
||||
for task_file in stage:
|
||||
stage_yamls.append(
|
||||
os.path.join(self.project_dir, task_file))
|
||||
run_yamls.append(stage_yamls)
|
||||
else:
|
||||
run_yamls.append(os.path.join(self.project_dir, stage))
|
||||
self.run_yamls = run_yamls
|
||||
configs_to_save = self._overwrite_task(job_config)
|
||||
self._save_configs(configs_to_save)
|
||||
|
||||
def __getitem__(self, idx):
|
||||
yaml_files = self.run_yamls[idx]
|
||||
if isinstance(yaml_files, list):
|
||||
return [JobLauncher(yaml_file) for yaml_file in yaml_files]
|
||||
return [JobLauncher(yaml_files)]
|
||||
|
||||
def __len__(self):
|
||||
return len(self.run_yamls)
|
||||
|
||||
def _save_configs(self, configs_to_save: dict):
|
||||
# save
|
||||
os.makedirs(self.project_dir, exist_ok=True)
|
||||
for config_file in configs_to_save:
|
||||
config = configs_to_save[config_file]
|
||||
print("saving", config_file)
|
||||
OmegaConf.save(config=config, f=config_file)
|
||||
|
||||
def _overwrite_task(self, job_config):
|
||||
configs_to_save = {}
|
||||
self.base_project_dir = os.path.join("projects", job_config.base_dir)
|
||||
self.base_run_dir = os.path.join("runs", job_config.base_dir)
|
||||
|
||||
for config_sets in job_config.task_group:
|
||||
overwrite_config = job_config.task_group[config_sets]
|
||||
if (
|
||||
overwrite_config.task_list is None
|
||||
or len(overwrite_config.task_list) == 0
|
||||
):
|
||||
print(
|
||||
"[warning]",
|
||||
job_config.task_group,
|
||||
"has no task_list specified.")
|
||||
# we don't want this added to a final config.
|
||||
task_list = overwrite_config.pop("task_list", None)
|
||||
for config_file in task_list:
|
||||
config_file_path = os.path.join(
|
||||
self.base_project_dir, config_file)
|
||||
config = recursive_config(config_file_path)
|
||||
# overwrite it.
|
||||
if overwrite_config:
|
||||
config = OmegaConf.merge(config, overwrite_config)
|
||||
overwrite_dir(config, self.run_dir, basedir=self.base_run_dir)
|
||||
save_file_path = os.path.join(self.project_dir, config_file)
|
||||
configs_to_save[save_file_path] = config
|
||||
return configs_to_save
|
||||
|
||||
|
||||
def main(args):
|
||||
job_type = args.jobtype if args.jobtype else None
|
||||
# parse multiple pipelines.
|
||||
pipelines = [Pipeline(fn) for fn in args.yamls.split(",")]
|
||||
|
||||
for pipe_id, pipeline in enumerate(pipelines):
|
||||
if not hasattr(pipeline, "project_dir"):
|
||||
for job in pipeline[0]:
|
||||
job(job_type=job_type, dryrun=args.dryrun)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("yamls", type=str)
|
||||
parser.add_argument(
|
||||
"--dryrun",
|
||||
action="store_true",
|
||||
help="run config and prepare to submit without launch the job.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--jobtype", type=str, default="",
|
||||
help="force to run jobs as specified.")
|
||||
args = parser.parse_args()
|
||||
main(args)
|
12
examples/MMPT/mmpt/__init__.py
Normal file
12
examples/MMPT/mmpt/__init__.py
Normal file
@ -0,0 +1,12 @@
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
#
|
||||
# This source code is licensed under the MIT license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
try:
|
||||
# fairseq user dir
|
||||
from .datasets import FairseqMMDataset
|
||||
from .losses import FairseqCriterion
|
||||
from .models import FairseqMMModel
|
||||
from .tasks import FairseqMMTask
|
||||
except ImportError:
|
||||
pass
|
10
examples/MMPT/mmpt/datasets/__init__.py
Normal file
10
examples/MMPT/mmpt/datasets/__init__.py
Normal file
@ -0,0 +1,10 @@
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
#
|
||||
# This source code is licensed under the MIT license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
from .mmdataset import *
|
||||
|
||||
try:
|
||||
from .fairseqmmdataset import *
|
||||
except ImportError:
|
||||
pass
|
57
examples/MMPT/mmpt/datasets/fairseqmmdataset.py
Normal file
57
examples/MMPT/mmpt/datasets/fairseqmmdataset.py
Normal file
@ -0,0 +1,57 @@
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
#
|
||||
# This source code is licensed under the MIT license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
"""
|
||||
TODO (huxu): fairseq wrapper class for all dataset you defined: mostly MMDataset.
|
||||
"""
|
||||
|
||||
from collections import OrderedDict
|
||||
|
||||
from torch.utils.data import Dataset
|
||||
from torch.utils.data.dataloader import default_collate
|
||||
from fairseq.data import FairseqDataset, data_utils
|
||||
|
||||
|
||||
class FairseqMMDataset(FairseqDataset):
|
||||
"""
|
||||
A wrapper class for MMDataset for fairseq.
|
||||
"""
|
||||
|
||||
def __init__(self, mmdataset):
|
||||
if not isinstance(mmdataset, Dataset):
|
||||
raise TypeError("mmdataset must be of type `torch.utils.data.dataset`.")
|
||||
self.mmdataset = mmdataset
|
||||
|
||||
def set_epoch(self, epoch, **unused):
|
||||
super().set_epoch(epoch)
|
||||
self.epoch = epoch
|
||||
|
||||
def __getitem__(self, idx):
|
||||
with data_utils.numpy_seed(43211, self.epoch, idx):
|
||||
return self.mmdataset[idx]
|
||||
|
||||
def __len__(self):
|
||||
return len(self.mmdataset)
|
||||
|
||||
def collater(self, samples):
|
||||
if hasattr(self.mmdataset, "collator"):
|
||||
return self.mmdataset.collator(samples)
|
||||
if len(samples) == 0:
|
||||
return {}
|
||||
if isinstance(samples[0], dict):
|
||||
batch = OrderedDict()
|
||||
for key in samples[0]:
|
||||
if samples[0][key] is not None:
|
||||
batch[key] = default_collate([sample[key] for sample in samples])
|
||||
return batch
|
||||
else:
|
||||
return default_collate(samples)
|
||||
|
||||
def size(self, index):
|
||||
"""dummy implementation: we don't use --max-tokens"""
|
||||
return 1
|
||||
|
||||
def num_tokens(self, index):
|
||||
"""dummy implementation: we don't use --max-tokens"""
|
||||
return 1
|
111
examples/MMPT/mmpt/datasets/mmdataset.py
Normal file
111
examples/MMPT/mmpt/datasets/mmdataset.py
Normal file
@ -0,0 +1,111 @@
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
#
|
||||
# This source code is licensed under the MIT license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
|
||||
import torch
|
||||
|
||||
from collections import OrderedDict
|
||||
|
||||
from torch.utils.data import Dataset
|
||||
from torch.utils.data.dataloader import default_collate
|
||||
|
||||
from ..utils import set_seed
|
||||
|
||||
|
||||
class MMDataset(Dataset):
|
||||
"""
|
||||
A generic multi-modal dataset.
|
||||
Args:
|
||||
`meta_processor`: a meta processor,
|
||||
handling loading meta data and return video_id and text_id.
|
||||
`video_processor`: a video processor,
|
||||
handling e.g., decoding, loading .np files.
|
||||
`text_processor`: a text processor,
|
||||
handling e.g., tokenization.
|
||||
`aligner`: combine the video and text feature
|
||||
as one training example.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
meta_processor,
|
||||
video_processor,
|
||||
text_processor,
|
||||
align_processor,
|
||||
):
|
||||
self.split = meta_processor.split
|
||||
self.meta_processor = meta_processor
|
||||
self.video_processor = video_processor
|
||||
self.text_processor = text_processor
|
||||
self.align_processor = align_processor
|
||||
|
||||
def __len__(self):
|
||||
return len(self.meta_processor)
|
||||
|
||||
def __getitem__(self, idx):
|
||||
if self.split == "test":
|
||||
set_seed(idx)
|
||||
video_id, text_id = self.meta_processor[idx]
|
||||
video_feature = self.video_processor(video_id)
|
||||
text_feature = self.text_processor(text_id)
|
||||
output = self.align_processor(video_id, video_feature, text_feature)
|
||||
# TODO (huxu): the following is for debug purpose.
|
||||
output.update({"idx": idx})
|
||||
return output
|
||||
|
||||
def collater(self, samples):
|
||||
"""This collator is deprecated.
|
||||
set self.collator = MMDataset.collater.
|
||||
see collator in FairseqMMDataset.
|
||||
"""
|
||||
|
||||
if len(samples) == 0:
|
||||
return {}
|
||||
if isinstance(samples[0], dict):
|
||||
batch = OrderedDict()
|
||||
for key in samples[0]:
|
||||
if samples[0][key] is not None:
|
||||
batch[key] = default_collate(
|
||||
[sample[key] for sample in samples])
|
||||
# if torch.is_tensor(batch[key]):
|
||||
# print(key, batch[key].size())
|
||||
# else:
|
||||
# print(key, len(batch[key]))
|
||||
return batch
|
||||
else:
|
||||
return default_collate(samples)
|
||||
|
||||
def print_example(self, output):
|
||||
print("[one example]", output["video_id"])
|
||||
if (
|
||||
hasattr(self.align_processor, "subsampling")
|
||||
and self.align_processor.subsampling is not None
|
||||
and self.align_processor.subsampling > 1
|
||||
):
|
||||
for key in output:
|
||||
if torch.is_tensor(output[key]):
|
||||
output[key] = output[key][0]
|
||||
|
||||
# search tokenizer to translate ids back.
|
||||
tokenizer = None
|
||||
if hasattr(self.text_processor, "tokenizer"):
|
||||
tokenizer = self.text_processor.tokenizer
|
||||
elif hasattr(self.align_processor, "tokenizer"):
|
||||
tokenizer = self.align_processor.tokenizer
|
||||
if tokenizer is not None:
|
||||
caps = output["caps"].tolist()
|
||||
if isinstance(caps[0], list):
|
||||
caps = caps[0]
|
||||
print("caps", tokenizer.decode(caps))
|
||||
print("caps", tokenizer.convert_ids_to_tokens(caps))
|
||||
|
||||
for key, value in output.items():
|
||||
if torch.is_tensor(value):
|
||||
if len(value.size()) >= 3: # attention_mask.
|
||||
print(key, value.size())
|
||||
print(key, "first", value[0, :, :])
|
||||
print(key, "last", value[-1, :, :])
|
||||
else:
|
||||
print(key, value)
|
||||
print("[end of one example]")
|
13
examples/MMPT/mmpt/evaluators/__init__.py
Normal file
13
examples/MMPT/mmpt/evaluators/__init__.py
Normal file
@ -0,0 +1,13 @@
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
#
|
||||
# This source code is licensed under the MIT license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
from .metric import *
|
||||
from .evaluator import *
|
||||
|
||||
|
||||
# experimental.
|
||||
try:
|
||||
from .expmetric import *
|
||||
except ImportError:
|
||||
pass
|
54
examples/MMPT/mmpt/evaluators/evaluator.py
Normal file
54
examples/MMPT/mmpt/evaluators/evaluator.py
Normal file
@ -0,0 +1,54 @@
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
#
|
||||
# This source code is licensed under the MIT license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
import os
|
||||
import glob
|
||||
import numpy as np
|
||||
|
||||
from . import metric as metric_path
|
||||
from . import predictor as predictor_path
|
||||
|
||||
|
||||
class Evaluator(object):
|
||||
"""
|
||||
perform evaluation on a single (downstream) task.
|
||||
make this both offline and online.
|
||||
TODO(huxu) saving evaluation results.
|
||||
"""
|
||||
|
||||
def __init__(self, config, eval_dataloader=None):
|
||||
if config.metric is None:
|
||||
raise ValueError("config.metric is", config.metric)
|
||||
metric_cls = getattr(metric_path, config.metric)
|
||||
self.metric = metric_cls(config)
|
||||
if config.predictor is None:
|
||||
raise ValueError("config.predictor is", config.predictor)
|
||||
predictor_cls = getattr(predictor_path, config.predictor)
|
||||
self.predictor = predictor_cls(config)
|
||||
self.eval_dataloader = eval_dataloader
|
||||
|
||||
def __call__(self):
|
||||
try:
|
||||
print(self.predictor.pred_dir)
|
||||
for pred_file in glob.glob(
|
||||
self.predictor.pred_dir + "/*_merged.npy"):
|
||||
outputs = np.load(pred_file)
|
||||
results = self.metric.compute_metrics(outputs)
|
||||
self.metric.print_computed_metrics(results)
|
||||
|
||||
outputs = np.load(os.path.join(
|
||||
self.predictor.pred_dir, "merged.npy"))
|
||||
results = self.metric.compute_metrics(outputs)
|
||||
return {"results": results, "metric": self.metric}
|
||||
except FileNotFoundError:
|
||||
print("\n[missing]", self.predictor.pred_dir)
|
||||
return {}
|
||||
|
||||
def evaluate(self, model, eval_dataloader=None, output_file="merged"):
|
||||
if eval_dataloader is None:
|
||||
eval_dataloader = self.eval_dataloader
|
||||
outputs = self.predictor.predict_loop(
|
||||
model, eval_dataloader, output_file)
|
||||
results = self.metric.compute_metrics(**outputs)
|
||||
return results
|
313
examples/MMPT/mmpt/evaluators/metric.py
Normal file
313
examples/MMPT/mmpt/evaluators/metric.py
Normal file
@ -0,0 +1,313 @@
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
#
|
||||
# This source code is licensed under the MIT license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
|
||||
import numpy as np
|
||||
import json
|
||||
|
||||
|
||||
class Metric(object):
|
||||
def __init__(self, config, metric_names):
|
||||
self.metric_names = metric_names
|
||||
|
||||
def best_metric(self, metric):
|
||||
return metric[self.metric_names[0]]
|
||||
|
||||
def save_metrics(self, fn, metrics):
|
||||
with open(fn, "w") as fw:
|
||||
json.dump(fw, metrics)
|
||||
|
||||
def print_computed_metrics(self, metrics):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class RetrievalMetric(Metric):
|
||||
"""
|
||||
this is modified from `howto100m/metrics.py`.
|
||||
History of changes:
|
||||
refactor as a class.
|
||||
add metric_key in __init__
|
||||
"""
|
||||
|
||||
def __init__(self, config, metric_names=["R1", "R5", "R10", "MR"]):
|
||||
super().__init__(config, metric_names)
|
||||
self.error = False # TODO(huxu): add to config to print error.
|
||||
|
||||
def compute_metrics(self, outputs, texts, **kwargs):
|
||||
x = outputs
|
||||
sx = np.sort(-x, axis=1)
|
||||
d = np.diag(-x)
|
||||
d = d[:, np.newaxis]
|
||||
ind = sx - d
|
||||
ind = np.where(ind == 0)
|
||||
ind = ind[1]
|
||||
metrics = {}
|
||||
metrics["R1"] = float(np.sum(ind == 0)) / len(ind)
|
||||
metrics["R5"] = float(np.sum(ind < 5)) / len(ind)
|
||||
metrics["R10"] = float(np.sum(ind < 10)) / len(ind)
|
||||
metrics["MR"] = np.median(ind) + 1
|
||||
|
||||
max_idx = np.argmax(outputs, axis=1)
|
||||
if self.error:
|
||||
# print top-20 errors.
|
||||
error = []
|
||||
for ex_idx in range(20):
|
||||
error.append((texts[ex_idx], texts[max_idx[ex_idx]]))
|
||||
metrics["error"] = error
|
||||
return metrics
|
||||
|
||||
def print_computed_metrics(self, metrics):
|
||||
r1 = metrics["R1"]
|
||||
r5 = metrics["R5"]
|
||||
r10 = metrics["R10"]
|
||||
mr = metrics["MR"]
|
||||
print(
|
||||
"R@1: {:.4f} - R@5: {:.4f} - R@10: {:.4f} - Median R: {}".format(
|
||||
r1, r5, r10, mr
|
||||
)
|
||||
)
|
||||
if "error" in metrics:
|
||||
print(metrics["error"])
|
||||
|
||||
|
||||
class DiDeMoMetric(Metric):
|
||||
"""
|
||||
History of changes:
|
||||
python 2.x to python 3.x.
|
||||
merge utils.py into eval to save one file.
|
||||
reference: https://github.com/LisaAnne/LocalizingMoments/blob/master/utils/eval.py
|
||||
Code to evaluate your results on the DiDeMo dataset.
|
||||
"""
|
||||
def __init__(self, config, metric_names=["rank1", "rank5", "miou"]):
|
||||
super().__init__(config, metric_names)
|
||||
|
||||
def compute_metrics(self, outputs, targets, **kwargs):
|
||||
assert len(outputs) == len(targets)
|
||||
rank1, rank5, miou = self._eval_predictions(outputs, targets)
|
||||
metrics = {
|
||||
"rank1": rank1,
|
||||
"rank5": rank5,
|
||||
"miou": miou
|
||||
}
|
||||
return metrics
|
||||
|
||||
def print_computed_metrics(self, metrics):
|
||||
rank1 = metrics["rank1"]
|
||||
rank5 = metrics["rank5"]
|
||||
miou = metrics["miou"]
|
||||
# print("Average rank@1: %f" % rank1)
|
||||
# print("Average rank@5: %f" % rank5)
|
||||
# print("Average iou: %f" % miou)
|
||||
|
||||
print(
|
||||
"Average rank@1: {:.4f} Average rank@5: {:.4f} Average iou: {:.4f}".format(
|
||||
rank1, rank5, miou
|
||||
)
|
||||
)
|
||||
|
||||
def _iou(self, pred, gt):
|
||||
intersection = max(0, min(pred[1], gt[1]) + 1 - max(pred[0], gt[0]))
|
||||
union = max(pred[1], gt[1]) + 1 - min(pred[0], gt[0])
|
||||
return float(intersection)/union
|
||||
|
||||
def _rank(self, pred, gt):
|
||||
return pred.index(tuple(gt)) + 1
|
||||
|
||||
def _eval_predictions(self, segments, data):
|
||||
'''
|
||||
Inputs:
|
||||
segments: For each item in the ground truth data, rank possible video segments given the description and video.
|
||||
In DiDeMo, there are 21 posible moments extracted for each video so the list of video segments will be of length 21.
|
||||
The first video segment should be the video segment that best corresponds to the text query.
|
||||
There are 4180 sentence in the validation data, so when evaluating a model on the val dataset,
|
||||
segments should be a list of lenght 4180, and each item in segments should be a list of length 21.
|
||||
data: ground truth data
|
||||
'''
|
||||
average_ranks = []
|
||||
average_iou = []
|
||||
for s, d in zip(segments, data):
|
||||
pred = s[0]
|
||||
ious = [self._iou(pred, t) for t in d['times']]
|
||||
average_iou.append(np.mean(np.sort(ious)[-3:]))
|
||||
ranks = [self._rank(s, t) for t in d['times'] if tuple(t) in s] # if t in s] is added for s, e not in prediction.
|
||||
average_ranks.append(np.mean(np.sort(ranks)[:3]))
|
||||
rank1 = np.sum(np.array(average_ranks) <= 1)/float(len(average_ranks))
|
||||
rank5 = np.sum(np.array(average_ranks) <= 5)/float(len(average_ranks))
|
||||
miou = np.mean(average_iou)
|
||||
|
||||
# print("Average rank@1: %f" % rank1)
|
||||
# print("Average rank@5: %f" % rank5)
|
||||
# print("Average iou: %f" % miou)
|
||||
return rank1, rank5, miou
|
||||
|
||||
|
||||
class NLGMetric(Metric):
|
||||
def __init__(
|
||||
self,
|
||||
config,
|
||||
metric_names=[
|
||||
"Bleu_1", "Bleu_2", "Bleu_3", "Bleu_4",
|
||||
"METEOR", "ROUGE_L", "CIDEr"
|
||||
]
|
||||
):
|
||||
super().__init__(config, metric_names)
|
||||
# please install NLGEval from `https://github.com/Maluuba/nlg-eval`
|
||||
from nlgeval import NLGEval
|
||||
self.nlg = NLGEval()
|
||||
|
||||
def compute_metrics(self, outputs, targets, **kwargs):
|
||||
return self.nlg.compute_metrics(
|
||||
hyp_list=outputs, ref_list=targets)
|
||||
|
||||
def print_computed_metrics(self, metrics):
|
||||
Bleu_1 = metrics["Bleu_1"]
|
||||
Bleu_2 = metrics["Bleu_2"]
|
||||
Bleu_3 = metrics["Bleu_3"]
|
||||
Bleu_4 = metrics["Bleu_4"]
|
||||
METEOR = metrics["METEOR"]
|
||||
ROUGE_L = metrics["ROUGE_L"]
|
||||
CIDEr = metrics["CIDEr"]
|
||||
|
||||
print(
|
||||
"Bleu_1: {:.4f} - Bleu_2: {:.4f} - Bleu_3: {:.4f} - Bleu_4: {:.4f} - METEOR: {:.4f} - ROUGE_L: {:.4f} - CIDEr: {:.4f}".format(
|
||||
Bleu_1, Bleu_2, Bleu_3, Bleu_4, METEOR, ROUGE_L, CIDEr
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
class QAMetric(Metric):
|
||||
def __init__(
|
||||
self,
|
||||
config,
|
||||
metric_names=["acc"]
|
||||
):
|
||||
super().__init__(config, metric_names)
|
||||
|
||||
def compute_metrics(self, outputs, targets, **kwargs):
|
||||
from sklearn.metrics import accuracy_score
|
||||
return {"acc": accuracy_score(targets, outputs)}
|
||||
|
||||
def print_computed_metrics(self, metrics):
|
||||
print("acc: {:.4f}".format(metrics["acc"]))
|
||||
|
||||
|
||||
class COINActionSegmentationMetric(Metric):
|
||||
"""
|
||||
COIN dataset listed 3 repos for Action Segmentation.
|
||||
Action Sets, NeuralNetwork-Viterbi, TCFPN-ISBA.
|
||||
The first and second are the same.
|
||||
https://github.com/alexanderrichard/action-sets/blob/master/eval.py
|
||||
|
||||
Future reference for the third:
|
||||
`https://github.com/Zephyr-D/TCFPN-ISBA/blob/master/utils/metrics.py`
|
||||
"""
|
||||
def __init__(self, config, metric_name=["frame_acc"]):
|
||||
super().__init__(config, metric_name)
|
||||
|
||||
def compute_metrics(self, outputs, targets):
|
||||
n_frames = 0
|
||||
n_errors = 0
|
||||
n_errors = sum(outputs != targets)
|
||||
n_frames = len(targets)
|
||||
return {"frame_acc": 1.0 - float(n_errors) / n_frames}
|
||||
|
||||
def print_computed_metrics(self, metrics):
|
||||
fa = metrics["frame_acc"]
|
||||
print("frame accuracy:", fa)
|
||||
|
||||
|
||||
class CrossTaskMetric(Metric):
|
||||
def __init__(self, config, metric_names=["recall"]):
|
||||
super().__init__(config, metric_names)
|
||||
|
||||
def compute_metrics(self, outputs, targets, **kwargs):
|
||||
"""refactored from line 166:
|
||||
https://github.com/DmZhukov/CrossTask/blob/master/train.py"""
|
||||
|
||||
recalls = self._get_recalls(Y_true=targets, Y_pred=outputs)
|
||||
results = {}
|
||||
for task, rec in recalls.items():
|
||||
results[str(task)] = rec
|
||||
|
||||
avg_recall = np.mean(list(recalls.values()))
|
||||
results["recall"] = avg_recall
|
||||
return results
|
||||
|
||||
def print_computed_metrics(self, metrics):
|
||||
print('Recall: {0:0.3f}'.format(metrics["recall"]))
|
||||
for task in metrics:
|
||||
if task != "recall":
|
||||
print('Task {0}. Recall = {1:0.3f}'.format(
|
||||
task, metrics[task]))
|
||||
|
||||
def _get_recalls(self, Y_true, Y_pred):
|
||||
"""refactored from
|
||||
https://github.com/DmZhukov/CrossTask/blob/master/train.py"""
|
||||
|
||||
step_match = {task: 0 for task in Y_true.keys()}
|
||||
step_total = {task: 0 for task in Y_true.keys()}
|
||||
for task, ys_true in Y_true.items():
|
||||
ys_pred = Y_pred[task]
|
||||
for vid in set(ys_pred.keys()).intersection(set(ys_true.keys())):
|
||||
y_true = ys_true[vid]
|
||||
y_pred = ys_pred[vid]
|
||||
step_total[task] += (y_true.sum(axis=0) > 0).sum()
|
||||
step_match[task] += (y_true*y_pred).sum()
|
||||
recalls = {
|
||||
task: step_match[task] / n for task, n in step_total.items()}
|
||||
return recalls
|
||||
|
||||
|
||||
class ActionRecognitionMetric(Metric):
|
||||
def __init__(
|
||||
self,
|
||||
config,
|
||||
metric_names=["acc", "acc_splits", "r1_splits", "r5_splits", "r10_splits"]
|
||||
):
|
||||
super().__init__(config, metric_names)
|
||||
|
||||
def compute_metrics(self, outputs, targets, splits, **kwargs):
|
||||
all_video_embd = outputs
|
||||
labels = targets
|
||||
split1, split2, split3 = splits
|
||||
accs = []
|
||||
r1s = []
|
||||
r5s = []
|
||||
r10s = []
|
||||
for split in range(3):
|
||||
if split == 0:
|
||||
s = split1
|
||||
elif split == 1:
|
||||
s = split2
|
||||
else:
|
||||
s = split3
|
||||
|
||||
X_pred = all_video_embd[np.where(s == 2)[0]]
|
||||
label_test = labels[np.where(s == 2)[0]]
|
||||
logits = X_pred
|
||||
X_pred = np.argmax(X_pred, axis=1)
|
||||
acc = np.sum(X_pred == label_test) / float(len(X_pred))
|
||||
accs.append(acc)
|
||||
# compute recall.
|
||||
sorted_pred = (-logits).argsort(axis=-1)
|
||||
label_test_sp = label_test.reshape(-1, 1)
|
||||
|
||||
r1 = np.mean((sorted_pred[:, :1] == label_test_sp).sum(axis=1), axis=0)
|
||||
r5 = np.mean((sorted_pred[:, :5] == label_test_sp).sum(axis=1), axis=0)
|
||||
r10 = np.mean((sorted_pred[:, :10] == label_test_sp).sum(axis=1), axis=0)
|
||||
r1s.append(r1)
|
||||
r5s.append(r5)
|
||||
r10s.append(r10)
|
||||
|
||||
return {"acc": accs[0], "acc_splits": accs, "r1_splits": r1s, "r5_splits": r5s, "r10_splits": r10s}
|
||||
|
||||
def print_computed_metrics(self, metrics):
|
||||
for split, acc in enumerate(metrics["acc_splits"]):
|
||||
print("Top 1 accuracy on split {}: {}; r1 {}; r5 {}; r10 {}".format(
|
||||
split + 1, acc,
|
||||
metrics["r1_splits"][split],
|
||||
metrics["r5_splits"][split],
|
||||
metrics["r10_splits"][split],
|
||||
)
|
||||
)
|
595
examples/MMPT/mmpt/evaluators/predictor.py
Normal file
595
examples/MMPT/mmpt/evaluators/predictor.py
Normal file
@ -0,0 +1,595 @@
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
#
|
||||
# This source code is licensed under the MIT license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
import os
|
||||
import random
|
||||
import json
|
||||
import numpy as np
|
||||
import torch
|
||||
import pickle
|
||||
import math
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
|
||||
class Predictor(object):
|
||||
"""this base class is used to save predictions to disk
|
||||
(and being called by a evaluator later).
|
||||
Predictor has minimum support of single gpu prediction.
|
||||
"""
|
||||
def __init__(self, config):
|
||||
self.pred_dir = None # on-the-fly eval does not save the results.
|
||||
if hasattr(config, "eval") and config.eval is not None:
|
||||
self.pred_dir = config.eval.save_path
|
||||
os.makedirs(self.pred_dir, exist_ok=True)
|
||||
|
||||
def __call__(self, outputs):
|
||||
"""extract the prediction and save it."""
|
||||
raise NotImplementedError
|
||||
|
||||
def predict_loop(self, model, eval_dataloader, output_file=None):
|
||||
"""on-the-fly prediction on a single gpu."""
|
||||
self.full_scores = []
|
||||
model.eval()
|
||||
model = model.to(0)
|
||||
with torch.no_grad():
|
||||
for data in eval_dataloader:
|
||||
data = self.to_ctx(data)
|
||||
outputs = model(**data)
|
||||
outputs.update(data)
|
||||
self(outputs)
|
||||
return self.finalize(output_file)
|
||||
|
||||
def finalize(self, output_file):
|
||||
pass
|
||||
|
||||
def to_ctx(self, data, ctx=0, dtype=None):
|
||||
if isinstance(data, dict):
|
||||
for key in data:
|
||||
if torch.is_tensor(data[key]):
|
||||
if dtype is not None and data[key].dtype == torch.float32:
|
||||
data[key] = data[key].to(dtype)
|
||||
data[key] = data[key].to(ctx)
|
||||
return data
|
||||
else:
|
||||
raise ValueError("non-dict type of batch is not supported yet.")
|
||||
|
||||
|
||||
class NLGPredictor(Predictor):
|
||||
"""Predicting Text from MMFusion models."""
|
||||
"""TODO: make a context."""
|
||||
def __init__(self, config):
|
||||
super().__init__(config)
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
self.tokenizer = AutoTokenizer.from_pretrained(
|
||||
config.dataset.bert_name,
|
||||
bos_token="[CLS]", eos_token="[SEP]")
|
||||
self.bos_token_id = self.tokenizer.bos_token_id
|
||||
self.eos_token_id = self.tokenizer.eos_token_id
|
||||
|
||||
def predict_loop(self, model, eval_dataloader, output_file=None):
|
||||
"""TODO: refactor base classes."""
|
||||
ctx = 0
|
||||
outputs = {"outputs": [], "targets": [[]]}
|
||||
model.eval()
|
||||
model = model.to(ctx)
|
||||
with torch.no_grad():
|
||||
for data in tqdm(eval_dataloader):
|
||||
data = self.to_ctx(data, ctx)
|
||||
self(data, model, outputs)
|
||||
return self.finalize(outputs, output_file)
|
||||
|
||||
def __call__(self, data, model, outputs):
|
||||
data.update({
|
||||
"bos_token_id": self.bos_token_id,
|
||||
"eos_token_id": self.eos_token_id
|
||||
})
|
||||
|
||||
output = model.generate(**data)
|
||||
assert len(output) == len(data["ref"])
|
||||
for idx, _output in enumerate(output):
|
||||
generated_text = self.tokenizer.decode(
|
||||
_output, skip_special_tokens=True)
|
||||
if generated_text == "":
|
||||
generated_text = "none"
|
||||
outputs["outputs"].append(generated_text)
|
||||
outputs["targets"][0].append(data["ref"][idx])
|
||||
if random.random() < 0.001:
|
||||
print("_output", _output)
|
||||
print("generated_text", generated_text)
|
||||
print("ref", data["ref"][idx])
|
||||
|
||||
def finalize(self, outputs, output_file=None):
|
||||
if output_file is not None:
|
||||
with open(os.path.join(
|
||||
self.pred_dir, output_file + ".json"), "w") as fw:
|
||||
json.dump(outputs, fw, indent=4)
|
||||
return outputs
|
||||
|
||||
|
||||
class RetrievalPredictor(Predictor):
|
||||
"""generated `pooled_video` and `pooled_text`."""
|
||||
def __init__(self, config):
|
||||
super().__init__(config)
|
||||
from transformers import AutoTokenizer
|
||||
self.tokenizer = AutoTokenizer.from_pretrained(
|
||||
config.dataset.bert_name)
|
||||
|
||||
def predict_loop(
|
||||
self,
|
||||
model,
|
||||
eval_dataloader,
|
||||
output_file="retrieval.npy"
|
||||
):
|
||||
"""on-the-fly prediction on a single gpu."""
|
||||
full_scores = []
|
||||
texts = []
|
||||
model.eval()
|
||||
model = model.cuda()
|
||||
with torch.no_grad():
|
||||
for data in eval_dataloader:
|
||||
# convert to dict.
|
||||
if not isinstance(data, dict):
|
||||
data = {
|
||||
"caps": data[0],
|
||||
"cmasks": data[1],
|
||||
"vfeats": data[2],
|
||||
"vmasks": data[3],
|
||||
"video_id": data[4]
|
||||
}
|
||||
data = self.to_ctx(data)
|
||||
outputs = model(**data)
|
||||
outputs.update(data)
|
||||
self(outputs, full_scores)
|
||||
for _cap in data["caps"]:
|
||||
texts.append(
|
||||
self.tokenizer.decode(_cap, skip_special_tokens=True)
|
||||
)
|
||||
|
||||
return self.finalize(full_scores, texts, output_file)
|
||||
|
||||
def __call__(self, sample, full_scores):
|
||||
scores = self._get_pooled_outputs(sample)
|
||||
self._append_scores(scores, full_scores)
|
||||
|
||||
def finalize(self, full_scores, texts, output_file=None):
|
||||
outputs = self._aggregate_scores(full_scores)
|
||||
if output_file is not None:
|
||||
np.save(os.path.join(self.pred_dir, output_file + ".npy"), outputs)
|
||||
return {"outputs": outputs, "texts": texts}
|
||||
|
||||
def _get_pooled_outputs(self, outputs):
|
||||
if "pooled_video" in outputs:
|
||||
return outputs["pooled_video"], outputs["pooled_text"]
|
||||
else:
|
||||
raise ValueError("unknown format of outputs.")
|
||||
|
||||
def _append_scores(self, scores, full_scores):
|
||||
assert len(scores) == 2
|
||||
if len(full_scores) == 0:
|
||||
full_scores.append([])
|
||||
full_scores.append([])
|
||||
full_scores[0].append(scores[0].cpu().detach().numpy())
|
||||
full_scores[1].append(scores[1].cpu().detach().numpy())
|
||||
|
||||
def _aggregate_scores(self, scores):
|
||||
assert len(scores) == 2
|
||||
video_hidden = np.concatenate(scores[0], axis=0)
|
||||
text_hidden = np.concatenate(scores[1], axis=0)
|
||||
# clear up.
|
||||
self.full_scores = []
|
||||
return np.matmul(text_hidden, video_hidden.T)
|
||||
|
||||
|
||||
class QAPredictor(Predictor):
|
||||
"""generated `pooled_video` and `pooled_text`."""
|
||||
def __init__(self, config):
|
||||
super().__init__(config)
|
||||
"""predictor maintains scores and aggregate them."""
|
||||
|
||||
def predict_loop(self, model, eval_dataloader, output_file="qa.npy"):
|
||||
"""on-the-fly prediction on a single gpu."""
|
||||
self.full_scores = []
|
||||
model.eval()
|
||||
model = model.cuda()
|
||||
with torch.no_grad():
|
||||
for data in eval_dataloader:
|
||||
# reshape ans and dup video 5 times.
|
||||
v_len = data["vfeats"].size(1)
|
||||
hidden_size = data["vfeats"].size(2)
|
||||
data["vfeats"] = data["vfeats"].unsqueeze(1).repeat(1, 5, 1, 1).view(-1, v_len, hidden_size)
|
||||
data["vmasks"] = data["vmasks"].unsqueeze(1).repeat(1, 5, 1).view(-1, v_len)
|
||||
|
||||
t_len = data["caps"].size(-1)
|
||||
data["caps"] = data["caps"].view(-1, t_len)
|
||||
data["cmasks"] = data["cmasks"].view(-1, t_len)
|
||||
|
||||
data = self.to_ctx(data)
|
||||
outputs = model(**data)
|
||||
outputs.update(data)
|
||||
self(outputs)
|
||||
return self.finalize(output_file)
|
||||
|
||||
def __call__(self, sample):
|
||||
hidden_size = sample["pooled_video"].size(-1)
|
||||
pooled_video = sample["pooled_video"].view(-1, 5, hidden_size)
|
||||
pooled_text = sample["pooled_text"].view(-1, 5, hidden_size)
|
||||
scores = torch.bmm(pooled_video, pooled_text.transpose(2, 1))
|
||||
scores = scores.argmax(-1)
|
||||
self._append_scores(scores[:, 0], sample["answers"], self.full_scores)
|
||||
|
||||
def finalize(self, output_file=None):
|
||||
outputs, targets = self._aggregate_scores(self.full_scores)
|
||||
if output_file is not None:
|
||||
np.save(os.path.join(self.pred_dir, output_file + ".npy"), outputs)
|
||||
return {"outputs": outputs, "targets": targets}
|
||||
|
||||
def _append_scores(self, scores, answers, full_scores):
|
||||
if len(full_scores) == 0:
|
||||
full_scores.append([])
|
||||
full_scores.append([])
|
||||
full_scores[0].append(scores.cpu().detach().numpy())
|
||||
full_scores[1].append(answers.cpu().detach().numpy())
|
||||
|
||||
def _aggregate_scores(self, scores):
|
||||
assert len(scores) == 2
|
||||
outputs = np.concatenate(scores[0], axis=0)
|
||||
targets = np.concatenate(scores[1], axis=0)
|
||||
# clear up.
|
||||
self.full_scores = []
|
||||
return outputs, targets
|
||||
|
||||
|
||||
class CrossTaskPredictor(Predictor):
|
||||
"""
|
||||
CrossTaskPredictor needs to compute the average of logits
|
||||
for overlapped sliding-window.
|
||||
"""
|
||||
def __init__(self, config):
|
||||
super().__init__(config)
|
||||
self.lsm = torch.nn.LogSoftmax(dim=1)
|
||||
self.max_video_len = config.dataset.max_video_len
|
||||
self.sliding_window = config.dataset.sliding_window
|
||||
self.sliding_window_size = config.dataset.sliding_window_size
|
||||
self.annotation_path = config.dataset.annotation_path
|
||||
|
||||
def predict_loop(self, model, eval_dataloader, output_file="result.pkl"):
|
||||
"""refactored from line 144:
|
||||
https://github.com/DmZhukov/CrossTask/blob/master/train.py
|
||||
"""
|
||||
ctx = 0
|
||||
model.eval()
|
||||
model = model.to(ctx)
|
||||
# this is not a loss but just compute neg_log_prob.
|
||||
Y_pred = {}
|
||||
Y_true = {}
|
||||
with torch.no_grad():
|
||||
for batch in eval_dataloader:
|
||||
self(batch, model, Y_pred, Y_true)
|
||||
return self.finalize(Y_pred, Y_true, output_file)
|
||||
|
||||
def __call__(self, sample, model, Y_pred, Y_true):
|
||||
# please install dp from `https://github.com/DmZhukov/CrossTask`
|
||||
from dp import dp
|
||||
vid, task = sample['video_id'][0], sample['task'][0]
|
||||
sample = self.to_ctx(sample)
|
||||
# compute the average logits over sliding windows.
|
||||
output = model(**sample)
|
||||
batch_logits = output["logits"].cpu()
|
||||
|
||||
video_len = sample["video_len"][0]
|
||||
|
||||
# the following version is slow.
|
||||
logits = torch.zeros((video_len, batch_logits.size(1)))
|
||||
logits_counts = torch.zeros((video_len, 1), dtype=torch.long)
|
||||
# use the same loop as aligner to recover.
|
||||
batch_logit_idx = 0
|
||||
for window_start in range(0, video_len, self.sliding_window):
|
||||
video_end = min(video_len - window_start, self.sliding_window_size)
|
||||
logits[window_start: window_start + video_end] += batch_logits[
|
||||
batch_logit_idx: batch_logit_idx + video_end]
|
||||
batch_logit_idx += video_end
|
||||
logits_counts[window_start: window_start + video_end] += torch.ones((video_end, 1), dtype=torch.long)
|
||||
|
||||
if (video_len - window_start) <= self.sliding_window_size:
|
||||
break
|
||||
|
||||
logits /= logits_counts
|
||||
assert logits.size() == (video_len, batch_logits.size(1)), "{}, {}".format(logits.size(), video_len)
|
||||
|
||||
O = self.lsm(logits)
|
||||
y = np.zeros(O.size(), dtype=np.float32)
|
||||
dp(y, -O.detach().cpu().numpy())
|
||||
if task not in Y_pred:
|
||||
Y_pred[task] = {}
|
||||
Y_pred[task][vid] = y
|
||||
annot_path = os.path.join(
|
||||
self.annotation_path, task+'_'+vid+'.csv')
|
||||
if os.path.exists(annot_path):
|
||||
if task not in Y_true:
|
||||
Y_true[task] = {}
|
||||
Y_true[task][vid] = self._read_assignment(
|
||||
*y.shape, annot_path)
|
||||
|
||||
def finalize(self, Y_pred, Y_true, output_file=None):
|
||||
if output_file is not None:
|
||||
with open(
|
||||
os.path.join(self.pred_dir, output_file + ".pkl"),
|
||||
"wb") as fw:
|
||||
pickle.dump(
|
||||
{"Y_pred": Y_pred, "Y_true": Y_true}, fw,
|
||||
protocol=pickle.HIGHEST_PROTOCOL)
|
||||
return {"outputs": Y_pred, "targets": Y_true}
|
||||
|
||||
def _read_assignment(self, T, K, path):
|
||||
"""
|
||||
refactored from https://github.com/DmZhukov/CrossTask/blob/master/data.py
|
||||
Howto interpret contraints on loss that is going to be minimized:
|
||||
lambd is a big number;
|
||||
self.lambd * C is a big number for all valid position (csv stores invalids)
|
||||
|
||||
def forward(self, O, Y, C):
|
||||
return (Y*(self.lambd * C - self.lsm(O))).mean(dim=0).sum()
|
||||
|
||||
This will load the csv file and fill-in the step col from start to end rows.
|
||||
"""
|
||||
|
||||
Y = np.zeros([T, K], dtype=np.uint8)
|
||||
with open(path, 'r') as f:
|
||||
for line in f:
|
||||
step, start, end = line.strip().split(',')
|
||||
start = int(math.floor(float(start)))
|
||||
end = int(math.ceil(float(end)))
|
||||
step = int(step) - 1
|
||||
Y[start:end, step] = 1
|
||||
return Y
|
||||
|
||||
|
||||
class COINPredictor(Predictor):
|
||||
"""
|
||||
COINPredictor is similar to CrossTask on sliding windows.
|
||||
"""
|
||||
def __init__(self, config):
|
||||
super().__init__(config)
|
||||
self.max_video_len = config.dataset.max_video_len
|
||||
self.sliding_window = config.dataset.sliding_window
|
||||
self.sliding_window_size = config.dataset.sliding_window_size
|
||||
|
||||
def predict_loop(self, model, eval_dataloader, output_file="result.pkl"):
|
||||
"""refactored from line 144:
|
||||
https://github.com/DmZhukov/CrossTask/blob/master/train.py
|
||||
"""
|
||||
ctx = 0
|
||||
model.eval()
|
||||
model = model.to(ctx)
|
||||
# this is not a loss but just compute neg_log_prob.
|
||||
Y_pred = []
|
||||
Y_true = []
|
||||
with torch.no_grad():
|
||||
for batch in eval_dataloader:
|
||||
self(batch, model, Y_pred, Y_true)
|
||||
return self.finalize(Y_pred, Y_true, output_file)
|
||||
|
||||
def __call__(self, sample, model, Y_pred, Y_true):
|
||||
sample = self.to_ctx(sample)
|
||||
# compute the average logits over sliding windows.
|
||||
output = model(**sample)
|
||||
logits = self._merge_windows(sample, output)
|
||||
Y_pred.append(logits.argmax(dim=1))
|
||||
Y_true.append(sample["video_targets"].squeeze(0).cpu())
|
||||
|
||||
def _merge_windows(self, sample, output):
|
||||
targets = sample["targets"].reshape(-1).cpu()
|
||||
valid_mask = targets != -100
|
||||
targets = targets[valid_mask]
|
||||
batch_logits = output["logits"].cpu()
|
||||
batch_logits = batch_logits.reshape(-1, batch_logits.size(-1))
|
||||
batch_logits = batch_logits[valid_mask]
|
||||
|
||||
video_len = sample["video_len"][0]
|
||||
|
||||
# the following version is slow.
|
||||
logits = torch.zeros((video_len, batch_logits.size(1)))
|
||||
logits_counts = torch.zeros((video_len, 1), dtype=torch.long)
|
||||
# use the same loop as aligner to recover.
|
||||
batch_logit_idx = 0
|
||||
for window_start in range(0, video_len, self.sliding_window):
|
||||
video_end = min(video_len - window_start, self.sliding_window_size)
|
||||
logits[window_start: window_start + video_end] += batch_logits[
|
||||
batch_logit_idx: batch_logit_idx + video_end]
|
||||
batch_logit_idx += video_end
|
||||
logits_counts[window_start: window_start + video_end] += torch.ones((video_end, 1), dtype=torch.long)
|
||||
if (video_len - window_start) <= self.sliding_window_size:
|
||||
break
|
||||
logits /= logits_counts
|
||||
assert logits.size() == (video_len, batch_logits.size(1)), "{}, {}".format(logits.size(), video_len)
|
||||
return logits
|
||||
|
||||
def finalize(self, Y_pred, Y_true, output_file=None):
|
||||
Y_pred = torch.cat(Y_pred, dim=0).numpy()
|
||||
Y_true = torch.cat(Y_true, dim=0).numpy()
|
||||
assert len(Y_pred) == len(Y_true)
|
||||
|
||||
error_mask = Y_pred != Y_true
|
||||
print("sample error", Y_pred[error_mask][:10], Y_true[error_mask][:10])
|
||||
print("sample error", Y_pred[error_mask][10:20], Y_true[error_mask][10:20])
|
||||
|
||||
if output_file is not None:
|
||||
with open(
|
||||
os.path.join(self.pred_dir, output_file + ".pkl"),
|
||||
"wb") as fw:
|
||||
pickle.dump(
|
||||
{"Y_pred": Y_pred, "Y_true": Y_true}, fw,
|
||||
protocol=pickle.HIGHEST_PROTOCOL)
|
||||
return {"outputs": Y_pred, "targets": Y_true}
|
||||
|
||||
|
||||
class COINZSPredictor(COINPredictor):
|
||||
"""
|
||||
COINZSPredictor for COIN zero-shot prediction.
|
||||
"""
|
||||
|
||||
def __init__(self, config):
|
||||
super().__init__(config)
|
||||
self.dataset_config = config.dataset
|
||||
|
||||
def predict_loop(self, model, eval_dataloader, output_file="result.pkl"):
|
||||
"""refactored from line 144:
|
||||
https://github.com/DmZhukov/CrossTask/blob/master/train.py
|
||||
"""
|
||||
ctx = 0
|
||||
model.eval()
|
||||
model = model.to(ctx)
|
||||
|
||||
with torch.no_grad():
|
||||
outputs = eval_dataloader.dataset.meta_processor.meta_text_labels(
|
||||
self.dataset_config)
|
||||
outputs = self.to_ctx(outputs, ctx)
|
||||
label_hidden_states = model.forward_text(**outputs).cpu()
|
||||
label_sim = label_hidden_states @ label_hidden_states.t()
|
||||
num_labels = label_sim.size(0)
|
||||
eye_mask = ~torch.eye(num_labels, dtype=torch.bool)
|
||||
label_sim = label_sim.masked_select(eye_mask).view(num_labels, num_labels - 1)
|
||||
lbd = label_sim.max()
|
||||
|
||||
# this is not a loss but just compute neg_log_prob.
|
||||
Y_pred = []
|
||||
Y_true = []
|
||||
with torch.no_grad():
|
||||
for batch in eval_dataloader:
|
||||
self(batch, label_hidden_states, model, lbd, Y_pred, Y_true)
|
||||
return self.finalize(Y_pred, Y_true, output_file)
|
||||
|
||||
def reshape_subsample(self, sample):
|
||||
for key in sample:
|
||||
if torch.is_tensor(sample[key]):
|
||||
sample[key] = self.flat_subsample(sample[key])
|
||||
return sample
|
||||
|
||||
def flat_subsample(self, tensor):
|
||||
if len(tensor.size()) > 1 and tensor.size(0) == 1:
|
||||
tensor = tensor.squeeze(0)
|
||||
return tensor
|
||||
|
||||
def __call__(self, sample, label_hidden_states, model, lbd, Y_pred, Y_true):
|
||||
sample = self.reshape_subsample(sample)
|
||||
sample = self.to_ctx(sample)
|
||||
# compute the average logits over sliding windows.
|
||||
sample["output_hidden_states"] = True
|
||||
video_outputs = model.forward_video(**sample).cpu()
|
||||
output = {"logits": video_outputs[:, 1:sample["vmasks"].size(1)+1] @ label_hidden_states.t()}
|
||||
logits = self._merge_windows(sample, output)
|
||||
# logic of zero-shot for sequence labeling.
|
||||
logits_argmax = logits.argmax(dim=1) + 1 # 0 is "O" label.
|
||||
logits_max = logits.max(dim=1)[0]
|
||||
|
||||
pred = torch.zeros_like(logits_argmax)
|
||||
label_select = logits_max > lbd # 73 or 74
|
||||
pred[label_select] = logits_argmax[label_select]
|
||||
|
||||
Y_pred.append(pred)
|
||||
Y_true.append(sample["video_targets"].squeeze(0).cpu())
|
||||
|
||||
def finalize(self, Y_pred, Y_true, output_file=None):
|
||||
Y_pred = torch.cat(Y_pred, dim=0).numpy()
|
||||
Y_true = torch.cat(Y_true, dim=0).numpy()
|
||||
assert len(Y_pred) == len(Y_true)
|
||||
|
||||
error_mask = Y_pred != Y_true
|
||||
print("sample error", Y_pred[error_mask][:10], Y_true[error_mask][:10])
|
||||
print("sample error", Y_pred[error_mask][10:20], Y_true[error_mask][10:20])
|
||||
|
||||
if output_file is not None:
|
||||
with open(
|
||||
os.path.join(self.pred_dir, output_file + ".pkl"),
|
||||
"wb") as fw:
|
||||
pickle.dump(
|
||||
{"Y_pred": Y_pred, "Y_true": Y_true}, fw,
|
||||
protocol=pickle.HIGHEST_PROTOCOL)
|
||||
return {"outputs": Y_pred, "targets": Y_true}
|
||||
|
||||
|
||||
class DiDeMoPredictor(Predictor):
|
||||
"""reference: https://github.com/LisaAnne/LocalizingMoments/blob/master/utils/eval.py
|
||||
https://github.com/LisaAnne/LocalizingMoments/blob/master/utils/data_processing.py
|
||||
"""
|
||||
def __init__(self, config):
|
||||
super().__init__(config)
|
||||
# load targets.
|
||||
with open(config.dataset.test_path) as data_file:
|
||||
self.test_data = json.load(data_file)
|
||||
|
||||
def predict_loop(self, model, eval_dataloader, output_file="didemo.npy"):
|
||||
"""
|
||||
TODO: two solutions here.
|
||||
"""
|
||||
import itertools
|
||||
# 21 chunks.
|
||||
self.possible_segments = [(0,0), (1,1), (2,2), (3,3), (4,4), (5,5)]
|
||||
for i in itertools.combinations(range(6), 2):
|
||||
self.possible_segments.append(i)
|
||||
# pick segments from a video.
|
||||
|
||||
"""on-the-fly prediction on a single gpu."""
|
||||
self.full_scores = []
|
||||
model.eval()
|
||||
model = model.cuda()
|
||||
with torch.no_grad():
|
||||
for data in eval_dataloader:
|
||||
# TODO special forwarding logic here.
|
||||
data = self.to_ctx(data)
|
||||
data["output_hidden_states"] = True
|
||||
hidden_video = model.forward_video(**data)
|
||||
data["output_hidden_states"] = False
|
||||
pooled_text = model.forward_text(**data)
|
||||
outputs = {
|
||||
"hidden_video": hidden_video,
|
||||
"pooled_text": pooled_text
|
||||
}
|
||||
outputs.update(data)
|
||||
self(outputs)
|
||||
return self.finalize(output_file)
|
||||
|
||||
def __call__(self, sample):
|
||||
# TODO: make an index select from self.possible_segments.
|
||||
hidden_video = sample["hidden_video"]
|
||||
pooled_text = sample["pooled_text"]
|
||||
vmasks = sample["vmasks"]
|
||||
# probably maintain valid results here.
|
||||
|
||||
hidden_video = hidden_video[:, 1:-1, :]
|
||||
# probably maintain valid results here.
|
||||
pooled_video = []
|
||||
for s, e in self.possible_segments:
|
||||
pooled_video.append(
|
||||
torch.mean(
|
||||
hidden_video[:, int(s*5):int((e+1)*5), :],
|
||||
dim=1, keepdim=True)
|
||||
)
|
||||
pooled_video = torch.cat(pooled_video, dim=1)
|
||||
scores = torch.bmm(
|
||||
pooled_video, pooled_text.unsqueeze(-1)).squeeze(-1).cpu()
|
||||
|
||||
ranks = scores.argsort(dim=-1, descending=True)
|
||||
|
||||
for batch_idx, rank in enumerate(ranks):
|
||||
rank_of_moment = []
|
||||
for m_idx, moment in enumerate(rank):
|
||||
s, e = self.possible_segments[moment.item()]
|
||||
if torch.any(
|
||||
vmasks[batch_idx, int(s*5):int((e+1)*5)]
|
||||
):
|
||||
rank_of_moment.append((s, e))
|
||||
self.full_scores.append(rank_of_moment)
|
||||
|
||||
def finalize(self, output_file=None):
|
||||
outputs = self._aggregate_scores(self.full_scores)
|
||||
if output_file is not None:
|
||||
np.save(os.path.join(self.pred_dir, output_file + ".npy"), outputs)
|
||||
return {"outputs": outputs, "targets": self.test_data}
|
||||
|
||||
def _aggregate_scores(self, scores):
|
||||
self.full_scores = []
|
||||
return scores
|
16
examples/MMPT/mmpt/losses/__init__.py
Normal file
16
examples/MMPT/mmpt/losses/__init__.py
Normal file
@ -0,0 +1,16 @@
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
#
|
||||
# This source code is licensed under the MIT license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
from .loss import *
|
||||
from .nce import *
|
||||
|
||||
try:
|
||||
from .fairseqmmloss import *
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
try:
|
||||
from .expnce import *
|
||||
except ImportError:
|
||||
pass
|
63
examples/MMPT/mmpt/losses/fairseqmmloss.py
Normal file
63
examples/MMPT/mmpt/losses/fairseqmmloss.py
Normal file
@ -0,0 +1,63 @@
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
#
|
||||
# This source code is licensed under the MIT license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
|
||||
"""
|
||||
TODO (huxu): a general fairseq criterion for all your pre-defined losses.
|
||||
"""
|
||||
|
||||
from fairseq.criterions import FairseqCriterion, register_criterion
|
||||
from fairseq import metrics
|
||||
|
||||
|
||||
@register_criterion("mmloss")
|
||||
class MMCriterion(FairseqCriterion):
|
||||
def __init__(self, task):
|
||||
super().__init__(task)
|
||||
# TODO (huxu): wrap forward call of loss_fn and eval_fn into task.
|
||||
self.mmtask = task.mmtask
|
||||
|
||||
def forward(self, model, sample):
|
||||
"""Compute the loss for the given sample.
|
||||
Returns a tuple with three elements:
|
||||
1) the loss
|
||||
2) the sample size, which is used as the denominator for the gradient
|
||||
3) logging outputs to display while training
|
||||
"""
|
||||
outputs = self.mmtask(model, sample)
|
||||
|
||||
loss, loss_scalar, max_len, batch_size, sample_size = (
|
||||
outputs["loss"],
|
||||
outputs["loss_scalar"],
|
||||
outputs["max_len"],
|
||||
outputs["batch_size"],
|
||||
outputs["sample_size"],
|
||||
)
|
||||
|
||||
logging_output = {
|
||||
"loss": loss_scalar,
|
||||
"ntokens": max_len * batch_size, # dummy report.
|
||||
"nsentences": batch_size, # dummy report.
|
||||
"sample_size": sample_size,
|
||||
}
|
||||
|
||||
return loss, 1, logging_output
|
||||
|
||||
@staticmethod
|
||||
def reduce_metrics(logging_outputs) -> None:
|
||||
"""Aggregate logging outputs from data parallel training."""
|
||||
"""since we use NCE, our actual batch_size is 1 per GPU.
|
||||
Then we take the mean of each worker."""
|
||||
loss_sum = sum(log.get("loss", 0.0) for log in logging_outputs)
|
||||
sample_size = sum(log.get("sample_size", 0) for log in logging_outputs)
|
||||
metrics.log_scalar("loss", loss_sum / sample_size, round=3)
|
||||
|
||||
@staticmethod
|
||||
def logging_outputs_can_be_summed() -> bool:
|
||||
"""
|
||||
Whether the logging outputs returned by `forward` can be summed
|
||||
across workers prior to calling `reduce_metrics`. Setting this
|
||||
to True will improves distributed training speed.
|
||||
"""
|
||||
return True
|
87
examples/MMPT/mmpt/losses/loss.py
Normal file
87
examples/MMPT/mmpt/losses/loss.py
Normal file
@ -0,0 +1,87 @@
|
||||
# Copyright (c) Facebook, Inc. All Rights Reserved
|
||||
|
||||
import torch
|
||||
|
||||
from torch import nn
|
||||
|
||||
|
||||
class Loss(object):
|
||||
def __call__(self, *args, **kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
# Dummy Loss for testing.
|
||||
class DummyLoss(Loss):
|
||||
def __init__(self):
|
||||
self.loss = nn.CrossEntropyLoss()
|
||||
|
||||
def __call__(self, logits, targets, **kwargs):
|
||||
return self.loss(logits, targets)
|
||||
|
||||
|
||||
class DummyK400Loss(Loss):
|
||||
"""dummy k400 loss for MViT."""
|
||||
def __init__(self):
|
||||
self.loss = nn.CrossEntropyLoss()
|
||||
|
||||
def __call__(self, logits, targets, **kwargs):
|
||||
return self.loss(
|
||||
logits, torch.randint(0, 400, (logits.size(0),), device=logits.device))
|
||||
|
||||
|
||||
class CrossEntropy(Loss):
|
||||
def __init__(self):
|
||||
self.loss = nn.CrossEntropyLoss()
|
||||
|
||||
def __call__(self, logits, targets, **kwargs):
|
||||
return self.loss(logits.reshape(-1, logits.size(-1)), targets.reshape(-1))
|
||||
|
||||
|
||||
class ArgmaxCrossEntropy(Loss):
|
||||
def __init__(self):
|
||||
self.loss = nn.CrossEntropyLoss()
|
||||
|
||||
def __call__(self, logits, targets, **kwargs):
|
||||
return self.loss(logits, targets.argmax(dim=1))
|
||||
|
||||
|
||||
class BCE(Loss):
|
||||
def __init__(self):
|
||||
self.loss = nn.BCEWithLogitsLoss()
|
||||
|
||||
def __call__(self, logits, targets, **kwargs):
|
||||
targets = targets.squeeze(0)
|
||||
return self.loss(logits, targets)
|
||||
|
||||
|
||||
class NLGLoss(Loss):
|
||||
def __init__(self):
|
||||
self.loss = nn.CrossEntropyLoss()
|
||||
|
||||
def __call__(self, logits, text_label, **kwargs):
|
||||
targets = text_label[text_label != -100]
|
||||
return self.loss(logits, targets)
|
||||
|
||||
|
||||
class MSE(Loss):
|
||||
def __init__(self):
|
||||
self.loss = nn.MSELoss()
|
||||
|
||||
def __call__(self, logits, targets, **kwargs):
|
||||
return self.loss(logits, targets)
|
||||
|
||||
|
||||
class L1(Loss):
|
||||
def __init__(self):
|
||||
self.loss = nn.L1Loss()
|
||||
|
||||
def __call__(self, logits, targets, **kwargs):
|
||||
return self.loss(logits, targets)
|
||||
|
||||
|
||||
class SmoothL1(Loss):
|
||||
def __init__(self):
|
||||
self.loss = nn.SmoothL1Loss()
|
||||
|
||||
def __call__(self, logits, targets, **kwargs):
|
||||
return self.loss(logits, targets)
|
156
examples/MMPT/mmpt/losses/nce.py
Normal file
156
examples/MMPT/mmpt/losses/nce.py
Normal file
@ -0,0 +1,156 @@
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
#
|
||||
# This source code is licensed under the MIT license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
|
||||
"""
|
||||
softmax-based NCE loss, used by this project.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
from torch import nn
|
||||
|
||||
from .loss import Loss
|
||||
|
||||
|
||||
class NCE(Loss):
|
||||
def __init__(self):
|
||||
# TODO (huxu): define temperature.
|
||||
self.loss = nn.CrossEntropyLoss()
|
||||
|
||||
def __call__(self, align_scores, **kargs):
|
||||
# note: we reuse the same shape as cls head in BERT (batch_size, 2)
|
||||
# but NCE only needs one logits.
|
||||
# (so we drop all weights in the second neg logits.)
|
||||
align_scores = align_scores[:, :1]
|
||||
# duplicate negative examples
|
||||
batch_size = align_scores.size(0) // 2
|
||||
pos_scores = align_scores[:batch_size]
|
||||
neg_scores = align_scores[batch_size:].view(1, batch_size).repeat(
|
||||
batch_size, 1)
|
||||
scores = torch.cat([pos_scores, neg_scores], dim=1)
|
||||
return self.loss(
|
||||
scores,
|
||||
torch.zeros(
|
||||
(batch_size,),
|
||||
dtype=torch.long,
|
||||
device=align_scores.device),
|
||||
)
|
||||
|
||||
|
||||
class T2VContraLoss(Loss):
|
||||
"""NCE for MM joint space, on softmax text2video matrix.
|
||||
"""
|
||||
def __init__(self):
|
||||
# TODO (huxu): define temperature.
|
||||
self.loss = nn.CrossEntropyLoss()
|
||||
|
||||
def __call__(self, pooled_video, pooled_text, **kargs):
|
||||
batch_size = pooled_video.size(0)
|
||||
logits = torch.mm(pooled_text, pooled_video.transpose(1, 0))
|
||||
targets = torch.arange(
|
||||
batch_size,
|
||||
dtype=torch.long,
|
||||
device=pooled_video.device)
|
||||
return self.loss(logits, targets)
|
||||
|
||||
|
||||
class V2TContraLoss(Loss):
|
||||
"""NCE for MM joint space, with softmax on video2text matrix."""
|
||||
|
||||
def __init__(self):
|
||||
# TODO (huxu): define temperature.
|
||||
self.loss = nn.CrossEntropyLoss()
|
||||
|
||||
def __call__(self, pooled_video, pooled_text, **kargs):
|
||||
batch_size = pooled_video.size(0)
|
||||
logits = torch.mm(pooled_video, pooled_text.transpose(1, 0))
|
||||
targets = torch.arange(
|
||||
batch_size,
|
||||
dtype=torch.long,
|
||||
device=pooled_video.device)
|
||||
return self.loss(logits, targets)
|
||||
|
||||
|
||||
class MMContraLoss(Loss):
|
||||
def __init__(self):
|
||||
self.loss = nn.CrossEntropyLoss()
|
||||
|
||||
def __call__(self, pooled_video, pooled_text, **kwargs):
|
||||
logits_per_video = pooled_video @ pooled_text.t()
|
||||
logits_per_text = pooled_text @ pooled_video.t()
|
||||
|
||||
targets = torch.arange(
|
||||
pooled_video.size(0),
|
||||
dtype=torch.long,
|
||||
device=pooled_video.device)
|
||||
loss_video = self.loss(logits_per_video, targets)
|
||||
loss_text = self.loss(logits_per_text, targets)
|
||||
return loss_video + loss_text
|
||||
|
||||
|
||||
class MTM(Loss):
|
||||
"""Combination of MFM and MLM."""
|
||||
|
||||
def __init__(self):
|
||||
self.loss = nn.CrossEntropyLoss()
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
video_logits,
|
||||
text_logits,
|
||||
video_label,
|
||||
text_label,
|
||||
**kwargs
|
||||
):
|
||||
text_logits = torch.cat([
|
||||
text_logits,
|
||||
torch.zeros(
|
||||
(text_logits.size(0), 1), device=text_logits.device)
|
||||
], dim=1)
|
||||
vt_logits = torch.cat([video_logits, text_logits], dim=0)
|
||||
# loss for video.
|
||||
video_label = torch.zeros(
|
||||
(video_logits.size(0),),
|
||||
dtype=torch.long,
|
||||
device=video_logits.device
|
||||
)
|
||||
|
||||
# loss for text.
|
||||
text_label = text_label.reshape(-1)
|
||||
labels_mask = text_label != -100
|
||||
selected_text_label = text_label[labels_mask]
|
||||
|
||||
vt_label = torch.cat([video_label, selected_text_label], dim=0)
|
||||
return self.loss(vt_logits, vt_label)
|
||||
|
||||
|
||||
class MFMMLM(Loss):
|
||||
"""Combination of MFM and MLM."""
|
||||
|
||||
def __init__(self):
|
||||
self.loss = nn.CrossEntropyLoss()
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
video_logits,
|
||||
text_logits,
|
||||
video_label,
|
||||
text_label,
|
||||
**kwargs
|
||||
):
|
||||
# loss for video.
|
||||
video_label = torch.zeros(
|
||||
(video_logits.size(0),),
|
||||
dtype=torch.long,
|
||||
device=video_logits.device
|
||||
)
|
||||
masked_frame_loss = self.loss(video_logits, video_label)
|
||||
|
||||
# loss for text.
|
||||
text_label = text_label.reshape(-1)
|
||||
labels_mask = text_label != -100
|
||||
selected_text_label = text_label[labels_mask]
|
||||
masked_lm_loss = self.loss(text_logits, selected_text_label)
|
||||
return masked_frame_loss + masked_lm_loss
|
17
examples/MMPT/mmpt/models/__init__.py
Normal file
17
examples/MMPT/mmpt/models/__init__.py
Normal file
@ -0,0 +1,17 @@
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
#
|
||||
# This source code is licensed under the MIT license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
from .mmfusion import *
|
||||
from .transformermodel import *
|
||||
from .mmfusionnlg import *
|
||||
|
||||
try:
|
||||
from .fairseqmmmodel import *
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
try:
|
||||
from .expmmfusion import *
|
||||
except ImportError:
|
||||
pass
|
51
examples/MMPT/mmpt/models/fairseqmmmodel.py
Normal file
51
examples/MMPT/mmpt/models/fairseqmmmodel.py
Normal file
@ -0,0 +1,51 @@
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
#
|
||||
# This source code is licensed under the MIT license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
|
||||
from fairseq.models import (
|
||||
BaseFairseqModel,
|
||||
register_model,
|
||||
register_model_architecture
|
||||
)
|
||||
|
||||
|
||||
@register_model("mmmodel")
|
||||
class FairseqMMModel(BaseFairseqModel):
|
||||
"""a fairseq wrapper of model built by `task`."""
|
||||
|
||||
@classmethod
|
||||
def build_model(cls, args, task):
|
||||
return FairseqMMModel(task.mmtask.model)
|
||||
|
||||
def __init__(self, mmmodel):
|
||||
super().__init__()
|
||||
self.mmmodel = mmmodel
|
||||
|
||||
def forward(self, *args, **kwargs):
|
||||
return self.mmmodel(*args, **kwargs)
|
||||
|
||||
def upgrade_state_dict_named(self, state_dict, name):
|
||||
|
||||
super().upgrade_state_dict_named(state_dict, name)
|
||||
|
||||
keys_to_delete = []
|
||||
|
||||
for key in state_dict:
|
||||
if key not in self.state_dict():
|
||||
keys_to_delete.append(key)
|
||||
for key in keys_to_delete:
|
||||
print("[INFO]", key, "not used anymore.")
|
||||
del state_dict[key]
|
||||
|
||||
# copy any newly defined parameters.
|
||||
for key in self.state_dict():
|
||||
if key not in state_dict:
|
||||
print("[INFO] adding", key)
|
||||
state_dict[key] = self.state_dict()[key]
|
||||
|
||||
|
||||
# a dummy arch, we config the model.
|
||||
@register_model_architecture("mmmodel", "mmarch")
|
||||
def mmarch(args):
|
||||
pass
|
926
examples/MMPT/mmpt/models/mmfusion.py
Normal file
926
examples/MMPT/mmpt/models/mmfusion.py
Normal file
@ -0,0 +1,926 @@
|
||||
# coding=utf-8
|
||||
# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team.
|
||||
# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# Copyright (c) Facebook, Inc. All Rights Reserved
|
||||
|
||||
|
||||
import torch
|
||||
|
||||
from torch import nn
|
||||
|
||||
try:
|
||||
from transformers import AutoConfig, AutoTokenizer
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
from . import transformermodel
|
||||
|
||||
|
||||
class MMPTModel(nn.Module):
|
||||
"""An e2e wrapper of inference model.
|
||||
"""
|
||||
@classmethod
|
||||
def from_pretrained(cls, config, checkpoint="checkpoint_best.pt"):
|
||||
import os
|
||||
from ..utils import recursive_config
|
||||
from ..tasks import Task
|
||||
config = recursive_config(config)
|
||||
mmtask = Task.config_task(config)
|
||||
checkpoint_path = os.path.join(config.eval.save_path, checkpoint)
|
||||
mmtask.build_model(checkpoint=checkpoint_path)
|
||||
# TODO(huxu): make the video encoder configurable.
|
||||
from ..processors.models.s3dg import S3D
|
||||
video_encoder = S3D('pretrained_models/s3d_dict.npy', 512)
|
||||
video_encoder.load_state_dict(
|
||||
torch.load('pretrained_models/s3d_howto100m.pth'))
|
||||
from transformers import AutoTokenizer
|
||||
tokenizer = AutoTokenizer.from_pretrained(
|
||||
config.dataset.bert_name, use_fast=config.dataset.use_fast
|
||||
)
|
||||
from ..processors import Aligner
|
||||
aligner = Aligner(config.dataset)
|
||||
return (
|
||||
MMPTModel(config, mmtask.model, video_encoder),
|
||||
tokenizer,
|
||||
aligner
|
||||
)
|
||||
|
||||
def __init__(self, config, model, video_encoder, **kwargs):
|
||||
super().__init__()
|
||||
self.max_video_len = config.dataset.max_video_len
|
||||
self.video_encoder = video_encoder
|
||||
self.model = model
|
||||
|
||||
def forward(self, video_frames, caps, cmasks, return_score=False):
|
||||
bsz = video_frames.size(0)
|
||||
assert bsz == 1, "only bsz=1 is supported now."
|
||||
seq_len = video_frames.size(1)
|
||||
video_frames = video_frames.view(-1, *video_frames.size()[2:])
|
||||
vfeats = self.video_encoder(video_frames.permute(0, 4, 1, 2, 3))
|
||||
vfeats = vfeats['video_embedding']
|
||||
vfeats = vfeats.view(bsz, seq_len, vfeats.size(-1))
|
||||
padding = torch.zeros(
|
||||
bsz, self.max_video_len - seq_len, vfeats.size(-1))
|
||||
vfeats = torch.cat([vfeats, padding], dim=1)
|
||||
vmasks = torch.cat([
|
||||
torch.ones((bsz, seq_len), dtype=torch.bool),
|
||||
torch.zeros((bsz, self.max_video_len - seq_len), dtype=torch.bool)
|
||||
],
|
||||
dim=1
|
||||
)
|
||||
output = self.model(caps, cmasks, vfeats, vmasks)
|
||||
if return_score:
|
||||
output = {"score": torch.bmm(
|
||||
output["pooled_video"][:, None, :],
|
||||
output["pooled_text"][:, :, None]
|
||||
).squeeze(-1).squeeze(-1)}
|
||||
return output
|
||||
|
||||
|
||||
class MMFusion(nn.Module):
|
||||
"""a MMPT wrapper class for MMBert style models.
|
||||
TODO: move isolated mask to a subclass.
|
||||
"""
|
||||
def __init__(self, config, **kwargs):
|
||||
super().__init__()
|
||||
transformer_config = AutoConfig.from_pretrained(
|
||||
config.dataset.bert_name)
|
||||
self.hidden_size = transformer_config.hidden_size
|
||||
self.is_train = False
|
||||
if config.dataset.train_path is not None:
|
||||
self.is_train = True
|
||||
# 0 means no iso; 1-12 means iso up to that layer.
|
||||
self.num_hidden_layers = transformer_config.num_hidden_layers
|
||||
self.last_iso_layer = 0
|
||||
if config.dataset.num_iso_layer is not None:
|
||||
self.last_iso_layer = config.dataset.num_iso_layer - 1 + 1
|
||||
|
||||
if config.model.mm_encoder_cls is not None:
|
||||
mm_encoder_cls = getattr(transformermodel, config.model.mm_encoder_cls)
|
||||
model_config = AutoConfig.from_pretrained(config.dataset.bert_name)
|
||||
model_config.max_video_len = config.dataset.max_video_len
|
||||
# TODO: a general way to add parameter for a model.
|
||||
model_config.use_seg_emb = config.model.use_seg_emb
|
||||
self.mm_encoder = mm_encoder_cls.from_pretrained(
|
||||
config.dataset.bert_name, config=model_config)
|
||||
elif config.model.video_encoder_cls is not None\
|
||||
and config.model.text_encoder_cls is not None:
|
||||
video_encoder_cls = getattr(transformermodel, config.model.video_encoder_cls)
|
||||
model_config = AutoConfig.from_pretrained(config.dataset.bert_name)
|
||||
model_config.max_video_len = config.dataset.max_video_len
|
||||
# TODO: make each model a set of config class.
|
||||
if hasattr(model_config, "num_layers"):
|
||||
model_config.num_layers = config.model.num_hidden_video_layers
|
||||
else:
|
||||
model_config.num_hidden_layers = config.model.num_hidden_video_layers
|
||||
self.video_encoder = video_encoder_cls.from_pretrained(
|
||||
config.dataset.bert_name, config=model_config)
|
||||
# exact same NLP model from Huggingface.
|
||||
text_encoder_cls = getattr(transformermodel, config.model.text_encoder_cls)
|
||||
self.text_encoder = text_encoder_cls.from_pretrained(
|
||||
config.dataset.bert_name)
|
||||
else:
|
||||
raise ValueError("the encoder must be either MM or two backbones.")
|
||||
|
||||
def forward(
|
||||
self,
|
||||
caps,
|
||||
cmasks,
|
||||
vfeats,
|
||||
vmasks,
|
||||
**kwargs
|
||||
):
|
||||
raise NotImplementedError(
|
||||
"Please derive MMFusion module."
|
||||
)
|
||||
|
||||
def _mm_on_the_fly(
|
||||
self,
|
||||
cmasks,
|
||||
vmasks,
|
||||
attention_mask
|
||||
):
|
||||
"""helper function for mask, seg_ids and token_type_ids."""
|
||||
if attention_mask is None:
|
||||
attention_mask = self._mm_attention_mask(cmasks, vmasks)
|
||||
|
||||
"""
|
||||
0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1
|
||||
| first sequence | second sequence |
|
||||
"""
|
||||
token_type_ids = torch.cat(
|
||||
[
|
||||
torch.zeros(
|
||||
(vmasks.size(0), vmasks.size(1) + 2),
|
||||
dtype=torch.long,
|
||||
device=vmasks.device,
|
||||
),
|
||||
torch.ones(
|
||||
(cmasks.size(0), cmasks.size(1) - 2),
|
||||
dtype=torch.long,
|
||||
device=cmasks.device,
|
||||
),
|
||||
],
|
||||
dim=1,
|
||||
)
|
||||
return attention_mask, token_type_ids
|
||||
|
||||
def _mm_attention_mask(self, cmasks, vmasks):
|
||||
assert cmasks.size(0) == vmasks.size(0), "{}, {}, {}, {}".format(
|
||||
str(cmasks.size()),
|
||||
str(vmasks.size()),
|
||||
str(cmasks.size(0)),
|
||||
str(vmasks.size(0)),
|
||||
)
|
||||
|
||||
mm_mask = torch.cat([cmasks[:, :1], vmasks, cmasks[:, 1:]], dim=1)
|
||||
if self.last_iso_layer == 0:
|
||||
# hard attention mask.
|
||||
return mm_mask
|
||||
else:
|
||||
# a gpu iso mask; 0 : num_iso_layer is isolated;
|
||||
# num_iso_layer: are MM-fused.
|
||||
# make an iso layer
|
||||
batch_size = cmasks.size(0)
|
||||
iso_mask = self._make_iso_mask(batch_size, cmasks, vmasks)
|
||||
mm_mask = mm_mask[:, None, :].repeat(1, mm_mask.size(-1), 1)
|
||||
iso_mm_masks = []
|
||||
# hard attention mask.
|
||||
iso_mask = iso_mask[:, None, :, :].repeat(
|
||||
1, self.last_iso_layer, 1, 1)
|
||||
iso_mm_masks.append(iso_mask)
|
||||
if self.last_iso_layer < self.num_hidden_layers:
|
||||
mm_mask = mm_mask[:, None, :, :].repeat(
|
||||
1, self.num_hidden_layers - self.last_iso_layer, 1, 1
|
||||
)
|
||||
iso_mm_masks.append(mm_mask)
|
||||
iso_mm_masks = torch.cat(iso_mm_masks, dim=1)
|
||||
return iso_mm_masks
|
||||
|
||||
def _make_iso_mask(self, batch_size, cmasks, vmasks):
|
||||
cls_self_mask = torch.cat(
|
||||
[
|
||||
torch.ones(
|
||||
(batch_size, 1), dtype=torch.bool, device=cmasks.device),
|
||||
torch.zeros(
|
||||
(batch_size, cmasks.size(1) + vmasks.size(1) - 1),
|
||||
dtype=torch.bool, device=cmasks.device)
|
||||
], dim=1)
|
||||
|
||||
iso_video_mask = torch.cat(
|
||||
[
|
||||
# [CLS] is not used.
|
||||
torch.zeros(
|
||||
(batch_size, 1), dtype=torch.bool, device=cmasks.device
|
||||
),
|
||||
vmasks,
|
||||
# assume to be 1.
|
||||
cmasks[:, 1:2],
|
||||
# 2 means [CLS] + [SEP]
|
||||
torch.zeros(
|
||||
(batch_size, cmasks.size(1) - 2),
|
||||
dtype=torch.bool,
|
||||
device=cmasks.device,
|
||||
),
|
||||
],
|
||||
dim=1,
|
||||
)
|
||||
iso_text_mask = torch.cat(
|
||||
[
|
||||
torch.zeros(
|
||||
(batch_size, 2 + vmasks.size(1)),
|
||||
dtype=torch.bool,
|
||||
device=cmasks.device,
|
||||
), # [CLS] is not used.
|
||||
cmasks[:, 2:], # assume to be 1.
|
||||
],
|
||||
dim=1,
|
||||
)
|
||||
cls_self_mask = cls_self_mask[:, None, :]
|
||||
iso_video_mask = iso_video_mask[:, None, :].repeat(
|
||||
1, vmasks.size(1) + 1, 1)
|
||||
iso_text_mask = iso_text_mask[:, None, :].repeat(
|
||||
1, cmasks.size(1) - 2, 1)
|
||||
return torch.cat([cls_self_mask, iso_video_mask, iso_text_mask], dim=1)
|
||||
|
||||
def _pooling_vt_layer(
|
||||
self,
|
||||
layered_sequence_output,
|
||||
cmasks,
|
||||
vmasks
|
||||
):
|
||||
layer_idx = self.last_iso_layer \
|
||||
if self.last_iso_layer > 0 else self.num_hidden_layers
|
||||
hidden_state = layered_sequence_output[layer_idx]
|
||||
# also output pooled_video and pooled_text.
|
||||
batch_size = cmasks.size(0)
|
||||
# pool the modality.
|
||||
text_offset = vmasks.size(1) + 2 # [CLS] + [SEP]
|
||||
# video tokens + [SEP]
|
||||
video_outputs = hidden_state[:, 1:text_offset]
|
||||
video_attention_mask = torch.cat(
|
||||
[
|
||||
vmasks,
|
||||
torch.ones(
|
||||
(batch_size, 1), dtype=torch.bool, device=vmasks.device),
|
||||
],
|
||||
dim=1,
|
||||
)
|
||||
assert video_outputs.size(1) == video_attention_mask.size(1)
|
||||
pooled_video = torch.sum(
|
||||
video_outputs * video_attention_mask.unsqueeze(-1), dim=1
|
||||
) / video_attention_mask.sum(1, keepdim=True)
|
||||
# pooled_video = torch.mean(video_outputs[0], dim=1)
|
||||
|
||||
# text tokens + [SEP]
|
||||
text_attention_mask = cmasks[:, 2:]
|
||||
text_outputs = hidden_state[:, text_offset:]
|
||||
assert text_outputs.size(1) == text_attention_mask.size(1)
|
||||
pooled_text = torch.sum(
|
||||
text_outputs * text_attention_mask.unsqueeze(-1), dim=1
|
||||
) / text_attention_mask.sum(1, keepdim=True)
|
||||
return pooled_video, pooled_text
|
||||
|
||||
|
||||
class MMFusionMFMMLM(MMFusion):
|
||||
"""forward function for MFM and MLM."""
|
||||
def forward(
|
||||
self,
|
||||
caps,
|
||||
cmasks,
|
||||
vfeats,
|
||||
vmasks,
|
||||
attention_mask=None,
|
||||
video_label=None,
|
||||
text_label=None,
|
||||
**kwargs
|
||||
):
|
||||
output_hidden_states = False if self.is_train else True
|
||||
|
||||
target_vfeats, non_masked_frame_mask = None, None
|
||||
if video_label is not None:
|
||||
target_vfeats = vfeats.masked_select(
|
||||
video_label.unsqueeze(-1)).view(
|
||||
-1, vfeats.size(-1)
|
||||
)
|
||||
# mask video token.
|
||||
vfeats[video_label] = 0.0
|
||||
non_masked_frame_mask = vmasks.clone()
|
||||
non_masked_frame_mask[video_label] = False
|
||||
|
||||
attention_mask, token_type_ids = self._mm_on_the_fly(
|
||||
cmasks, vmasks, attention_mask)
|
||||
|
||||
outputs = self.mm_encoder(
|
||||
input_ids=caps,
|
||||
input_video_embeds=vfeats,
|
||||
attention_mask=attention_mask,
|
||||
token_type_ids=token_type_ids,
|
||||
masked_frame_labels=video_label,
|
||||
target_video_hidden_states=target_vfeats,
|
||||
non_masked_frame_mask=non_masked_frame_mask,
|
||||
masked_lm_labels=text_label,
|
||||
output_hidden_states=output_hidden_states,
|
||||
)
|
||||
|
||||
video_logits, text_logits = outputs[0], outputs[1]
|
||||
|
||||
if self.is_train: # return earlier for training.
|
||||
return {
|
||||
"video_logits": video_logits,
|
||||
"text_logits": text_logits,
|
||||
}
|
||||
|
||||
pooled_video, pooled_text = self._pooling_vt_layer(
|
||||
outputs[2], cmasks, vmasks)
|
||||
return {"pooled_video": pooled_video, "pooled_text": pooled_text}
|
||||
|
||||
|
||||
class MMFusionMTM(MMFusionMFMMLM):
|
||||
def __init__(self, config, **kwargs):
|
||||
super().__init__(config)
|
||||
"""
|
||||
For reproducibility:
|
||||
self.mm_encoder will be initialized then discarded.
|
||||
"""
|
||||
from .transformermodel import MMBertForMTM
|
||||
model_config = AutoConfig.from_pretrained(config.dataset.bert_name)
|
||||
model_config.max_video_len = config.dataset.max_video_len
|
||||
model_config.use_seg_emb = config.model.use_seg_emb
|
||||
self.mm_encoder = MMBertForMTM.from_pretrained(
|
||||
config.dataset.bert_name, config=model_config)
|
||||
|
||||
|
||||
class MMFusionShare(MMFusion):
|
||||
"""A retrival wrapper using mm_encoder as both video/text backbone.
|
||||
TODO: move formally.
|
||||
"""
|
||||
def forward(
|
||||
self,
|
||||
caps,
|
||||
cmasks,
|
||||
vfeats,
|
||||
vmasks,
|
||||
attention_mask=None,
|
||||
video_label=None,
|
||||
text_label=None,
|
||||
output_hidden_states=False,
|
||||
**kwargs
|
||||
):
|
||||
pooled_video = self.forward_video(
|
||||
vfeats,
|
||||
vmasks,
|
||||
caps,
|
||||
cmasks,
|
||||
output_hidden_states
|
||||
)
|
||||
|
||||
pooled_text = self.forward_text(
|
||||
caps,
|
||||
cmasks,
|
||||
output_hidden_states
|
||||
)
|
||||
|
||||
return {"pooled_video": pooled_video, "pooled_text": pooled_text}
|
||||
|
||||
def forward_video(
|
||||
self,
|
||||
vfeats,
|
||||
vmasks,
|
||||
caps,
|
||||
cmasks,
|
||||
output_hidden_states=False,
|
||||
**kwargs
|
||||
):
|
||||
input_ids = caps[:, :2]
|
||||
|
||||
attention_mask = torch.cat([
|
||||
cmasks[:, :1],
|
||||
vmasks,
|
||||
cmasks[:, 1:2]
|
||||
], dim=1)
|
||||
|
||||
token_type_ids = torch.zeros(
|
||||
(vmasks.size(0), vmasks.size(1) + 2),
|
||||
dtype=torch.long,
|
||||
device=vmasks.device)
|
||||
|
||||
outputs = self.mm_encoder(
|
||||
input_ids=input_ids,
|
||||
input_video_embeds=vfeats,
|
||||
attention_mask=attention_mask,
|
||||
token_type_ids=token_type_ids,
|
||||
output_hidden_states=True
|
||||
)
|
||||
video_outputs = outputs[0]
|
||||
|
||||
if output_hidden_states:
|
||||
return video_outputs
|
||||
|
||||
batch_size = cmasks.size(0)
|
||||
|
||||
video_attention_mask = torch.cat(
|
||||
[
|
||||
torch.zeros(
|
||||
(batch_size, 1), dtype=torch.bool, device=vmasks.device),
|
||||
vmasks,
|
||||
torch.ones(
|
||||
(batch_size, 1), dtype=torch.bool, device=vmasks.device),
|
||||
],
|
||||
dim=1,
|
||||
)
|
||||
assert video_outputs.size(1) == video_attention_mask.size(1)
|
||||
|
||||
video_attention_mask = video_attention_mask.type(video_outputs.dtype) \
|
||||
/ video_attention_mask.sum(1, keepdim=True)
|
||||
|
||||
pooled_video = torch.bmm(
|
||||
video_outputs.transpose(2, 1),
|
||||
video_attention_mask.unsqueeze(2)
|
||||
).squeeze(-1)
|
||||
return pooled_video # video_outputs
|
||||
|
||||
def forward_text(
|
||||
self,
|
||||
caps,
|
||||
cmasks,
|
||||
output_hidden_states=False,
|
||||
**kwargs
|
||||
):
|
||||
input_ids = torch.cat([
|
||||
caps[:, :1], caps[:, 2:],
|
||||
], dim=1)
|
||||
|
||||
attention_mask = torch.cat([
|
||||
cmasks[:, :1],
|
||||
cmasks[:, 2:]
|
||||
], dim=1)
|
||||
|
||||
token_type_ids = torch.cat([
|
||||
torch.zeros(
|
||||
(cmasks.size(0), 1),
|
||||
dtype=torch.long,
|
||||
device=cmasks.device),
|
||||
torch.ones(
|
||||
(cmasks.size(0), cmasks.size(1) - 2),
|
||||
dtype=torch.long,
|
||||
device=cmasks.device)
|
||||
], dim=1)
|
||||
|
||||
outputs = self.mm_encoder(
|
||||
input_ids=input_ids,
|
||||
input_video_embeds=None,
|
||||
attention_mask=attention_mask,
|
||||
token_type_ids=token_type_ids,
|
||||
output_hidden_states=True
|
||||
)
|
||||
text_outputs = outputs[0]
|
||||
|
||||
if output_hidden_states:
|
||||
return text_outputs
|
||||
|
||||
batch_size = caps.size(0)
|
||||
# text tokens + [SEP]
|
||||
text_attention_mask = torch.cat([
|
||||
torch.zeros(
|
||||
(batch_size, 1), dtype=torch.bool, device=cmasks.device),
|
||||
cmasks[:, 2:]
|
||||
], dim=1)
|
||||
|
||||
assert text_outputs.size(1) == text_attention_mask.size(1)
|
||||
|
||||
text_attention_mask = text_attention_mask.type(text_outputs.dtype) \
|
||||
/ text_attention_mask.sum(1, keepdim=True)
|
||||
|
||||
pooled_text = torch.bmm(
|
||||
text_outputs.transpose(2, 1),
|
||||
text_attention_mask.unsqueeze(2)
|
||||
).squeeze(-1)
|
||||
return pooled_text # text_outputs
|
||||
|
||||
|
||||
class MMFusionSeparate(MMFusionShare):
|
||||
def forward_video(
|
||||
self,
|
||||
vfeats,
|
||||
vmasks,
|
||||
caps,
|
||||
cmasks,
|
||||
output_hidden_states=False,
|
||||
**kwargs
|
||||
):
|
||||
input_ids = caps[:, :2]
|
||||
|
||||
attention_mask = torch.cat([
|
||||
cmasks[:, :1],
|
||||
vmasks,
|
||||
cmasks[:, 1:2]
|
||||
], dim=1)
|
||||
|
||||
token_type_ids = torch.zeros(
|
||||
(vmasks.size(0), vmasks.size(1) + 2),
|
||||
dtype=torch.long,
|
||||
device=vmasks.device)
|
||||
|
||||
outputs = self.video_encoder(
|
||||
input_ids=input_ids,
|
||||
input_video_embeds=vfeats,
|
||||
attention_mask=attention_mask,
|
||||
token_type_ids=token_type_ids,
|
||||
output_hidden_states=True
|
||||
)
|
||||
video_outputs = outputs[0]
|
||||
|
||||
if output_hidden_states:
|
||||
return video_outputs
|
||||
|
||||
batch_size = cmasks.size(0)
|
||||
|
||||
video_attention_mask = torch.cat(
|
||||
[
|
||||
torch.zeros(
|
||||
(batch_size, 1), dtype=torch.bool, device=vmasks.device),
|
||||
vmasks,
|
||||
torch.ones(
|
||||
(batch_size, 1), dtype=torch.bool, device=vmasks.device),
|
||||
],
|
||||
dim=1,
|
||||
)
|
||||
assert video_outputs.size(1) == video_attention_mask.size(1)
|
||||
|
||||
video_attention_mask = video_attention_mask.type(video_outputs.dtype) \
|
||||
/ video_attention_mask.sum(1, keepdim=True)
|
||||
|
||||
pooled_video = torch.bmm(
|
||||
video_outputs.transpose(2, 1),
|
||||
video_attention_mask.unsqueeze(2)
|
||||
).squeeze(-1)
|
||||
return pooled_video # video_outputs
|
||||
|
||||
def forward_text(
|
||||
self,
|
||||
caps,
|
||||
cmasks,
|
||||
output_hidden_states=False,
|
||||
**kwargs
|
||||
):
|
||||
input_ids = torch.cat([
|
||||
caps[:, :1], caps[:, 2:],
|
||||
], dim=1)
|
||||
|
||||
attention_mask = torch.cat([
|
||||
cmasks[:, :1],
|
||||
cmasks[:, 2:]
|
||||
], dim=1)
|
||||
# different from sharing, we use all-0 type.
|
||||
token_type_ids = torch.zeros(
|
||||
(cmasks.size(0), cmasks.size(1) - 1),
|
||||
dtype=torch.long,
|
||||
device=cmasks.device)
|
||||
|
||||
outputs = self.text_encoder(
|
||||
input_ids=input_ids,
|
||||
attention_mask=attention_mask,
|
||||
token_type_ids=token_type_ids,
|
||||
output_hidden_states=True
|
||||
)
|
||||
text_outputs = outputs[0]
|
||||
|
||||
if output_hidden_states:
|
||||
return text_outputs
|
||||
|
||||
batch_size = caps.size(0)
|
||||
# text tokens + [SEP]
|
||||
text_attention_mask = torch.cat([
|
||||
torch.zeros(
|
||||
(batch_size, 1), dtype=torch.bool, device=cmasks.device),
|
||||
cmasks[:, 2:]
|
||||
], dim=1)
|
||||
|
||||
assert text_outputs.size(1) == text_attention_mask.size(1)
|
||||
|
||||
text_attention_mask = text_attention_mask.type(text_outputs.dtype) \
|
||||
/ text_attention_mask.sum(1, keepdim=True)
|
||||
|
||||
pooled_text = torch.bmm(
|
||||
text_outputs.transpose(2, 1),
|
||||
text_attention_mask.unsqueeze(2)
|
||||
).squeeze(-1)
|
||||
return pooled_text # text_outputs
|
||||
|
||||
|
||||
class MMFusionJoint(MMFusion):
|
||||
"""fine-tuning wrapper for retrival task."""
|
||||
|
||||
def forward(
|
||||
self,
|
||||
caps,
|
||||
cmasks,
|
||||
vfeats,
|
||||
vmasks,
|
||||
attention_mask=None,
|
||||
video_label=None,
|
||||
text_label=None,
|
||||
**kwargs
|
||||
):
|
||||
# TODO (huxu): other ways to do negative examples; move the following
|
||||
# into your criterion forward.
|
||||
output_hidden_states = True
|
||||
|
||||
attention_mask, token_type_ids = self._mm_on_the_fly(
|
||||
cmasks, vmasks, attention_mask)
|
||||
|
||||
separate_forward_split = (
|
||||
None if self.is_train else vmasks.size(1) + 2
|
||||
) # [CLS] + [SEP]
|
||||
|
||||
outputs = self.mm_encoder(
|
||||
input_ids=caps,
|
||||
input_video_embeds=vfeats,
|
||||
attention_mask=attention_mask,
|
||||
token_type_ids=token_type_ids,
|
||||
output_hidden_states=output_hidden_states,
|
||||
separate_forward_split=separate_forward_split,
|
||||
)
|
||||
|
||||
pooled_video, pooled_text = self._pooling_vt_layer(
|
||||
outputs[2], cmasks, vmasks)
|
||||
return {"pooled_video": pooled_video, "pooled_text": pooled_text}
|
||||
|
||||
|
||||
class MMFusionActionSegmentation(MMFusion):
|
||||
"""Fine-tuning wrapper for action segmentation.
|
||||
TODO: rename this for VLM.
|
||||
"""
|
||||
def forward(
|
||||
self,
|
||||
caps,
|
||||
cmasks,
|
||||
vfeats,
|
||||
vmasks,
|
||||
attention_mask=None,
|
||||
**kwargs
|
||||
):
|
||||
# ActionLocalization assume of batch_size=1, squeeze it.
|
||||
caps = caps.view(-1, caps.size(-1))
|
||||
cmasks = cmasks.view(-1, cmasks.size(-1))
|
||||
vfeats = vfeats.view(-1, vfeats.size(2), vfeats.size(3))
|
||||
vmasks = vmasks.view(-1, vmasks.size(-1))
|
||||
|
||||
# this may not cover all shapes of attention_mask.
|
||||
attention_mask = attention_mask.view(
|
||||
-1, attention_mask.size(2), attention_mask.size(3)) \
|
||||
if attention_mask is not None else None
|
||||
|
||||
# TODO (huxu): other ways to do negative examples; move the following
|
||||
# into your criterion forward.
|
||||
output_hidden_states = True
|
||||
|
||||
# video forwarding, text is dummy; never use attention_mask.
|
||||
attention_mask, token_type_ids = self._mm_on_the_fly(
|
||||
cmasks, vmasks, attention_mask)
|
||||
|
||||
logits = self.mm_encoder(
|
||||
input_ids=caps,
|
||||
input_video_embeds=vfeats,
|
||||
attention_mask=attention_mask,
|
||||
token_type_ids=token_type_ids,
|
||||
output_hidden_states=output_hidden_states,
|
||||
)
|
||||
return {"logits": logits[0][:, 1:vmasks.size(1)+1]}
|
||||
|
||||
|
||||
class MMFusionActionLocalization(MMFusion):
|
||||
"""fine-tuning model for retrival task."""
|
||||
|
||||
def __init__(self, config, **kwargs):
|
||||
super().__init__(config)
|
||||
tokenizer = AutoTokenizer.from_pretrained(
|
||||
config.dataset.bert_name)
|
||||
self.cls_token_id = tokenizer.cls_token_id
|
||||
self.sep_token_id = tokenizer.sep_token_id
|
||||
self.pad_token_id = tokenizer.pad_token_id
|
||||
|
||||
def forward(
|
||||
self,
|
||||
caps,
|
||||
cmasks,
|
||||
vfeats,
|
||||
vmasks,
|
||||
attention_mask=None,
|
||||
**kwargs
|
||||
):
|
||||
# ActionLocalization assume of batch_size=1, squeeze it.
|
||||
caps = caps.squeeze(0)
|
||||
cmasks = cmasks.squeeze(0)
|
||||
vfeats = vfeats.squeeze(0)
|
||||
vmasks = vmasks.squeeze(0)
|
||||
attention_mask = attention_mask.squeeze(0) if attention_mask is not None else None
|
||||
|
||||
# TODO (huxu): other ways to do negative examples; move the following
|
||||
# into your criterion forward.
|
||||
output_hidden_states = True
|
||||
|
||||
# a len1 dummy video token.
|
||||
dummy_vfeats = torch.zeros(
|
||||
(caps.size(0), 1, vfeats.size(-1)), device=vfeats.device, dtype=vfeats.dtype)
|
||||
dummy_vmasks = torch.ones(
|
||||
(caps.size(0), 1), dtype=torch.bool,
|
||||
device=vfeats.device)
|
||||
|
||||
dummy_caps = torch.LongTensor(
|
||||
[[self.cls_token_id, self.sep_token_id,
|
||||
self.pad_token_id, self.sep_token_id]],
|
||||
).to(caps.device).repeat(vfeats.size(0), 1)
|
||||
dummy_cmasks = torch.BoolTensor(
|
||||
[[0, 1, 0, 1]] # pad are valid for attention.
|
||||
).to(caps.device).repeat(vfeats.size(0), 1)
|
||||
|
||||
# video forwarding, text is dummy; never use attention_mask.
|
||||
attention_mask, token_type_ids = self._mm_on_the_fly(
|
||||
dummy_cmasks, vmasks, None)
|
||||
|
||||
outputs = self.mm_encoder(
|
||||
input_ids=dummy_caps,
|
||||
input_video_embeds=vfeats,
|
||||
attention_mask=attention_mask,
|
||||
token_type_ids=token_type_ids,
|
||||
output_hidden_states=output_hidden_states,
|
||||
)
|
||||
|
||||
layer_idx = self.last_iso_layer \
|
||||
if self.last_iso_layer > 0 else self.num_hidden_layers
|
||||
|
||||
video_seq = outputs[2][layer_idx][:, 1:vmasks.size(1)+1].masked_select(
|
||||
vmasks.unsqueeze(-1)
|
||||
).view(-1, self.hidden_size)
|
||||
|
||||
# text forwarding, video is dummy
|
||||
attention_mask, token_type_ids = self._mm_on_the_fly(
|
||||
cmasks, dummy_vmasks, None)
|
||||
|
||||
outputs = self.mm_encoder(
|
||||
input_ids=caps,
|
||||
input_video_embeds=dummy_vfeats,
|
||||
attention_mask=attention_mask,
|
||||
token_type_ids=token_type_ids,
|
||||
output_hidden_states=output_hidden_states,
|
||||
)
|
||||
|
||||
_, pooled_text = self._pooling_vt_layer(
|
||||
outputs[2], cmasks, dummy_vmasks)
|
||||
# this line is not right.
|
||||
logits = torch.mm(video_seq, pooled_text.transpose(1, 0))
|
||||
return {"logits": logits}
|
||||
|
||||
|
||||
# --------------- MMFusionSeparate for end tasks ---------------
|
||||
|
||||
class MMFusionSeparateActionSegmentation(MMFusionSeparate):
|
||||
"""Fine-tuning wrapper for action segmentation."""
|
||||
def forward(
|
||||
self,
|
||||
caps,
|
||||
cmasks,
|
||||
vfeats,
|
||||
vmasks,
|
||||
attention_mask=None,
|
||||
**kwargs
|
||||
):
|
||||
# ActionLocalization assume of batch_size=1, squeeze it.
|
||||
caps = caps.view(-1, caps.size(-1))
|
||||
cmasks = cmasks.view(-1, cmasks.size(-1))
|
||||
vfeats = vfeats.view(-1, vfeats.size(2), vfeats.size(3))
|
||||
vmasks = vmasks.view(-1, vmasks.size(-1))
|
||||
logits = self.forward_video(
|
||||
vfeats,
|
||||
vmasks,
|
||||
caps,
|
||||
cmasks,
|
||||
output_hidden_states=True
|
||||
)
|
||||
return {"logits": logits[:, 1:vmasks.size(1)+1]}
|
||||
|
||||
|
||||
class MMFusionSeparateActionLocalization(MMFusionSeparate):
|
||||
def __init__(self, config, **kwargs):
|
||||
super().__init__(config)
|
||||
tokenizer = AutoTokenizer.from_pretrained(
|
||||
config.dataset.bert_name)
|
||||
self.cls_token_id = tokenizer.cls_token_id
|
||||
self.sep_token_id = tokenizer.sep_token_id
|
||||
self.pad_token_id = tokenizer.pad_token_id
|
||||
|
||||
def forward(
|
||||
self,
|
||||
caps,
|
||||
cmasks,
|
||||
vfeats,
|
||||
vmasks,
|
||||
**kwargs
|
||||
):
|
||||
# ActionLocalization assume of batch_size=1, squeeze it.
|
||||
caps = caps.squeeze(0)
|
||||
cmasks = cmasks.squeeze(0)
|
||||
vfeats = vfeats.squeeze(0)
|
||||
vmasks = vmasks.squeeze(0)
|
||||
|
||||
# TODO (huxu): other ways to do negative examples; move the following
|
||||
# into your criterion forward.
|
||||
dummy_caps = torch.LongTensor(
|
||||
[[self.cls_token_id, self.sep_token_id,
|
||||
self.pad_token_id, self.sep_token_id]],
|
||||
).to(caps.device).repeat(vfeats.size(0), 1)
|
||||
dummy_cmasks = torch.BoolTensor(
|
||||
[[0, 1, 0, 1]] # pad are valid for attention.
|
||||
).to(caps.device).repeat(vfeats.size(0), 1)
|
||||
|
||||
outputs = self.forward_video(
|
||||
vfeats,
|
||||
vmasks,
|
||||
dummy_caps,
|
||||
dummy_cmasks,
|
||||
output_hidden_states=True
|
||||
)
|
||||
|
||||
video_seq = outputs[:, 1:vmasks.size(1)+1].masked_select(
|
||||
vmasks.unsqueeze(-1)
|
||||
).view(-1, self.hidden_size)
|
||||
|
||||
pooled_text = self.forward_text(
|
||||
caps,
|
||||
cmasks,
|
||||
output_hidden_states=False
|
||||
)
|
||||
|
||||
# this line is not right.
|
||||
logits = torch.mm(video_seq, pooled_text.transpose(1, 0))
|
||||
return {"logits": logits}
|
||||
|
||||
|
||||
class MMFusionShareActionLocalization(MMFusionShare):
|
||||
def __init__(self, config, **kwargs):
|
||||
super().__init__(config)
|
||||
tokenizer = AutoTokenizer.from_pretrained(
|
||||
config.dataset.bert_name)
|
||||
self.cls_token_id = tokenizer.cls_token_id
|
||||
self.sep_token_id = tokenizer.sep_token_id
|
||||
self.pad_token_id = tokenizer.pad_token_id
|
||||
|
||||
def forward(
|
||||
self,
|
||||
caps,
|
||||
cmasks,
|
||||
vfeats,
|
||||
vmasks,
|
||||
**kwargs
|
||||
):
|
||||
# ActionLocalization assume of batch_size=1, squeeze it.
|
||||
caps = caps.squeeze(0)
|
||||
cmasks = cmasks.squeeze(0)
|
||||
vfeats = vfeats.squeeze(0)
|
||||
vmasks = vmasks.squeeze(0)
|
||||
|
||||
# TODO (huxu): other ways to do negative examples; move the following
|
||||
# into your criterion forward.
|
||||
dummy_caps = torch.LongTensor(
|
||||
[[self.cls_token_id, self.sep_token_id,
|
||||
self.pad_token_id, self.sep_token_id]],
|
||||
).to(caps.device).repeat(vfeats.size(0), 1)
|
||||
dummy_cmasks = torch.BoolTensor(
|
||||
[[0, 1, 0, 1]] # pad are valid for attention.
|
||||
).to(caps.device).repeat(vfeats.size(0), 1)
|
||||
|
||||
outputs = self.forward_video(
|
||||
vfeats,
|
||||
vmasks,
|
||||
dummy_caps,
|
||||
dummy_cmasks,
|
||||
output_hidden_states=True
|
||||
)
|
||||
|
||||
video_seq = outputs[:, 1:vmasks.size(1)+1].masked_select(
|
||||
vmasks.unsqueeze(-1)
|
||||
).view(-1, self.hidden_size)
|
||||
|
||||
pooled_text = self.forward_text(
|
||||
caps,
|
||||
cmasks,
|
||||
output_hidden_states=False
|
||||
)
|
||||
|
||||
# this line is not right.
|
||||
logits = torch.mm(video_seq, pooled_text.transpose(1, 0))
|
||||
return {"logits": logits}
|
999
examples/MMPT/mmpt/models/mmfusionnlg.py
Normal file
999
examples/MMPT/mmpt/models/mmfusionnlg.py
Normal file
@ -0,0 +1,999 @@
|
||||
# coding=utf-8
|
||||
# Copyright 2018 The Google AI Language Team Authors, Facebook AI Research authors and The HuggingFace Inc. team.
|
||||
# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# Copyright (c) Facebook, Inc. All Rights Reserved
|
||||
|
||||
|
||||
import torch
|
||||
|
||||
from torch.nn import functional as F
|
||||
|
||||
from typing import Optional, Iterable
|
||||
|
||||
try:
|
||||
from transformers import BertPreTrainedModel
|
||||
from transformers.modeling_bert import BertOnlyMLMHead
|
||||
|
||||
from transformers.file_utils import ModelOutput
|
||||
from transformers.modeling_outputs import CausalLMOutput
|
||||
from transformers.generation_utils import (
|
||||
BeamHypotheses,
|
||||
top_k_top_p_filtering
|
||||
)
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
from .mmfusion import MMFusion
|
||||
from .transformermodel import MMBertModel
|
||||
from ..modules import VideoTokenMLP
|
||||
|
||||
|
||||
class MMFusionNLG(MMFusion):
|
||||
def __init__(self, config, **kwargs):
|
||||
super().__init__(config)
|
||||
if config.model.max_decode_length is not None:
|
||||
self.max_length = min(
|
||||
config.model.max_decode_length,
|
||||
config.dataset.max_len - config.dataset.max_video_len - 3
|
||||
)
|
||||
else:
|
||||
self.max_length = \
|
||||
config.dataset.max_len - config.dataset.max_video_len - 3
|
||||
self.gen_param = config.gen_param if config.gen_param is not None \
|
||||
else {}
|
||||
|
||||
def forward(
|
||||
self,
|
||||
caps,
|
||||
cmasks,
|
||||
vfeats,
|
||||
vmasks,
|
||||
attention_mask,
|
||||
video_label=None,
|
||||
text_label=None,
|
||||
**kwargs
|
||||
):
|
||||
"""use pre-trained LM header for generation."""
|
||||
attention_mask, token_type_ids = self._mm_on_the_fly(
|
||||
cmasks, vmasks, attention_mask)
|
||||
|
||||
outputs = self.mm_encoder(
|
||||
input_ids=caps,
|
||||
input_video_embeds=vfeats,
|
||||
attention_mask=attention_mask,
|
||||
token_type_ids=token_type_ids,
|
||||
masked_lm_labels=text_label,
|
||||
)
|
||||
return {"logits": outputs[0]}
|
||||
|
||||
@torch.no_grad()
|
||||
def generate(
|
||||
self,
|
||||
caps, cmasks, vfeats, vmasks,
|
||||
attention_mask=None,
|
||||
bos_token_id=None,
|
||||
eos_token_id=None,
|
||||
**kwargs
|
||||
):
|
||||
# a simplified interface from
|
||||
# https://huggingface.co/transformers/v3.4.0/_modules/transformers/generation_utils.html#GenerationMixin.generate
|
||||
|
||||
# caps now only have
|
||||
# [CLS], [SEP] (for video) and [CLS] (as bos_token)
|
||||
assert caps.size(1) == 3
|
||||
|
||||
attention_mask, token_type_ids = self._mm_on_the_fly(
|
||||
cmasks, vmasks, attention_mask)
|
||||
|
||||
output = self.mm_encoder.generate(
|
||||
input_ids=caps,
|
||||
input_video_embeds=vfeats,
|
||||
attention_mask=attention_mask,
|
||||
token_type_ids=token_type_ids,
|
||||
bos_token_id=bos_token_id,
|
||||
eos_token_id=eos_token_id,
|
||||
max_length=self.max_length,
|
||||
**self.gen_param
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
class MMBertForNLG(BertPreTrainedModel):
|
||||
def __init__(self, config):
|
||||
super().__init__(config)
|
||||
self.bert = MMBertModel(config)
|
||||
self.videomlp = VideoTokenMLP(config)
|
||||
# we do not use `BertGenerationOnlyLMHead`
|
||||
# because we can reuse pretraining.
|
||||
self.cls = BertOnlyMLMHead(config)
|
||||
self.hidden_size = config.hidden_size
|
||||
self.init_weights()
|
||||
|
||||
def get_output_embeddings(self):
|
||||
return self.cls.predictions.decoder
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids=None,
|
||||
input_video_embeds=None,
|
||||
attention_mask=None,
|
||||
token_type_ids=None,
|
||||
position_ids=None,
|
||||
head_mask=None,
|
||||
inputs_embeds=None,
|
||||
masked_lm_labels=None,
|
||||
output_attentions=None,
|
||||
output_hidden_states=None,
|
||||
return_dict=None,
|
||||
):
|
||||
# similar to MMBertForMFMMLM without MFM.
|
||||
video_tokens = self.videomlp(input_video_embeds)
|
||||
outputs = self.bert(
|
||||
input_ids,
|
||||
video_tokens,
|
||||
attention_mask=attention_mask,
|
||||
token_type_ids=token_type_ids,
|
||||
position_ids=position_ids,
|
||||
head_mask=head_mask,
|
||||
inputs_embeds=inputs_embeds,
|
||||
output_attentions=output_attentions,
|
||||
output_hidden_states=output_hidden_states,
|
||||
return_dict=return_dict,
|
||||
)
|
||||
|
||||
sequence_output = outputs[0]
|
||||
|
||||
prediction_scores = None
|
||||
if masked_lm_labels is not None:
|
||||
text_offset = input_video_embeds.size(1) + 1 # [CLS]
|
||||
# recover caps format: [CLS] [SEP] text [SEP]
|
||||
text_sequence_output = torch.cat(
|
||||
[sequence_output[:, :1], sequence_output[:, text_offset:]],
|
||||
dim=1
|
||||
)
|
||||
|
||||
# only compute select tokens to training to speed up.
|
||||
hidden_size = text_sequence_output.size(-1)
|
||||
# masked_lm_labels = masked_lm_labels.reshape(-1)
|
||||
labels_mask = masked_lm_labels != -100
|
||||
|
||||
selected_text_output = text_sequence_output.masked_select(
|
||||
labels_mask.unsqueeze(-1)
|
||||
).view(-1, hidden_size)
|
||||
prediction_scores = self.cls(selected_text_output)
|
||||
|
||||
if not return_dict:
|
||||
output = (
|
||||
prediction_scores,
|
||||
) + outputs[2:]
|
||||
return output
|
||||
|
||||
# for generation.
|
||||
text_offset = input_video_embeds.size(1) + 2 # [CLS]
|
||||
text_sequence_output = sequence_output[:, text_offset:]
|
||||
prediction_scores = self.cls(text_sequence_output)
|
||||
return CausalLMOutput(
|
||||
loss=None,
|
||||
logits=prediction_scores,
|
||||
)
|
||||
|
||||
def prepare_inputs_for_generation(
|
||||
self,
|
||||
input_ids,
|
||||
input_video_embeds,
|
||||
attention_mask=None,
|
||||
token_type_ids=None,
|
||||
**model_kwargs
|
||||
):
|
||||
# must return a dictionary.
|
||||
seq_len = input_ids.size(1) + input_video_embeds.size(1)
|
||||
if attention_mask is not None:
|
||||
if len(attention_mask.size()) == 4:
|
||||
attention_mask = attention_mask[:, :, :seq_len, :seq_len]
|
||||
elif len(attention_mask.size()) == 3:
|
||||
attention_mask = attention_mask[:, :seq_len, :seq_len]
|
||||
else:
|
||||
attention_mask = attention_mask[:, :seq_len]
|
||||
if token_type_ids is not None:
|
||||
token_type_ids = token_type_ids[:, :seq_len]
|
||||
|
||||
return {
|
||||
"input_ids": input_ids,
|
||||
"input_video_embeds": input_video_embeds,
|
||||
"attention_mask": attention_mask,
|
||||
"token_type_ids": token_type_ids,
|
||||
}
|
||||
|
||||
@torch.no_grad()
|
||||
def generate(
|
||||
self,
|
||||
input_ids: Optional[torch.LongTensor] = None,
|
||||
decoder_input_ids: Optional[torch.LongTensor] = None,
|
||||
max_length: Optional[int] = None,
|
||||
min_length: Optional[int] = None,
|
||||
do_sample: Optional[bool] = None,
|
||||
early_stopping: Optional[bool] = None,
|
||||
num_beams: Optional[int] = None,
|
||||
temperature: Optional[float] = None,
|
||||
top_k: Optional[int] = None,
|
||||
top_p: Optional[float] = None,
|
||||
repetition_penalty: Optional[float] = None,
|
||||
bad_words_ids: Optional[Iterable[int]] = None,
|
||||
bos_token_id: Optional[int] = None,
|
||||
pad_token_id: Optional[int] = None,
|
||||
eos_token_id: Optional[int] = None,
|
||||
length_penalty: Optional[float] = None,
|
||||
no_repeat_ngram_size: Optional[int] = None,
|
||||
num_return_sequences: Optional[int] = None,
|
||||
attention_mask: Optional[torch.LongTensor] = None,
|
||||
decoder_start_token_id: Optional[int] = None,
|
||||
use_cache: Optional[bool] = None,
|
||||
**model_kwargs
|
||||
) -> torch.LongTensor:
|
||||
r"""
|
||||
Generates sequences for models with a language modeling head. The method currently supports greedy decoding,
|
||||
beam-search decoding, sampling with temperature, sampling with top-k or nucleus sampling.
|
||||
Adapted in part from `Facebook's XLM beam search code
|
||||
<https://github.com/facebookresearch/XLM/blob/9e6f6814d17be4fe5b15f2e6c43eb2b2d76daeb4/src/model/transformer.py#L529>`__.
|
||||
Apart from :obj:`input_ids` and :obj:`attention_mask`, all the arguments below will default to the value of the
|
||||
attribute of the same name inside the :class:`~transformers.PretrainedConfig` of the model. The default values
|
||||
indicated are the default values of those config.
|
||||
Most of these parameters are explained in more detail in `this blog post
|
||||
<https://huggingface.co/blog/how-to-generate>`__.
|
||||
Parameters:
|
||||
input_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`):
|
||||
The sequence used as a prompt for the generation. If :obj:`None` the method initializes
|
||||
it as an empty :obj:`torch.LongTensor` of shape :obj:`(1,)`.
|
||||
decoder_input_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`):
|
||||
initial input_ids for the decoder of encoder-decoder type models. If :obj:`None` then only
|
||||
decoder_start_token_id is passed as the first token to the decoder.
|
||||
max_length (:obj:`int`, `optional`, defaults to 20):
|
||||
The maximum length of the sequence to be generated.
|
||||
min_length (:obj:`int`, `optional`, defaults to 10):
|
||||
The minimum length of the sequence to be generated.
|
||||
do_sample (:obj:`bool`, `optional`, defaults to :obj:`False`):
|
||||
Whether or not to use sampling ; use greedy decoding otherwise.
|
||||
early_stopping (:obj:`bool`, `optional`, defaults to :obj:`False`):
|
||||
Whether to stop the beam search when at least ``num_beams`` sentences are finished per batch or not.
|
||||
num_beams (:obj:`int`, `optional`, defaults to 1):
|
||||
Number of beams for beam search. 1 means no beam search.
|
||||
temperature (:obj:`float`, `optional`, defaults tp 1.0):
|
||||
The value used to module the next token probabilities.
|
||||
top_k (:obj:`int`, `optional`, defaults to 50):
|
||||
The number of highest probability vocabulary tokens to keep for top-k-filtering.
|
||||
top_p (:obj:`float`, `optional`, defaults to 1.0):
|
||||
If set to float < 1, only the most probable tokens with probabilities that add up to ``top_p`` or
|
||||
higher are kept for generation.
|
||||
repetition_penalty (:obj:`float`, `optional`, defaults to 1.0):
|
||||
The parameter for repetition penalty. 1.0 means no penalty. See `this paper
|
||||
<https://arxiv.org/pdf/1909.05858.pdf>`__ for more details.
|
||||
pad_token_id (:obj:`int`, `optional`):
|
||||
The id of the `padding` token.
|
||||
bos_token_id (:obj:`int`, `optional`):
|
||||
The id of the `beginning-of-sequence` token.
|
||||
eos_token_id (:obj:`int`, `optional`):
|
||||
The id of the `end-of-sequence` token.
|
||||
length_penalty (:obj:`float`, `optional`, defaults to 1.0):
|
||||
Exponential penalty to the length. 1.0 means no penalty.
|
||||
Set to values < 1.0 in order to encourage the model to generate shorter sequences, to a value > 1.0 in
|
||||
order to encourage the model to produce longer sequences.
|
||||
no_repeat_ngram_size (:obj:`int`, `optional`, defaults to 0):
|
||||
If set to int > 0, all ngrams of that size can only occur once.
|
||||
bad_words_ids(:obj:`List[int]`, `optional`):
|
||||
List of token ids that are not allowed to be generated. In order to get the tokens of the words that
|
||||
should not appear in the generated text, use :obj:`tokenizer.encode(bad_word, add_prefix_space=True)`.
|
||||
num_return_sequences(:obj:`int`, `optional`, defaults to 1):
|
||||
The number of independently computed returned sequences for each element in the batch.
|
||||
attention_mask (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`):
|
||||
Mask to avoid performing attention on padding token indices. Mask values are in ``[0, 1]``, 1 for
|
||||
tokens that are not masked, and 0 for masked tokens.
|
||||
If not provided, will default to a tensor the same shape as :obj:`input_ids` that masks the pad token.
|
||||
`What are attention masks? <../glossary.html#attention-mask>`__
|
||||
decoder_start_token_id (:obj:`int`, `optional`):
|
||||
If an encoder-decoder model starts decoding with a different token than `bos`, the id of that token.
|
||||
use_cache: (:obj:`bool`, `optional`, defaults to :obj:`True`):
|
||||
Whether or not the model should use the past last key/values attentions (if applicable to the model) to
|
||||
speed up decoding.
|
||||
model_kwargs:
|
||||
Additional model specific kwargs will be forwarded to the :obj:`forward` function of the model.
|
||||
Return:
|
||||
:obj:`torch.LongTensor` of shape :obj:`(batch_size * num_return_sequences, sequence_length)`:
|
||||
The generated sequences. The second dimension (sequence_length) is either equal to :obj:`max_length` or
|
||||
shorter if all batches finished early due to the :obj:`eos_token_id`.
|
||||
Examples::
|
||||
tokenizer = AutoTokenizer.from_pretrained('distilgpt2') # Initialize tokenizer
|
||||
model = AutoModelWithLMHead.from_pretrained('distilgpt2') # Download model and configuration from S3 and cache.
|
||||
outputs = model.generate(max_length=40) # do greedy decoding
|
||||
print('Generated: {}'.format(tokenizer.decode(outputs[0], skip_special_tokens=True)))
|
||||
tokenizer = AutoTokenizer.from_pretrained('openai-gpt') # Initialize tokenizer
|
||||
model = AutoModelWithLMHead.from_pretrained('openai-gpt') # Download model and configuration from S3 and cache.
|
||||
input_context = 'The dog'
|
||||
input_ids = tokenizer.encode(input_context, return_tensors='pt') # encode input context
|
||||
outputs = model.generate(input_ids=input_ids, num_beams=5, num_return_sequences=3, temperature=1.5) # generate 3 independent sequences using beam search decoding (5 beams) with sampling from initial context 'The dog'
|
||||
for i in range(3): # 3 output sequences were generated
|
||||
print('Generated {}: {}'.format(i, tokenizer.decode(outputs[i], skip_special_tokens=True)))
|
||||
tokenizer = AutoTokenizer.from_pretrained('distilgpt2') # Initialize tokenizer
|
||||
model = AutoModelWithLMHead.from_pretrained('distilgpt2') # Download model and configuration from S3 and cache.
|
||||
input_context = 'The dog'
|
||||
input_ids = tokenizer.encode(input_context, return_tensors='pt') # encode input context
|
||||
outputs = model.generate(input_ids=input_ids, max_length=40, temperature=0.7, num_return_sequences=3, do_sample=True) # generate 3 candidates using sampling
|
||||
for i in range(3): # 3 output sequences were generated
|
||||
print('Generated {}: {}'.format(i, tokenizer.decode(outputs[i], skip_special_tokens=True)))
|
||||
tokenizer = AutoTokenizer.from_pretrained('ctrl') # Initialize tokenizer
|
||||
model = AutoModelWithLMHead.from_pretrained('ctrl') # Download model and configuration from S3 and cache.
|
||||
input_context = 'Legal My neighbor is' # "Legal" is one of the control codes for ctrl
|
||||
input_ids = tokenizer.encode(input_context, return_tensors='pt') # encode input context
|
||||
outputs = model.generate(input_ids=input_ids, max_length=50, temperature=0.7, repetition_penalty=1.2) # generate sequences
|
||||
print('Generated: {}'.format(tokenizer.decode(outputs[0], skip_special_tokens=True)))
|
||||
tokenizer = AutoTokenizer.from_pretrained('gpt2') # Initialize tokenizer
|
||||
model = AutoModelWithLMHead.from_pretrained('gpt2') # Download model and configuration from S3 and cache.
|
||||
input_context = 'My cute dog' # "Legal" is one of the control codes for ctrl
|
||||
bad_words_ids = [tokenizer.encode(bad_word, add_prefix_space=True) for bad_word in ['idiot', 'stupid', 'shut up']]
|
||||
input_ids = tokenizer.encode(input_context, return_tensors='pt') # encode input context
|
||||
outputs = model.generate(input_ids=input_ids, max_length=100, do_sample=True, bad_words_ids=bad_words_ids) # generate sequences without allowing bad_words to be generated
|
||||
"""
|
||||
|
||||
# We cannot generate if the model does not have a LM head
|
||||
if self.get_output_embeddings() is None:
|
||||
raise AttributeError(
|
||||
"You tried to generate sequences with a model that does not have a LM Head."
|
||||
"Please use another model class (e.g. `OpenAIGPTLMHeadModel`, `XLNetLMHeadModel`, `GPT2LMHeadModel`, `CTRLLMHeadModel`, `T5WithLMHeadModel`, `TransfoXLLMHeadModel`, `XLMWithLMHeadModel`, `BartForConditionalGeneration` )"
|
||||
)
|
||||
|
||||
max_length = max_length if max_length is not None else self.config.max_length
|
||||
min_length = min_length if min_length is not None else self.config.min_length
|
||||
do_sample = do_sample if do_sample is not None else self.config.do_sample
|
||||
early_stopping = early_stopping if early_stopping is not None else self.config.early_stopping
|
||||
use_cache = use_cache if use_cache is not None else self.config.use_cache
|
||||
num_beams = num_beams if num_beams is not None else self.config.num_beams
|
||||
temperature = temperature if temperature is not None else self.config.temperature
|
||||
top_k = top_k if top_k is not None else self.config.top_k
|
||||
top_p = top_p if top_p is not None else self.config.top_p
|
||||
repetition_penalty = repetition_penalty if repetition_penalty is not None else self.config.repetition_penalty
|
||||
bos_token_id = bos_token_id if bos_token_id is not None else self.config.bos_token_id
|
||||
pad_token_id = pad_token_id if pad_token_id is not None else self.config.pad_token_id
|
||||
eos_token_id = eos_token_id if eos_token_id is not None else self.config.eos_token_id
|
||||
length_penalty = length_penalty if length_penalty is not None else self.config.length_penalty
|
||||
no_repeat_ngram_size = (
|
||||
no_repeat_ngram_size if no_repeat_ngram_size is not None else self.config.no_repeat_ngram_size
|
||||
)
|
||||
bad_words_ids = bad_words_ids if bad_words_ids is not None else self.config.bad_words_ids
|
||||
num_return_sequences = (
|
||||
num_return_sequences if num_return_sequences is not None else self.config.num_return_sequences
|
||||
)
|
||||
decoder_start_token_id = (
|
||||
decoder_start_token_id if decoder_start_token_id is not None else self.config.decoder_start_token_id
|
||||
)
|
||||
|
||||
if input_ids is not None:
|
||||
batch_size = input_ids.shape[0] # overriden by the input batch_size
|
||||
else:
|
||||
batch_size = 1
|
||||
|
||||
assert isinstance(max_length, int) and max_length > 0, "`max_length` should be a strictly positive integer."
|
||||
assert isinstance(min_length, int) and min_length >= 0, "`min_length` should be a positive integer."
|
||||
assert isinstance(do_sample, bool), "`do_sample` should be a boolean."
|
||||
assert isinstance(early_stopping, bool), "`early_stopping` should be a boolean."
|
||||
assert isinstance(use_cache, bool), "`use_cache` should be a boolean."
|
||||
assert isinstance(num_beams, int) and num_beams > 0, "`num_beams` should be a strictly positive integer."
|
||||
assert temperature > 0, "`temperature` should be strictly positive."
|
||||
assert isinstance(top_k, int) and top_k >= 0, "`top_k` should be a positive integer."
|
||||
assert 0 <= top_p <= 1, "`top_p` should be between 0 and 1."
|
||||
assert repetition_penalty >= 1.0, "`repetition_penalty` should be >= 1."
|
||||
assert input_ids is not None or (
|
||||
isinstance(bos_token_id, int) and bos_token_id >= 0
|
||||
), "If input_ids is not defined, `bos_token_id` should be a positive integer."
|
||||
assert pad_token_id is None or (
|
||||
isinstance(pad_token_id, int) and (pad_token_id >= 0)
|
||||
), "`pad_token_id` should be a positive integer."
|
||||
assert (eos_token_id is None) or (
|
||||
isinstance(eos_token_id, int) and (eos_token_id >= 0)
|
||||
), "`eos_token_id` should be a positive integer."
|
||||
assert length_penalty > 0, "`length_penalty` should be strictly positive."
|
||||
assert (
|
||||
isinstance(no_repeat_ngram_size, int) and no_repeat_ngram_size >= 0
|
||||
), "`no_repeat_ngram_size` should be a positive integer."
|
||||
assert (
|
||||
isinstance(num_return_sequences, int) and num_return_sequences > 0
|
||||
), "`num_return_sequences` should be a strictly positive integer."
|
||||
assert (
|
||||
bad_words_ids is None or isinstance(bad_words_ids, list) and isinstance(bad_words_ids[0], list)
|
||||
), "`bad_words_ids` is either `None` or a list of lists of tokens that should not be generated"
|
||||
|
||||
if input_ids is None:
|
||||
assert isinstance(bos_token_id, int) and bos_token_id >= 0, (
|
||||
"you should either supply a context to complete as `input_ids` input "
|
||||
"or a `bos_token_id` (integer >= 0) as a first token to start the generation."
|
||||
)
|
||||
input_ids = torch.full(
|
||||
(batch_size, 1),
|
||||
bos_token_id,
|
||||
dtype=torch.long,
|
||||
device=next(self.parameters()).device,
|
||||
)
|
||||
else:
|
||||
assert input_ids.dim() == 2, "Input prompt should be of shape (batch_size, sequence length)."
|
||||
|
||||
# not allow to duplicate outputs when greedy decoding
|
||||
if do_sample is False:
|
||||
if num_beams == 1:
|
||||
# no_beam_search greedy generation conditions
|
||||
assert (
|
||||
num_return_sequences == 1
|
||||
), "Greedy decoding will always produce the same output for num_beams == 1 and num_return_sequences > 1. Please set num_return_sequences = 1"
|
||||
|
||||
else:
|
||||
# beam_search greedy generation conditions
|
||||
assert (
|
||||
num_beams >= num_return_sequences
|
||||
), "Greedy beam search decoding cannot return more sequences than it has beams. Please set num_beams >= num_return_sequences"
|
||||
|
||||
# create attention mask if necessary
|
||||
# TODO (PVP): this should later be handled by the forward fn() in each model in the future see PR 3140
|
||||
if (attention_mask is None) and (pad_token_id is not None) and (pad_token_id in input_ids):
|
||||
attention_mask = input_ids.ne(pad_token_id).long()
|
||||
elif attention_mask is None:
|
||||
attention_mask = input_ids.new_ones(input_ids.shape)
|
||||
|
||||
# set pad_token_id to eos_token_id if not set. Important that this is done after
|
||||
# attention_mask is created
|
||||
if pad_token_id is None and eos_token_id is not None:
|
||||
print(
|
||||
"Setting `pad_token_id` to {} (first `eos_token_id`) to generate sequence".format(eos_token_id)
|
||||
)
|
||||
pad_token_id = eos_token_id
|
||||
|
||||
# vocab size
|
||||
if hasattr(self.config, "vocab_size"):
|
||||
vocab_size = self.config.vocab_size
|
||||
elif (
|
||||
self.config.is_encoder_decoder
|
||||
and hasattr(self.config, "decoder")
|
||||
and hasattr(self.config.decoder, "vocab_size")
|
||||
):
|
||||
vocab_size = self.config.decoder.vocab_size
|
||||
else:
|
||||
raise ValueError("either self.config.vocab_size or self.config.decoder.vocab_size needs to be defined")
|
||||
|
||||
# set effective batch size and effective batch multiplier according to do_sample
|
||||
if do_sample:
|
||||
effective_batch_size = batch_size * num_return_sequences
|
||||
effective_batch_mult = num_return_sequences
|
||||
else:
|
||||
effective_batch_size = batch_size
|
||||
effective_batch_mult = 1
|
||||
|
||||
if self.config.is_encoder_decoder:
|
||||
if decoder_start_token_id is None:
|
||||
# see if BOS token can be used for decoder_start_token_id
|
||||
if bos_token_id is not None:
|
||||
decoder_start_token_id = bos_token_id
|
||||
elif (
|
||||
hasattr(self.config, "decoder")
|
||||
and hasattr(self.config.decoder, "bos_token_id")
|
||||
and self.config.decoder.bos_token_id is not None
|
||||
):
|
||||
decoder_start_token_id = self.config.decoder.bos_token_id
|
||||
else:
|
||||
raise ValueError(
|
||||
"decoder_start_token_id or bos_token_id has to be defined for encoder-decoder generation"
|
||||
)
|
||||
|
||||
assert hasattr(self, "get_encoder"), "{} should have a 'get_encoder' function defined".format(self)
|
||||
assert callable(self.get_encoder), "{} should be a method".format(self.get_encoder)
|
||||
|
||||
# get encoder and store encoder outputs
|
||||
encoder = self.get_encoder()
|
||||
encoder_outputs: ModelOutput = encoder(input_ids, attention_mask=attention_mask, return_dict=True)
|
||||
|
||||
# Expand input ids if num_beams > 1 or num_return_sequences > 1
|
||||
if num_return_sequences > 1 or num_beams > 1:
|
||||
# TODO: make this a call-back function.
|
||||
# input_ids=caps,
|
||||
# input_video_embeds=vfeats,
|
||||
# attention_mask=attention_mask,
|
||||
# token_type_ids=token_type_ids,
|
||||
input_video_embeds = model_kwargs.pop("input_video_embeds", None)
|
||||
token_type_ids = model_kwargs.pop("token_type_ids", None)
|
||||
|
||||
input_ids_len = input_ids.shape[-1]
|
||||
input_ids = input_ids.unsqueeze(1).expand(
|
||||
batch_size, effective_batch_mult * num_beams, input_ids_len)
|
||||
|
||||
input_video_embeds_len, input_video_embeds_hidden = input_video_embeds.size(1), input_video_embeds.size(2)
|
||||
input_video_embeds = input_video_embeds.unsqueeze(1).expand(
|
||||
batch_size, effective_batch_mult * num_beams, input_video_embeds_len, input_video_embeds_hidden)
|
||||
|
||||
attention_mask_from_len, attention_mask_to_len = attention_mask.size(1), attention_mask.size(2)
|
||||
attention_mask = attention_mask.unsqueeze(1).expand(
|
||||
batch_size, effective_batch_mult * num_beams, attention_mask_from_len, attention_mask_to_len
|
||||
)
|
||||
|
||||
token_type_ids_len = token_type_ids.size(1)
|
||||
token_type_ids = token_type_ids.unsqueeze(1).expand(
|
||||
batch_size, effective_batch_mult * num_beams, token_type_ids_len
|
||||
)
|
||||
|
||||
# contiguous ...
|
||||
input_ids = input_ids.contiguous().view(
|
||||
effective_batch_size * num_beams, input_ids_len
|
||||
) # shape: (batch_size * num_return_sequences * num_beams, cur_len)
|
||||
|
||||
input_video_embeds = input_video_embeds.contiguous().view(
|
||||
effective_batch_size * num_beams, input_video_embeds_len, input_video_embeds_hidden)
|
||||
|
||||
attention_mask = attention_mask.contiguous().view(
|
||||
effective_batch_size * num_beams, attention_mask_from_len, attention_mask_to_len
|
||||
) # shape: (batch_size * num_return_sequences * num_beams, cur_len)
|
||||
|
||||
token_type_ids = token_type_ids.contiguous().view(
|
||||
effective_batch_size * num_beams, token_type_ids_len
|
||||
)
|
||||
|
||||
model_kwargs["input_video_embeds"] = input_video_embeds
|
||||
model_kwargs["token_type_ids"] = token_type_ids
|
||||
|
||||
if self.config.is_encoder_decoder:
|
||||
device = next(self.parameters()).device
|
||||
if decoder_input_ids is not None:
|
||||
# give initial decoder input ids
|
||||
input_ids = decoder_input_ids.repeat(effective_batch_size * num_beams, 1).to(device)
|
||||
else:
|
||||
# create empty decoder input_ids
|
||||
input_ids = torch.full(
|
||||
(effective_batch_size * num_beams, 1),
|
||||
decoder_start_token_id,
|
||||
dtype=torch.long,
|
||||
device=device,
|
||||
)
|
||||
cur_len = input_ids.shape[-1]
|
||||
|
||||
assert (
|
||||
batch_size == encoder_outputs.last_hidden_state.shape[0]
|
||||
), f"expected encoder_outputs.last_hidden_state to have 1st dimension bs={batch_size}, got {encoder_outputs.last_hidden_state.shape[0]} "
|
||||
|
||||
# expand batch_idx to assign correct encoder output for expanded input_ids (due to num_beams > 1 and num_return_sequences > 1)
|
||||
expanded_batch_idxs = (
|
||||
torch.arange(batch_size)
|
||||
.view(-1, 1)
|
||||
.repeat(1, num_beams * effective_batch_mult)
|
||||
.view(-1)
|
||||
.to(input_ids.device)
|
||||
)
|
||||
|
||||
# expand encoder_outputs
|
||||
encoder_outputs["last_hidden_state"] = encoder_outputs.last_hidden_state.index_select(
|
||||
0, expanded_batch_idxs
|
||||
)
|
||||
|
||||
# save encoder_outputs in `model_kwargs`
|
||||
model_kwargs["encoder_outputs"] = encoder_outputs
|
||||
|
||||
else:
|
||||
cur_len = input_ids.shape[-1]
|
||||
|
||||
assert (
|
||||
cur_len < max_length
|
||||
), f"The context has {cur_len} number of tokens, but `max_length` is only {max_length}. Please make sure that `max_length` is bigger than the number of tokens, by setting either `generate(max_length=...,...)` or `config.max_length = ...`"
|
||||
|
||||
if num_beams > 1:
|
||||
output = self._generate_beam_search(
|
||||
input_ids,
|
||||
cur_len=cur_len,
|
||||
max_length=max_length,
|
||||
min_length=min_length,
|
||||
do_sample=do_sample,
|
||||
early_stopping=early_stopping,
|
||||
temperature=temperature,
|
||||
top_k=top_k,
|
||||
top_p=top_p,
|
||||
repetition_penalty=repetition_penalty,
|
||||
no_repeat_ngram_size=no_repeat_ngram_size,
|
||||
bad_words_ids=bad_words_ids,
|
||||
pad_token_id=pad_token_id,
|
||||
eos_token_id=eos_token_id,
|
||||
batch_size=effective_batch_size,
|
||||
num_return_sequences=num_return_sequences,
|
||||
length_penalty=length_penalty,
|
||||
num_beams=num_beams,
|
||||
vocab_size=vocab_size,
|
||||
attention_mask=attention_mask,
|
||||
use_cache=use_cache,
|
||||
model_kwargs=model_kwargs,
|
||||
)
|
||||
else:
|
||||
output = self._generate_no_beam_search(
|
||||
input_ids,
|
||||
cur_len=cur_len,
|
||||
max_length=max_length,
|
||||
min_length=min_length,
|
||||
do_sample=do_sample,
|
||||
temperature=temperature,
|
||||
top_k=top_k,
|
||||
top_p=top_p,
|
||||
repetition_penalty=repetition_penalty,
|
||||
no_repeat_ngram_size=no_repeat_ngram_size,
|
||||
bad_words_ids=bad_words_ids,
|
||||
pad_token_id=pad_token_id,
|
||||
eos_token_id=eos_token_id,
|
||||
batch_size=effective_batch_size,
|
||||
attention_mask=attention_mask,
|
||||
use_cache=use_cache,
|
||||
model_kwargs=model_kwargs,
|
||||
)
|
||||
|
||||
return output
|
||||
|
||||
def _generate_beam_search(
|
||||
self,
|
||||
input_ids,
|
||||
cur_len,
|
||||
max_length,
|
||||
min_length,
|
||||
do_sample,
|
||||
early_stopping,
|
||||
temperature,
|
||||
top_k,
|
||||
top_p,
|
||||
repetition_penalty,
|
||||
no_repeat_ngram_size,
|
||||
bad_words_ids,
|
||||
pad_token_id,
|
||||
eos_token_id,
|
||||
batch_size,
|
||||
num_return_sequences,
|
||||
length_penalty,
|
||||
num_beams,
|
||||
vocab_size,
|
||||
attention_mask,
|
||||
use_cache,
|
||||
model_kwargs,
|
||||
):
|
||||
"""Generate sequences for each example with beam search."""
|
||||
|
||||
# generated hypotheses
|
||||
generated_hyps = [
|
||||
BeamHypotheses(num_beams, max_length, length_penalty, early_stopping=early_stopping)
|
||||
for _ in range(batch_size)
|
||||
]
|
||||
|
||||
# scores for each sentence in the beam
|
||||
beam_scores = torch.zeros((batch_size, num_beams), dtype=torch.float, device=input_ids.device)
|
||||
|
||||
# for greedy decoding it is made sure that only tokens of the first beam are considered to avoid sampling the exact same tokens three times
|
||||
if do_sample is False:
|
||||
beam_scores[:, 1:] = -1e9
|
||||
beam_scores = beam_scores.view(-1) # shape (batch_size * num_beams,)
|
||||
|
||||
# cache compute states
|
||||
past = None
|
||||
|
||||
# done sentences
|
||||
done = [False for _ in range(batch_size)]
|
||||
|
||||
while cur_len < max_length:
|
||||
model_inputs = self.prepare_inputs_for_generation(
|
||||
input_ids, past=past, attention_mask=attention_mask, use_cache=use_cache, **model_kwargs
|
||||
)
|
||||
outputs = self(**model_inputs, return_dict=True) # (batch_size * num_beams, cur_len, vocab_size)
|
||||
next_token_logits = outputs.logits[:, -1, :] # (batch_size * num_beams, vocab_size)
|
||||
|
||||
# if model has past, then set the past variable to speed up decoding
|
||||
if "past_key_values" in outputs:
|
||||
past = outputs.past_key_values
|
||||
elif "mems" in outputs:
|
||||
past = outputs.mems
|
||||
|
||||
if self.config.is_encoder_decoder and do_sample is False:
|
||||
# TODO (PVP) still a bit hacky here - there might be a better solution
|
||||
next_token_logits = self.adjust_logits_during_generation(
|
||||
next_token_logits, cur_len=cur_len, max_length=max_length
|
||||
)
|
||||
|
||||
scores = F.log_softmax(next_token_logits, dim=-1) # (batch_size * num_beams, vocab_size)
|
||||
|
||||
scores = self.postprocess_next_token_scores(
|
||||
scores=scores,
|
||||
input_ids=input_ids,
|
||||
no_repeat_ngram_size=no_repeat_ngram_size,
|
||||
bad_words_ids=bad_words_ids,
|
||||
cur_len=cur_len,
|
||||
min_length=min_length,
|
||||
max_length=max_length,
|
||||
eos_token_id=eos_token_id,
|
||||
repetition_penalty=repetition_penalty,
|
||||
batch_size=batch_size,
|
||||
num_beams=num_beams,
|
||||
)
|
||||
|
||||
assert scores.shape == (batch_size * num_beams, vocab_size), "Shapes of scores: {} != {}".format(
|
||||
scores.shape, (batch_size * num_beams, vocab_size)
|
||||
)
|
||||
|
||||
if do_sample:
|
||||
_scores = scores + beam_scores[:, None].expand_as(scores) # (batch_size * num_beams, vocab_size)
|
||||
# Temperature
|
||||
if temperature != 1.0:
|
||||
_scores = _scores / temperature
|
||||
# Top-p/top-k filtering
|
||||
_scores = top_k_top_p_filtering(
|
||||
_scores, top_k=top_k, top_p=top_p, min_tokens_to_keep=2
|
||||
) # (batch_size * num_beams, vocab_size)
|
||||
# re-organize to group the beam together to sample from all beam_idxs
|
||||
_scores = _scores.contiguous().view(
|
||||
batch_size, num_beams * vocab_size
|
||||
) # (batch_size, num_beams * vocab_size)
|
||||
|
||||
# Sample 2 next tokens for each beam (so we have some spare tokens and match output of greedy beam search)
|
||||
probs = F.softmax(_scores, dim=-1)
|
||||
next_tokens = torch.multinomial(probs, num_samples=2 * num_beams) # (batch_size, num_beams * 2)
|
||||
# Compute next scores
|
||||
next_scores = torch.gather(_scores, -1, next_tokens) # (batch_size, num_beams * 2)
|
||||
# sort the sampled vector to make sure that the first num_beams samples are the best
|
||||
next_scores, next_scores_indices = torch.sort(next_scores, descending=True, dim=1)
|
||||
next_tokens = torch.gather(next_tokens, -1, next_scores_indices) # (batch_size, num_beams * 2)
|
||||
|
||||
else:
|
||||
next_scores = scores + beam_scores[:, None].expand_as(scores) # (batch_size * num_beams, vocab_size)
|
||||
|
||||
# re-organize to group the beam together (we are keeping top hypothesis accross beams)
|
||||
next_scores = next_scores.view(
|
||||
batch_size, num_beams * vocab_size
|
||||
) # (batch_size, num_beams * vocab_size)
|
||||
|
||||
next_scores, next_tokens = torch.topk(next_scores, 2 * num_beams, dim=1, largest=True, sorted=True)
|
||||
|
||||
assert next_scores.size() == next_tokens.size() == (batch_size, 2 * num_beams)
|
||||
|
||||
# next batch beam content
|
||||
next_batch_beam = []
|
||||
|
||||
# for each sentence
|
||||
for batch_idx in range(batch_size):
|
||||
|
||||
# if we are done with this sentence, add a pad token
|
||||
if done[batch_idx]:
|
||||
assert (
|
||||
len(generated_hyps[batch_idx]) >= num_beams
|
||||
), "Batch can only be done if at least {} beams have been generated".format(num_beams)
|
||||
assert (
|
||||
eos_token_id is not None and pad_token_id is not None
|
||||
), "generated beams >= num_beams -> eos_token_id and pad_token have to be defined"
|
||||
next_batch_beam.extend([(0, pad_token_id, 0)] * num_beams) # pad the batch
|
||||
continue
|
||||
|
||||
# next sentence beam content, this will get added to next_batch_beam
|
||||
next_sent_beam = []
|
||||
|
||||
# next tokens for this sentence
|
||||
for beam_token_rank, (beam_token_id, beam_token_score) in enumerate(
|
||||
zip(next_tokens[batch_idx], next_scores[batch_idx])
|
||||
):
|
||||
# get beam and token IDs
|
||||
beam_id = beam_token_id // vocab_size
|
||||
token_id = beam_token_id % vocab_size
|
||||
|
||||
effective_beam_id = batch_idx * num_beams + beam_id
|
||||
# add to generated hypotheses if end of sentence
|
||||
if (eos_token_id is not None) and (token_id.item() == eos_token_id):
|
||||
# if beam_token does not belong to top num_beams tokens, it should not be added
|
||||
is_beam_token_worse_than_top_num_beams = beam_token_rank >= num_beams
|
||||
if is_beam_token_worse_than_top_num_beams:
|
||||
continue
|
||||
generated_hyps[batch_idx].add(
|
||||
input_ids[effective_beam_id].clone(),
|
||||
beam_token_score.item(),
|
||||
)
|
||||
else:
|
||||
# add next predicted token since it is not eos_token
|
||||
next_sent_beam.append((beam_token_score, token_id, effective_beam_id))
|
||||
|
||||
# once the beam for next step is full, don't add more tokens to it.
|
||||
if len(next_sent_beam) == num_beams:
|
||||
break
|
||||
|
||||
# Check if we are done so that we can save a pad step if all(done)
|
||||
done[batch_idx] = done[batch_idx] or generated_hyps[batch_idx].is_done(
|
||||
next_scores[batch_idx].max().item(), cur_len
|
||||
)
|
||||
|
||||
# update next beam content
|
||||
assert len(next_sent_beam) == num_beams, "Beam should always be full"
|
||||
next_batch_beam.extend(next_sent_beam)
|
||||
assert len(next_batch_beam) == num_beams * (batch_idx + 1), "We should have added num_beams each step"
|
||||
|
||||
# stop when we are done with each sentence
|
||||
if all(done):
|
||||
break
|
||||
|
||||
# sanity check / prepare next batch
|
||||
assert len(next_batch_beam) == batch_size * num_beams
|
||||
beam_scores = beam_scores.new([x[0] for x in next_batch_beam])
|
||||
beam_tokens = input_ids.new([x[1] for x in next_batch_beam])
|
||||
beam_idx = input_ids.new([x[2] for x in next_batch_beam])
|
||||
|
||||
# re-order batch and update current length
|
||||
input_ids = input_ids[beam_idx, :]
|
||||
input_ids = torch.cat([input_ids, beam_tokens.unsqueeze(1)], dim=-1)
|
||||
cur_len = cur_len + 1
|
||||
|
||||
# re-order internal states
|
||||
if past is not None:
|
||||
past = self._reorder_cache(past, beam_idx)
|
||||
|
||||
# extend attention_mask for new generated input if only decoder
|
||||
# (huxu): move out since we trim attention_mask by ourselves.
|
||||
# if self.config.is_encoder_decoder is False:
|
||||
# attention_mask = torch.cat(
|
||||
# [attention_mask, attention_mask.new_ones((attention_mask.shape[0], 1))], dim=-1
|
||||
# )
|
||||
|
||||
# finalize all open beam hypotheses and add to generated hypotheses
|
||||
for batch_idx in range(batch_size):
|
||||
if done[batch_idx]:
|
||||
continue
|
||||
|
||||
# test that beam scores match previously calculated scores if not eos and batch_idx not done
|
||||
if eos_token_id is not None and all(
|
||||
(token_id % vocab_size).item() != eos_token_id for token_id in next_tokens[batch_idx]
|
||||
):
|
||||
assert torch.all(
|
||||
next_scores[batch_idx, :num_beams] == beam_scores.view(batch_size, num_beams)[batch_idx]
|
||||
), "If batch_idx is not done, final next scores: {} have to equal to accumulated beam_scores: {}".format(
|
||||
next_scores[:, :num_beams][batch_idx],
|
||||
beam_scores.view(batch_size, num_beams)[batch_idx],
|
||||
)
|
||||
|
||||
# need to add best num_beams hypotheses to generated hyps
|
||||
for beam_id in range(num_beams):
|
||||
effective_beam_id = batch_idx * num_beams + beam_id
|
||||
final_score = beam_scores[effective_beam_id].item()
|
||||
final_tokens = input_ids[effective_beam_id]
|
||||
generated_hyps[batch_idx].add(final_tokens, final_score)
|
||||
|
||||
# depending on whether greedy generation is wanted or not define different output_batch_size and output_num_return_sequences_per_batch
|
||||
output_batch_size = batch_size if do_sample else batch_size * num_return_sequences
|
||||
output_num_return_sequences_per_batch = 1 if do_sample else num_return_sequences
|
||||
|
||||
# select the best hypotheses
|
||||
sent_lengths = input_ids.new(output_batch_size)
|
||||
best = []
|
||||
|
||||
# retrieve best hypotheses
|
||||
for i, hypotheses in enumerate(generated_hyps):
|
||||
sorted_hyps = sorted(hypotheses.beams, key=lambda x: x[0])
|
||||
for j in range(output_num_return_sequences_per_batch):
|
||||
effective_batch_idx = output_num_return_sequences_per_batch * i + j
|
||||
best_hyp = sorted_hyps.pop()[1]
|
||||
sent_lengths[effective_batch_idx] = len(best_hyp)
|
||||
best.append(best_hyp)
|
||||
|
||||
# prepare for adding eos
|
||||
sent_max_len = min(sent_lengths.max().item() + 1, max_length)
|
||||
decoded = input_ids.new(output_batch_size, sent_max_len)
|
||||
# shorter batches are padded if needed
|
||||
if sent_lengths.min().item() != sent_lengths.max().item():
|
||||
assert pad_token_id is not None, "`pad_token_id` has to be defined"
|
||||
decoded.fill_(pad_token_id)
|
||||
|
||||
# fill with hypotheses and eos_token_id if the latter fits in
|
||||
for i, hypo in enumerate(best):
|
||||
decoded[i, : sent_lengths[i]] = hypo
|
||||
if sent_lengths[i] < max_length:
|
||||
decoded[i, sent_lengths[i]] = eos_token_id
|
||||
|
||||
return decoded
|
||||
|
||||
def _generate_no_beam_search(
|
||||
self,
|
||||
input_ids,
|
||||
cur_len,
|
||||
max_length,
|
||||
min_length,
|
||||
do_sample,
|
||||
temperature,
|
||||
top_k,
|
||||
top_p,
|
||||
repetition_penalty,
|
||||
no_repeat_ngram_size,
|
||||
bad_words_ids,
|
||||
pad_token_id,
|
||||
eos_token_id,
|
||||
batch_size,
|
||||
attention_mask,
|
||||
use_cache,
|
||||
model_kwargs,
|
||||
):
|
||||
"""Generate sequences for each example without beam search (num_beams == 1).
|
||||
All returned sequence are generated independantly.
|
||||
"""
|
||||
# length of generated sentences / unfinished sentences
|
||||
unfinished_sents = input_ids.new(batch_size).fill_(1)
|
||||
sent_lengths = input_ids.new(batch_size).fill_(max_length)
|
||||
|
||||
past = None
|
||||
while cur_len < max_length:
|
||||
model_inputs = self.prepare_inputs_for_generation(
|
||||
input_ids, past=past, attention_mask=attention_mask, use_cache=use_cache, **model_kwargs
|
||||
)
|
||||
|
||||
outputs = self(**model_inputs, return_dict=True)
|
||||
next_token_logits = outputs.logits[:, -1, :]
|
||||
scores = self.postprocess_next_token_scores(
|
||||
scores=next_token_logits,
|
||||
input_ids=input_ids,
|
||||
no_repeat_ngram_size=no_repeat_ngram_size,
|
||||
bad_words_ids=bad_words_ids,
|
||||
cur_len=cur_len,
|
||||
min_length=min_length,
|
||||
max_length=max_length,
|
||||
eos_token_id=eos_token_id,
|
||||
repetition_penalty=repetition_penalty,
|
||||
batch_size=batch_size,
|
||||
num_beams=1,
|
||||
)
|
||||
|
||||
# if model has past, then set the past variable to speed up decoding
|
||||
if "past_key_values" in outputs:
|
||||
past = outputs.past_key_values
|
||||
elif "mems" in outputs:
|
||||
past = outputs.mems
|
||||
|
||||
if do_sample:
|
||||
# Temperature (higher temperature => more likely to sample low probability tokens)
|
||||
if temperature != 1.0:
|
||||
scores = scores / temperature
|
||||
# Top-p/top-k filtering
|
||||
next_token_logscores = top_k_top_p_filtering(scores, top_k=top_k, top_p=top_p)
|
||||
# Sample
|
||||
probs = F.softmax(next_token_logscores, dim=-1)
|
||||
next_token = torch.multinomial(probs, num_samples=1).squeeze(1)
|
||||
else:
|
||||
# Greedy decoding
|
||||
next_token = torch.argmax(next_token_logits, dim=-1)
|
||||
|
||||
# print(next_token_logits[0,next_token[0]], next_token_logits[0,eos_token_id])
|
||||
|
||||
# update generations and finished sentences
|
||||
if eos_token_id is not None:
|
||||
# pad finished sentences if eos_token_id exist
|
||||
tokens_to_add = next_token * unfinished_sents + (pad_token_id) * (1 - unfinished_sents)
|
||||
else:
|
||||
tokens_to_add = next_token
|
||||
|
||||
# add token and increase length by one
|
||||
input_ids = torch.cat([input_ids, tokens_to_add.unsqueeze(-1)], dim=-1)
|
||||
cur_len = cur_len + 1
|
||||
|
||||
if eos_token_id is not None:
|
||||
eos_in_sents = tokens_to_add == eos_token_id
|
||||
# if sentence is unfinished and the token to add is eos, sent_lengths is filled with current length
|
||||
is_sents_unfinished_and_token_to_add_is_eos = unfinished_sents.mul(eos_in_sents.long()).bool()
|
||||
sent_lengths.masked_fill_(is_sents_unfinished_and_token_to_add_is_eos, cur_len)
|
||||
# unfinished_sents is set to zero if eos in sentence
|
||||
unfinished_sents.mul_((~eos_in_sents).long())
|
||||
|
||||
# stop when there is a </s> in each sentence, or if we exceed the maximul length
|
||||
if unfinished_sents.max() == 0:
|
||||
break
|
||||
|
||||
|
||||
# extend attention_mask for new generated input if only decoder
|
||||
# if self.config.is_encoder_decoder is False:
|
||||
# attention_mask = torch.cat(
|
||||
# [attention_mask, attention_mask.new_ones((attention_mask.shape[0], 1))], dim=-1
|
||||
# )
|
||||
|
||||
return input_ids
|
734
examples/MMPT/mmpt/models/transformermodel.py
Normal file
734
examples/MMPT/mmpt/models/transformermodel.py
Normal file
@ -0,0 +1,734 @@
|
||||
# coding=utf-8
|
||||
# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team.
|
||||
# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# Copyright (c) Facebook, Inc. All Rights Reserved
|
||||
|
||||
import torch
|
||||
|
||||
from torch import nn
|
||||
|
||||
try:
|
||||
from transformers.modeling_bert import (
|
||||
BertPreTrainedModel,
|
||||
BertModel,
|
||||
BertEncoder,
|
||||
BertPredictionHeadTransform,
|
||||
)
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
from ..modules import VideoTokenMLP, MMBertEmbeddings
|
||||
|
||||
|
||||
# --------------- fine-tuning models ---------------
|
||||
class MMBertForJoint(BertPreTrainedModel):
|
||||
"""A BertModel with isolated attention mask to separate modality."""
|
||||
|
||||
def __init__(self, config):
|
||||
super().__init__(config)
|
||||
self.videomlp = VideoTokenMLP(config)
|
||||
self.bert = MMBertModel(config)
|
||||
self.init_weights()
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids=None,
|
||||
input_video_embeds=None,
|
||||
attention_mask=None,
|
||||
token_type_ids=None,
|
||||
position_ids=None,
|
||||
head_mask=None,
|
||||
inputs_embeds=None,
|
||||
next_sentence_label=None,
|
||||
output_attentions=None,
|
||||
output_hidden_states=None,
|
||||
return_dict=None,
|
||||
separate_forward_split=None,
|
||||
):
|
||||
return_dict = (
|
||||
return_dict if return_dict is not None
|
||||
else self.config.use_return_dict
|
||||
)
|
||||
video_tokens = self.videomlp(input_video_embeds)
|
||||
|
||||
outputs = self.bert(
|
||||
input_ids,
|
||||
video_tokens,
|
||||
attention_mask=attention_mask,
|
||||
token_type_ids=token_type_ids,
|
||||
position_ids=position_ids,
|
||||
head_mask=head_mask,
|
||||
inputs_embeds=inputs_embeds,
|
||||
output_attentions=output_attentions,
|
||||
output_hidden_states=output_hidden_states,
|
||||
return_dict=return_dict,
|
||||
separate_forward_split=separate_forward_split,
|
||||
)
|
||||
|
||||
return outputs
|
||||
|
||||
|
||||
class MMBertForTokenClassification(BertPreTrainedModel):
|
||||
"""A BertModel similar to MMJointUni, with extra wrapper layer
|
||||
to be fine-tuned from other pretrained MMFusion model."""
|
||||
|
||||
def __init__(self, config):
|
||||
super().__init__(config)
|
||||
self.videomlp = VideoTokenMLP(config)
|
||||
self.bert = MMBertModel(config)
|
||||
self.dropout = nn.Dropout(config.hidden_dropout_prob)
|
||||
# TODO(huxu): 779 is the number of classes for COIN: move to config?
|
||||
self.classifier = nn.Linear(config.hidden_size, 779)
|
||||
self.init_weights()
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids=None,
|
||||
input_video_embeds=None,
|
||||
attention_mask=None,
|
||||
token_type_ids=None,
|
||||
position_ids=None,
|
||||
head_mask=None,
|
||||
inputs_embeds=None,
|
||||
next_sentence_label=None,
|
||||
output_attentions=None,
|
||||
output_hidden_states=None,
|
||||
return_dict=None,
|
||||
separate_forward_split=None,
|
||||
):
|
||||
return_dict = (
|
||||
return_dict if return_dict is not None
|
||||
else self.config.use_return_dict
|
||||
)
|
||||
|
||||
video_tokens = self.videomlp(input_video_embeds)
|
||||
outputs = self.bert(
|
||||
input_ids,
|
||||
video_tokens,
|
||||
attention_mask=attention_mask,
|
||||
token_type_ids=token_type_ids,
|
||||
position_ids=position_ids,
|
||||
head_mask=head_mask,
|
||||
inputs_embeds=inputs_embeds,
|
||||
output_attentions=output_attentions,
|
||||
output_hidden_states=output_hidden_states,
|
||||
return_dict=return_dict,
|
||||
separate_forward_split=separate_forward_split,
|
||||
)
|
||||
|
||||
return (self.classifier(outputs[0]),)
|
||||
|
||||
|
||||
# ------------ pre-training models ----------------
|
||||
|
||||
class MMBertForEncoder(BertPreTrainedModel):
|
||||
"""A BertModel for Contrastive Learning."""
|
||||
def __init__(self, config):
|
||||
super().__init__(config)
|
||||
self.videomlp = VideoTokenMLP(config)
|
||||
self.bert = MMBertModel(config)
|
||||
self.init_weights()
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids=None,
|
||||
input_video_embeds=None,
|
||||
attention_mask=None,
|
||||
token_type_ids=None,
|
||||
position_ids=None,
|
||||
head_mask=None,
|
||||
inputs_embeds=None,
|
||||
output_attentions=None,
|
||||
output_hidden_states=None,
|
||||
return_dict=None,
|
||||
):
|
||||
return_dict = (
|
||||
return_dict if return_dict is not None
|
||||
else self.config.use_return_dict
|
||||
)
|
||||
if input_video_embeds is not None:
|
||||
video_tokens = self.videomlp(input_video_embeds)
|
||||
else:
|
||||
video_tokens = None
|
||||
|
||||
outputs = self.bert(
|
||||
input_ids,
|
||||
video_tokens,
|
||||
attention_mask=attention_mask,
|
||||
token_type_ids=token_type_ids,
|
||||
position_ids=position_ids,
|
||||
head_mask=head_mask,
|
||||
inputs_embeds=inputs_embeds,
|
||||
output_attentions=output_attentions,
|
||||
output_hidden_states=output_hidden_states,
|
||||
return_dict=return_dict,
|
||||
)
|
||||
return outputs
|
||||
|
||||
|
||||
class MMBertForMFMMLM(BertPreTrainedModel):
|
||||
"""A BertModel with shared prediction head on MFM-MLM."""
|
||||
def __init__(self, config):
|
||||
super().__init__(config)
|
||||
self.videomlp = VideoTokenMLP(config)
|
||||
self.bert = MMBertModel(config)
|
||||
self.cls = MFMMLMHead(config)
|
||||
self.hidden_size = config.hidden_size
|
||||
self.init_weights()
|
||||
|
||||
def get_output_embeddings(self):
|
||||
return self.cls.predictions.decoder
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids=None,
|
||||
input_video_embeds=None,
|
||||
attention_mask=None,
|
||||
token_type_ids=None,
|
||||
position_ids=None,
|
||||
head_mask=None,
|
||||
inputs_embeds=None,
|
||||
masked_frame_labels=None,
|
||||
target_video_hidden_states=None,
|
||||
non_masked_frame_mask=None,
|
||||
masked_lm_labels=None,
|
||||
output_attentions=None,
|
||||
output_hidden_states=None,
|
||||
return_dict=None,
|
||||
):
|
||||
return_dict = (
|
||||
return_dict if return_dict is not None
|
||||
else self.config.use_return_dict
|
||||
)
|
||||
if input_video_embeds is not None:
|
||||
video_tokens = self.videomlp(input_video_embeds)
|
||||
else:
|
||||
video_tokens = None
|
||||
|
||||
if target_video_hidden_states is not None:
|
||||
target_video_hidden_states = self.videomlp(
|
||||
target_video_hidden_states)
|
||||
|
||||
non_masked_frame_hidden_states = video_tokens.masked_select(
|
||||
non_masked_frame_mask.unsqueeze(-1)
|
||||
).view(-1, self.hidden_size)
|
||||
|
||||
outputs = self.bert(
|
||||
input_ids,
|
||||
video_tokens,
|
||||
attention_mask=attention_mask,
|
||||
token_type_ids=token_type_ids,
|
||||
position_ids=position_ids,
|
||||
head_mask=head_mask,
|
||||
inputs_embeds=inputs_embeds,
|
||||
output_attentions=output_attentions,
|
||||
output_hidden_states=output_hidden_states,
|
||||
return_dict=return_dict,
|
||||
)
|
||||
|
||||
sequence_output = outputs[0]
|
||||
|
||||
mfm_scores, prediction_scores = None, None
|
||||
if masked_frame_labels is not None and masked_lm_labels is not None:
|
||||
# split the sequence.
|
||||
text_offset = masked_frame_labels.size(1) + 1 # [CLS]
|
||||
video_sequence_output = sequence_output[
|
||||
:, 1:text_offset
|
||||
] # remove [SEP] as not in video_label.
|
||||
text_sequence_output = torch.cat(
|
||||
[sequence_output[:, :1], sequence_output[:, text_offset:]],
|
||||
dim=1
|
||||
)
|
||||
|
||||
hidden_size = video_sequence_output.size(-1)
|
||||
selected_video_output = video_sequence_output.masked_select(
|
||||
masked_frame_labels.unsqueeze(-1)
|
||||
).view(-1, hidden_size)
|
||||
|
||||
# only compute select tokens to training to speed up.
|
||||
hidden_size = text_sequence_output.size(-1)
|
||||
# masked_lm_labels = masked_lm_labels.reshape(-1)
|
||||
labels_mask = masked_lm_labels != -100
|
||||
|
||||
selected_text_output = text_sequence_output.masked_select(
|
||||
labels_mask.unsqueeze(-1)
|
||||
).view(-1, hidden_size)
|
||||
mfm_scores, prediction_scores = self.cls(
|
||||
selected_video_output,
|
||||
target_video_hidden_states,
|
||||
non_masked_frame_hidden_states,
|
||||
selected_text_output,
|
||||
)
|
||||
|
||||
output = (
|
||||
mfm_scores,
|
||||
prediction_scores,
|
||||
) + outputs
|
||||
return output
|
||||
|
||||
|
||||
class BertMFMMLMPredictionHead(nn.Module):
|
||||
def __init__(self, config):
|
||||
super().__init__()
|
||||
self.transform = BertPredictionHeadTransform(config)
|
||||
# The output weights are the same as the input embeddings, but there is
|
||||
# an output-only bias for each token.
|
||||
self.decoder = nn.Linear(
|
||||
config.hidden_size, config.vocab_size, bias=False)
|
||||
|
||||
self.bias = nn.Parameter(torch.zeros(config.vocab_size))
|
||||
|
||||
# Need a link between the two variables so that the bias is correctly
|
||||
# resized with `resize_token_embeddings`
|
||||
self.decoder.bias = self.bias
|
||||
|
||||
def forward(
|
||||
self,
|
||||
video_hidden_states=None,
|
||||
target_video_hidden_states=None,
|
||||
non_masked_frame_hidden_states=None,
|
||||
text_hidden_states=None,
|
||||
):
|
||||
video_logits, text_logits = None, None
|
||||
if video_hidden_states is not None:
|
||||
video_hidden_states = self.transform(video_hidden_states)
|
||||
non_masked_frame_logits = torch.mm(
|
||||
video_hidden_states,
|
||||
non_masked_frame_hidden_states.transpose(1, 0)
|
||||
)
|
||||
masked_frame_logits = torch.bmm(
|
||||
video_hidden_states.unsqueeze(1),
|
||||
target_video_hidden_states.unsqueeze(-1),
|
||||
).squeeze(-1)
|
||||
video_logits = torch.cat(
|
||||
[masked_frame_logits, non_masked_frame_logits], dim=1
|
||||
)
|
||||
|
||||
if text_hidden_states is not None:
|
||||
text_hidden_states = self.transform(text_hidden_states)
|
||||
text_logits = self.decoder(text_hidden_states)
|
||||
return video_logits, text_logits
|
||||
|
||||
|
||||
class MFMMLMHead(nn.Module):
|
||||
def __init__(self, config):
|
||||
super().__init__()
|
||||
self.predictions = BertMFMMLMPredictionHead(config)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
video_hidden_states=None,
|
||||
target_video_hidden_states=None,
|
||||
non_masked_frame_hidden_states=None,
|
||||
text_hidden_states=None,
|
||||
):
|
||||
video_logits, text_logits = self.predictions(
|
||||
video_hidden_states,
|
||||
target_video_hidden_states,
|
||||
non_masked_frame_hidden_states,
|
||||
text_hidden_states,
|
||||
)
|
||||
return video_logits, text_logits
|
||||
|
||||
|
||||
class MMBertForMTM(MMBertForMFMMLM):
|
||||
def __init__(self, config):
|
||||
BertPreTrainedModel.__init__(self, config)
|
||||
self.videomlp = VideoTokenMLP(config)
|
||||
self.bert = MMBertModel(config)
|
||||
self.cls = MTMHead(config)
|
||||
self.hidden_size = config.hidden_size
|
||||
self.init_weights()
|
||||
|
||||
|
||||
class BertMTMPredictionHead(nn.Module):
|
||||
def __init__(self, config):
|
||||
super().__init__()
|
||||
self.transform = BertPredictionHeadTransform(config)
|
||||
self.decoder = nn.Linear(
|
||||
config.hidden_size, config.vocab_size, bias=False)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
video_hidden_states=None,
|
||||
target_video_hidden_states=None,
|
||||
non_masked_frame_hidden_states=None,
|
||||
text_hidden_states=None,
|
||||
):
|
||||
non_masked_frame_hidden_states = non_masked_frame_hidden_states.transpose(1, 0)
|
||||
video_logits, text_logits = None, None
|
||||
if video_hidden_states is not None:
|
||||
video_hidden_states = self.transform(video_hidden_states)
|
||||
|
||||
masked_frame_logits = torch.bmm(
|
||||
video_hidden_states.unsqueeze(1),
|
||||
target_video_hidden_states.unsqueeze(-1),
|
||||
).squeeze(-1)
|
||||
|
||||
non_masked_frame_logits = torch.mm(
|
||||
video_hidden_states,
|
||||
non_masked_frame_hidden_states
|
||||
)
|
||||
video_on_vocab_logits = self.decoder(video_hidden_states)
|
||||
video_logits = torch.cat([
|
||||
masked_frame_logits,
|
||||
non_masked_frame_logits,
|
||||
video_on_vocab_logits], dim=1)
|
||||
|
||||
if text_hidden_states is not None:
|
||||
text_hidden_states = self.transform(text_hidden_states)
|
||||
# text first so label does not need to be shifted.
|
||||
text_on_vocab_logits = self.decoder(text_hidden_states)
|
||||
text_on_video_logits = torch.mm(
|
||||
text_hidden_states,
|
||||
non_masked_frame_hidden_states
|
||||
)
|
||||
text_logits = torch.cat([
|
||||
text_on_vocab_logits,
|
||||
text_on_video_logits
|
||||
], dim=1)
|
||||
|
||||
return video_logits, text_logits
|
||||
|
||||
|
||||
class MTMHead(nn.Module):
|
||||
def __init__(self, config):
|
||||
super().__init__()
|
||||
self.predictions = BertMTMPredictionHead(config)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
video_hidden_states=None,
|
||||
target_video_hidden_states=None,
|
||||
non_masked_frame_hidden_states=None,
|
||||
text_hidden_states=None,
|
||||
):
|
||||
video_logits, text_logits = self.predictions(
|
||||
video_hidden_states,
|
||||
target_video_hidden_states,
|
||||
non_masked_frame_hidden_states,
|
||||
text_hidden_states,
|
||||
)
|
||||
return video_logits, text_logits
|
||||
|
||||
|
||||
class MMBertModel(BertModel):
|
||||
"""MMBertModel has MMBertEmbedding to support video tokens."""
|
||||
|
||||
def __init__(self, config, add_pooling_layer=True):
|
||||
super().__init__(config)
|
||||
# overwrite embedding
|
||||
self.embeddings = MMBertEmbeddings(config)
|
||||
self.encoder = MultiLayerAttentionMaskBertEncoder(config)
|
||||
self.init_weights()
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids=None,
|
||||
input_video_embeds=None,
|
||||
attention_mask=None,
|
||||
token_type_ids=None,
|
||||
position_ids=None,
|
||||
head_mask=None,
|
||||
inputs_embeds=None,
|
||||
encoder_hidden_states=None,
|
||||
encoder_attention_mask=None,
|
||||
output_attentions=None,
|
||||
output_hidden_states=None,
|
||||
return_dict=None,
|
||||
separate_forward_split=None,
|
||||
):
|
||||
output_attentions = (
|
||||
output_attentions
|
||||
if output_attentions is not None
|
||||
else self.config.output_attentions
|
||||
)
|
||||
output_hidden_states = (
|
||||
output_hidden_states
|
||||
if output_hidden_states is not None
|
||||
else self.config.output_hidden_states
|
||||
)
|
||||
return_dict = (
|
||||
return_dict if return_dict is not None
|
||||
else self.config.use_return_dict
|
||||
)
|
||||
|
||||
if input_ids is not None and inputs_embeds is not None:
|
||||
raise ValueError(
|
||||
"You cannot specify both input_ids "
|
||||
"and inputs_embeds at the same time"
|
||||
)
|
||||
elif input_ids is not None:
|
||||
if input_video_embeds is not None:
|
||||
input_shape = (
|
||||
input_ids.size(0),
|
||||
input_ids.size(1) + input_video_embeds.size(1),
|
||||
)
|
||||
else:
|
||||
input_shape = (
|
||||
input_ids.size(0),
|
||||
input_ids.size(1),
|
||||
)
|
||||
elif inputs_embeds is not None:
|
||||
if input_video_embeds is not None:
|
||||
input_shape = (
|
||||
inputs_embeds.size(0),
|
||||
inputs_embeds.size(1) + input_video_embeds.size(1),
|
||||
)
|
||||
else:
|
||||
input_shape = (
|
||||
input_ids.size(0),
|
||||
input_ids.size(1),
|
||||
)
|
||||
else:
|
||||
raise ValueError(
|
||||
"You have to specify either input_ids or inputs_embeds")
|
||||
|
||||
device = input_ids.device if input_ids is not None \
|
||||
else inputs_embeds.device
|
||||
|
||||
if attention_mask is None:
|
||||
attention_mask = torch.ones(input_shape, device=device)
|
||||
if token_type_ids is None:
|
||||
token_type_ids = torch.zeros(
|
||||
input_shape, dtype=torch.long, device=device)
|
||||
|
||||
# We can provide a self-attention mask of dimensions
|
||||
# [batch_size, from_seq_length, to_seq_length]
|
||||
# ourselves in which case
|
||||
# we just need to make it broadcastable to all heads.
|
||||
extended_attention_mask: torch.Tensor = \
|
||||
self.get_extended_attention_mask(
|
||||
attention_mask, input_shape, device)
|
||||
|
||||
# If a 2D or 3D attention mask is provided for the cross-attention
|
||||
# we need to make broadcastable to
|
||||
# [batch_size, num_heads, seq_length, seq_length]
|
||||
if self.config.is_decoder and encoder_hidden_states is not None:
|
||||
(
|
||||
encoder_batch_size,
|
||||
encoder_sequence_length,
|
||||
_,
|
||||
) = encoder_hidden_states.size()
|
||||
encoder_hidden_shape = (
|
||||
encoder_batch_size, encoder_sequence_length)
|
||||
if encoder_attention_mask is None:
|
||||
encoder_attention_mask = torch.ones(
|
||||
encoder_hidden_shape, device=device)
|
||||
encoder_extended_attention_mask = self.invert_attention_mask(
|
||||
encoder_attention_mask
|
||||
)
|
||||
else:
|
||||
encoder_extended_attention_mask = None
|
||||
|
||||
# Prepare head mask if needed
|
||||
# 1.0 in head_mask indicate we keep the head
|
||||
# attention_probs has shape bsz x n_heads x N x N
|
||||
# input head_mask has shape [num_heads] or
|
||||
# [num_hidden_layers x num_heads]
|
||||
# and head_mask is converted to shape
|
||||
# [num_hidden_layers x batch x num_heads x seq_length x seq_length]
|
||||
|
||||
head_mask = self.get_head_mask(
|
||||
head_mask, self.config.num_hidden_layers)
|
||||
|
||||
embedding_output = self.embeddings(
|
||||
input_ids,
|
||||
input_video_embeds,
|
||||
position_ids=position_ids,
|
||||
token_type_ids=token_type_ids,
|
||||
inputs_embeds=inputs_embeds,
|
||||
)
|
||||
|
||||
if separate_forward_split is not None:
|
||||
split_embedding_output = \
|
||||
embedding_output[:, :separate_forward_split]
|
||||
split_extended_attention_mask = extended_attention_mask[
|
||||
:, :, :, :separate_forward_split, :separate_forward_split
|
||||
]
|
||||
split_encoder_outputs = self.encoder(
|
||||
split_embedding_output,
|
||||
attention_mask=split_extended_attention_mask,
|
||||
head_mask=head_mask,
|
||||
encoder_hidden_states=encoder_hidden_states,
|
||||
encoder_attention_mask=encoder_extended_attention_mask,
|
||||
output_attentions=output_attentions,
|
||||
output_hidden_states=output_hidden_states,
|
||||
return_dict=return_dict,
|
||||
)
|
||||
assert (
|
||||
len(split_encoder_outputs) <= 2
|
||||
), "we do not support merge on attention for now."
|
||||
encoder_outputs = []
|
||||
encoder_outputs.append([split_encoder_outputs[0]])
|
||||
if len(split_encoder_outputs) == 2:
|
||||
encoder_outputs.append([])
|
||||
for _all_hidden_states in split_encoder_outputs[1]:
|
||||
encoder_outputs[-1].append([_all_hidden_states])
|
||||
|
||||
split_embedding_output = \
|
||||
embedding_output[:, separate_forward_split:]
|
||||
split_extended_attention_mask = extended_attention_mask[
|
||||
:, :, :, separate_forward_split:, separate_forward_split:
|
||||
]
|
||||
|
||||
split_encoder_outputs = self.encoder(
|
||||
split_embedding_output,
|
||||
attention_mask=split_extended_attention_mask,
|
||||
head_mask=head_mask,
|
||||
encoder_hidden_states=encoder_hidden_states,
|
||||
encoder_attention_mask=encoder_extended_attention_mask,
|
||||
output_attentions=output_attentions,
|
||||
output_hidden_states=output_hidden_states,
|
||||
return_dict=return_dict,
|
||||
)
|
||||
|
||||
assert (
|
||||
len(split_encoder_outputs) <= 2
|
||||
), "we do not support merge on attention for now."
|
||||
encoder_outputs[0].append(split_encoder_outputs[0])
|
||||
encoder_outputs[0] = torch.cat(encoder_outputs[0], dim=1)
|
||||
if len(split_encoder_outputs) == 2:
|
||||
for layer_idx, _all_hidden_states in enumerate(
|
||||
split_encoder_outputs[1]
|
||||
):
|
||||
encoder_outputs[1][layer_idx].append(_all_hidden_states)
|
||||
encoder_outputs[1][layer_idx] = torch.cat(
|
||||
encoder_outputs[1][layer_idx], dim=1
|
||||
)
|
||||
encoder_outputs = tuple(encoder_outputs)
|
||||
else:
|
||||
encoder_outputs = self.encoder(
|
||||
embedding_output,
|
||||
attention_mask=extended_attention_mask,
|
||||
head_mask=head_mask,
|
||||
encoder_hidden_states=encoder_hidden_states,
|
||||
encoder_attention_mask=encoder_extended_attention_mask,
|
||||
output_attentions=output_attentions,
|
||||
output_hidden_states=output_hidden_states,
|
||||
return_dict=return_dict,
|
||||
)
|
||||
|
||||
sequence_output = encoder_outputs[0]
|
||||
pooled_output = (
|
||||
self.pooler(sequence_output) if self.pooler is not None else None
|
||||
)
|
||||
|
||||
return (sequence_output, pooled_output) + encoder_outputs[1:]
|
||||
|
||||
def get_extended_attention_mask(self, attention_mask, input_shape, device):
|
||||
"""This is borrowed from `modeling_utils.py` with the support of
|
||||
multi-layer attention masks.
|
||||
The second dim is expected to be number of layers.
|
||||
See `MMAttentionMaskProcessor`.
|
||||
Makes broadcastable attention and causal masks so that future
|
||||
and masked tokens are ignored.
|
||||
|
||||
Arguments:
|
||||
attention_mask (:obj:`torch.Tensor`):
|
||||
Mask with ones indicating tokens to attend to,
|
||||
zeros for tokens to ignore.
|
||||
input_shape (:obj:`Tuple[int]`):
|
||||
The shape of the input to the model.
|
||||
device: (:obj:`torch.device`):
|
||||
The device of the input to the model.
|
||||
|
||||
Returns:
|
||||
:obj:`torch.Tensor` The extended attention mask, \
|
||||
with a the same dtype as :obj:`attention_mask.dtype`.
|
||||
"""
|
||||
# We can provide a self-attention mask of dimensions
|
||||
# [batch_size, from_seq_length, to_seq_length]
|
||||
# ourselves in which case we just need to make it broadcastable
|
||||
# to all heads.
|
||||
if attention_mask.dim() == 4:
|
||||
extended_attention_mask = attention_mask[:, :, None, :, :]
|
||||
extended_attention_mask = extended_attention_mask.to(
|
||||
dtype=self.dtype
|
||||
) # fp16 compatibility
|
||||
extended_attention_mask = (1.0 - extended_attention_mask) \
|
||||
* -10000.0
|
||||
return extended_attention_mask
|
||||
else:
|
||||
return super().get_extended_attention_mask(
|
||||
attention_mask, input_shape, device
|
||||
)
|
||||
|
||||
|
||||
class MultiLayerAttentionMaskBertEncoder(BertEncoder):
|
||||
"""extend BertEncoder with the capability of
|
||||
multiple layers of attention mask."""
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states,
|
||||
attention_mask=None,
|
||||
head_mask=None,
|
||||
encoder_hidden_states=None,
|
||||
encoder_attention_mask=None,
|
||||
output_attentions=False,
|
||||
output_hidden_states=False,
|
||||
return_dict=False,
|
||||
):
|
||||
all_hidden_states = () if output_hidden_states else None
|
||||
all_attentions = () if output_attentions else None
|
||||
for i, layer_module in enumerate(self.layer):
|
||||
if output_hidden_states:
|
||||
all_hidden_states = all_hidden_states + (hidden_states,)
|
||||
layer_head_mask = head_mask[i] if head_mask is not None else None
|
||||
|
||||
layer_attention_mask = (
|
||||
attention_mask[:, i, :, :, :]
|
||||
if attention_mask.dim() == 5
|
||||
else attention_mask
|
||||
)
|
||||
|
||||
if getattr(self.config, "gradient_checkpointing", False):
|
||||
|
||||
def create_custom_forward(module):
|
||||
def custom_forward(*inputs):
|
||||
return module(*inputs, output_attentions)
|
||||
|
||||
return custom_forward
|
||||
|
||||
layer_outputs = torch.utils.checkpoint.checkpoint(
|
||||
create_custom_forward(layer_module),
|
||||
hidden_states,
|
||||
layer_attention_mask,
|
||||
layer_head_mask,
|
||||
encoder_hidden_states,
|
||||
encoder_attention_mask,
|
||||
)
|
||||
else:
|
||||
layer_outputs = layer_module(
|
||||
hidden_states,
|
||||
layer_attention_mask,
|
||||
layer_head_mask,
|
||||
encoder_hidden_states,
|
||||
encoder_attention_mask,
|
||||
output_attentions,
|
||||
)
|
||||
hidden_states = layer_outputs[0]
|
||||
if output_attentions:
|
||||
all_attentions = all_attentions + (layer_outputs[1],)
|
||||
|
||||
if output_hidden_states:
|
||||
all_hidden_states = all_hidden_states + (hidden_states,)
|
||||
|
||||
return tuple(
|
||||
v
|
||||
for v in [hidden_states, all_hidden_states, all_attentions]
|
||||
if v is not None
|
||||
)
|
10
examples/MMPT/mmpt/modules/__init__.py
Normal file
10
examples/MMPT/mmpt/modules/__init__.py
Normal file
@ -0,0 +1,10 @@
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
#
|
||||
# This source code is licensed under the MIT license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
from .mm import *
|
||||
|
||||
try:
|
||||
from .expmm import *
|
||||
except ImportError:
|
||||
pass
|
145
examples/MMPT/mmpt/modules/mm.py
Normal file
145
examples/MMPT/mmpt/modules/mm.py
Normal file
@ -0,0 +1,145 @@
|
||||
# coding=utf-8
|
||||
# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team.
|
||||
# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# Copyright (c) Facebook, Inc. All Rights Reserved
|
||||
|
||||
|
||||
import torch
|
||||
|
||||
from torch import nn
|
||||
|
||||
try:
|
||||
from transformers.modeling_bert import (
|
||||
BertEmbeddings,
|
||||
ACT2FN,
|
||||
)
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
|
||||
class VideoTokenMLP(nn.Module):
|
||||
def __init__(self, config):
|
||||
super().__init__()
|
||||
input_dim = config.input_dim if hasattr(config, "input_dim") else 512
|
||||
self.linear1 = nn.Linear(input_dim, config.hidden_size)
|
||||
self.LayerNorm = nn.LayerNorm(config.hidden_size)
|
||||
self.activation = ACT2FN[config.hidden_act]
|
||||
self.linear2 = nn.Linear(config.hidden_size, config.hidden_size)
|
||||
|
||||
def forward(self, hidden_states):
|
||||
hidden_states = self.linear1(hidden_states)
|
||||
hidden_states = self.activation(hidden_states)
|
||||
hidden_states = self.LayerNorm(hidden_states)
|
||||
hidden_states = self.linear2(hidden_states)
|
||||
return hidden_states
|
||||
|
||||
|
||||
class MMBertEmbeddings(BertEmbeddings):
|
||||
def __init__(self, config):
|
||||
super().__init__(config)
|
||||
self.max_video_len = config.max_video_len
|
||||
if hasattr(config, "use_seg_emb") and config.use_seg_emb:
|
||||
"""the original VLM paper uses seg_embeddings for temporal space.
|
||||
although not used it changed the randomness of initialization.
|
||||
we keep it for reproducibility.
|
||||
"""
|
||||
self.seg_embeddings = nn.Embedding(256, config.hidden_size)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids,
|
||||
input_video_embeds,
|
||||
token_type_ids=None,
|
||||
position_ids=None,
|
||||
inputs_embeds=None,
|
||||
):
|
||||
input_tensor = input_ids if input_ids is not None else inputs_embeds
|
||||
if input_video_embeds is not None:
|
||||
input_shape = (
|
||||
input_tensor.size(0),
|
||||
input_tensor.size(1) + input_video_embeds.size(1),
|
||||
)
|
||||
else:
|
||||
input_shape = (input_tensor.size(0), input_tensor.size(1))
|
||||
|
||||
if position_ids is None:
|
||||
"""
|
||||
Auto skip position embeddings for text only case.
|
||||
use cases:
|
||||
(1) action localization and segmentation:
|
||||
feed in len-1 dummy video token needs text part to
|
||||
skip input_video_embeds.size(1) for the right
|
||||
position_ids for video [SEP] and rest text tokens.
|
||||
(2) MMFusionShare for two forward passings:
|
||||
in `forward_text`: input_video_embeds is None.
|
||||
need to skip video [SEP] token.
|
||||
|
||||
# video_len + 1: [CLS] + video_embed
|
||||
# self.max_video_len + 1: [SEP] for video.
|
||||
# self.max_video_len + 2: [SEP] for video.
|
||||
# self.max_video_len + input_ids.size(1): rest for text.
|
||||
"""
|
||||
if input_video_embeds is not None:
|
||||
video_len = input_video_embeds.size(1)
|
||||
starting_offset = self.max_video_len + 1 # video [SEP]
|
||||
ending_offset = self.max_video_len + input_ids.size(1)
|
||||
else:
|
||||
video_len = 0
|
||||
starting_offset = self.max_video_len + 2 # first text token.
|
||||
ending_offset = self.max_video_len + input_ids.size(1) + 1
|
||||
position_ids = torch.cat([
|
||||
self.position_ids[:, :video_len + 1],
|
||||
self.position_ids[:, starting_offset:ending_offset]
|
||||
], dim=1)
|
||||
|
||||
if token_type_ids is None:
|
||||
token_type_ids = torch.zeros(
|
||||
input_shape, dtype=torch.long, device=self.position_ids.device
|
||||
)
|
||||
|
||||
"""
|
||||
the format of input_ids is [CLS] [SEP] caption [SEP] padding.
|
||||
the goal is to build [CLS] video tokens [SEP] caption [SEP] .
|
||||
"""
|
||||
if inputs_embeds is None:
|
||||
inputs_embeds = self.word_embeddings(input_ids)
|
||||
if input_video_embeds is not None:
|
||||
inputs_mm_embeds = torch.cat([
|
||||
inputs_embeds[:, :1], input_video_embeds, inputs_embeds[:, 1:]
|
||||
], dim=1)
|
||||
else:
|
||||
# text only for `MMFusionShare`.
|
||||
inputs_mm_embeds = inputs_embeds
|
||||
|
||||
position_embeddings = self.position_embeddings(position_ids)
|
||||
token_type_embeddings = self.token_type_embeddings(token_type_ids)
|
||||
embeddings = inputs_mm_embeds + position_embeddings
|
||||
embeddings += token_type_embeddings
|
||||
|
||||
embeddings = self.LayerNorm(embeddings)
|
||||
embeddings = self.dropout(embeddings)
|
||||
return embeddings
|
||||
|
||||
|
||||
class AlignHead(nn.Module):
|
||||
"""this will load pre-trained weights for NSP, which is desirable."""
|
||||
|
||||
def __init__(self, config):
|
||||
super().__init__()
|
||||
self.seq_relationship = nn.Linear(config.hidden_size, 2)
|
||||
|
||||
def forward(self, dropout_pooled_output):
|
||||
logits = self.seq_relationship(dropout_pooled_output)
|
||||
return logits
|
429
examples/MMPT/mmpt/modules/retri.py
Normal file
429
examples/MMPT/mmpt/modules/retri.py
Normal file
@ -0,0 +1,429 @@
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
#
|
||||
# This source code is licensed under the MIT license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
import os
|
||||
import numpy as np
|
||||
import pickle
|
||||
import time
|
||||
|
||||
try:
|
||||
import faiss
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
from collections import defaultdict
|
||||
|
||||
from ..utils import get_local_rank, print_on_rank0
|
||||
|
||||
|
||||
class VectorRetriever(object):
|
||||
"""
|
||||
How2 Video Retriver.
|
||||
Reference usage of FAISS:
|
||||
https://github.com/fairinternal/fairseq-py/blob/paraphrase_pretraining/fairseq/data/multilingual_faiss_dataset.py
|
||||
"""
|
||||
|
||||
def __init__(self, hidden_size, cent, db_type, examples_per_cent_to_train):
|
||||
if db_type == "flatl2":
|
||||
quantizer = faiss.IndexFlatL2(hidden_size) # the other index
|
||||
self.db = faiss.IndexIVFFlat(
|
||||
quantizer, hidden_size, cent, faiss.METRIC_L2)
|
||||
elif db_type == "pq":
|
||||
self.db = faiss.index_factory(
|
||||
hidden_size, f"IVF{cent}_HNSW32,PQ32"
|
||||
)
|
||||
else:
|
||||
raise ValueError("unknown type of db", db_type)
|
||||
self.train_thres = cent * examples_per_cent_to_train
|
||||
self.train_cache = []
|
||||
self.train_len = 0
|
||||
self.videoid_to_vectoridx = {}
|
||||
self.vectoridx_to_videoid = None
|
||||
self.make_direct_maps_done = False
|
||||
|
||||
def make_direct_maps(self):
|
||||
faiss.downcast_index(self.db).make_direct_map()
|
||||
|
||||
def __len__(self):
|
||||
return self.db.ntotal
|
||||
|
||||
def save(self, out_dir):
|
||||
faiss.write_index(
|
||||
self.db,
|
||||
os.path.join(out_dir, "faiss_idx")
|
||||
)
|
||||
with open(
|
||||
os.path.join(
|
||||
out_dir, "videoid_to_vectoridx.pkl"),
|
||||
"wb") as fw:
|
||||
pickle.dump(
|
||||
self.videoid_to_vectoridx, fw,
|
||||
protocol=pickle.HIGHEST_PROTOCOL
|
||||
)
|
||||
|
||||
def load(self, out_dir):
|
||||
fn = os.path.join(out_dir, "faiss_idx")
|
||||
self.db = faiss.read_index(fn)
|
||||
with open(
|
||||
os.path.join(out_dir, "videoid_to_vectoridx.pkl"), "rb") as fr:
|
||||
self.videoid_to_vectoridx = pickle.load(fr)
|
||||
|
||||
def add(self, hidden_states, video_ids, last=False):
|
||||
assert len(hidden_states) == len(video_ids), "{}, {}".format(
|
||||
str(len(hidden_states)), str(len(video_ids)))
|
||||
assert len(hidden_states.shape) == 2
|
||||
assert hidden_states.dtype == np.float32
|
||||
|
||||
valid_idx = []
|
||||
for idx, video_id in enumerate(video_ids):
|
||||
if video_id not in self.videoid_to_vectoridx:
|
||||
valid_idx.append(idx)
|
||||
self.videoid_to_vectoridx[video_id] = \
|
||||
len(self.videoid_to_vectoridx)
|
||||
|
||||
hidden_states = hidden_states[valid_idx]
|
||||
if not self.db.is_trained:
|
||||
self.train_cache.append(hidden_states)
|
||||
self.train_len += hidden_states.shape[0]
|
||||
if self.train_len < self.train_thres:
|
||||
return
|
||||
self.finalize_training()
|
||||
else:
|
||||
self.db.add(hidden_states)
|
||||
|
||||
def finalize_training(self):
|
||||
hidden_states = np.concatenate(self.train_cache, axis=0)
|
||||
del self.train_cache
|
||||
local_rank = get_local_rank()
|
||||
if local_rank == 0:
|
||||
start = time.time()
|
||||
print("training db on", self.train_thres, "/", self.train_len)
|
||||
self.db.train(hidden_states[:self.train_thres])
|
||||
if local_rank == 0:
|
||||
print("training db for", time.time() - start)
|
||||
self.db.add(hidden_states)
|
||||
|
||||
def search(
|
||||
self,
|
||||
query_hidden_states,
|
||||
orig_dist,
|
||||
):
|
||||
if len(self.videoid_to_vectoridx) != self.db.ntotal:
|
||||
raise ValueError(
|
||||
"cannot search: size mismatch in-between index and db",
|
||||
len(self.videoid_to_vectoridx),
|
||||
self.db.ntotal
|
||||
)
|
||||
|
||||
if self.vectoridx_to_videoid is None:
|
||||
self.vectoridx_to_videoid = {
|
||||
self.videoid_to_vectoridx[videoid]: videoid
|
||||
for videoid in self.videoid_to_vectoridx
|
||||
}
|
||||
assert len(self.vectoridx_to_videoid) \
|
||||
== len(self.videoid_to_vectoridx)
|
||||
|
||||
# MultilingualFaissDataset uses the following; not sure the purpose.
|
||||
# faiss.ParameterSpace().set_index_parameter(self.db, "nprobe", 10)
|
||||
queried_dist, index = self.db.search(query_hidden_states, 1)
|
||||
queried_dist, index = queried_dist[:, 0], index[:, 0]
|
||||
|
||||
outputs = np.array(
|
||||
[self.vectoridx_to_videoid[_index]
|
||||
if _index != -1 else (-1, -1, -1) for _index in index],
|
||||
dtype=np.int32)
|
||||
outputs[queried_dist <= orig_dist] = -1
|
||||
return outputs
|
||||
|
||||
def search_by_video_ids(
|
||||
self,
|
||||
video_ids,
|
||||
retri_factor
|
||||
):
|
||||
if len(self.videoid_to_vectoridx) != self.db.ntotal:
|
||||
raise ValueError(
|
||||
len(self.videoid_to_vectoridx),
|
||||
self.db.ntotal
|
||||
)
|
||||
|
||||
if not self.make_direct_maps_done:
|
||||
self.make_direct_maps()
|
||||
|
||||
if self.vectoridx_to_videoid is None:
|
||||
self.vectoridx_to_videoid = {
|
||||
self.videoid_to_vectoridx[videoid]: videoid
|
||||
for videoid in self.videoid_to_vectoridx
|
||||
}
|
||||
assert len(self.vectoridx_to_videoid) \
|
||||
== len(self.videoid_to_vectoridx)
|
||||
|
||||
query_hidden_states = []
|
||||
vector_ids = []
|
||||
for video_id in video_ids:
|
||||
vector_id = self.videoid_to_vectoridx[video_id]
|
||||
vector_ids.append(vector_id)
|
||||
query_hidden_state = self.db.reconstruct(vector_id)
|
||||
query_hidden_states.append(query_hidden_state)
|
||||
query_hidden_states = np.stack(query_hidden_states)
|
||||
|
||||
# MultilingualFaissDataset uses the following; not sure the reason.
|
||||
# faiss.ParameterSpace().set_index_parameter(self.db, "nprobe", 10)
|
||||
_, index = self.db.search(query_hidden_states, retri_factor)
|
||||
outputs = []
|
||||
for sample_idx, sample in enumerate(index):
|
||||
# the first video_id is always the video itself.
|
||||
cands = [video_ids[sample_idx]]
|
||||
for vector_idx in sample:
|
||||
if vector_idx >= 0 \
|
||||
and vector_ids[sample_idx] != vector_idx:
|
||||
cands.append(
|
||||
self.vectoridx_to_videoid[vector_idx]
|
||||
)
|
||||
outputs.append(cands)
|
||||
return outputs
|
||||
|
||||
|
||||
class VectorRetrieverDM(VectorRetriever):
|
||||
"""
|
||||
with direct map.
|
||||
How2 Video Retriver.
|
||||
Reference usage of FAISS:
|
||||
https://github.com/fairinternal/fairseq-py/blob/paraphrase_pretraining/fairseq/data/multilingual_faiss_dataset.py
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size,
|
||||
cent,
|
||||
db_type,
|
||||
examples_per_cent_to_train
|
||||
):
|
||||
super().__init__(
|
||||
hidden_size, cent, db_type, examples_per_cent_to_train)
|
||||
self.make_direct_maps_done = False
|
||||
|
||||
def make_direct_maps(self):
|
||||
faiss.downcast_index(self.db).make_direct_map()
|
||||
self.make_direct_maps_done = True
|
||||
|
||||
def search(
|
||||
self,
|
||||
query_hidden_states,
|
||||
orig_dist,
|
||||
):
|
||||
if len(self.videoid_to_vectoridx) != self.db.ntotal:
|
||||
raise ValueError(
|
||||
len(self.videoid_to_vectoridx),
|
||||
self.db.ntotal
|
||||
)
|
||||
|
||||
if not self.make_direct_maps_done:
|
||||
self.make_direct_maps()
|
||||
if self.vectoridx_to_videoid is None:
|
||||
self.vectoridx_to_videoid = {
|
||||
self.videoid_to_vectoridx[videoid]: videoid
|
||||
for videoid in self.videoid_to_vectoridx
|
||||
}
|
||||
assert len(self.vectoridx_to_videoid) \
|
||||
== len(self.videoid_to_vectoridx)
|
||||
|
||||
# MultilingualFaissDataset uses the following; not sure the reason.
|
||||
# faiss.ParameterSpace().set_index_parameter(self.db, "nprobe", 10)
|
||||
queried_dist, index = self.db.search(query_hidden_states, 1)
|
||||
outputs = []
|
||||
for sample_idx, sample in enumerate(index):
|
||||
# and queried_dist[sample_idx] < thres \
|
||||
if sample >= 0 \
|
||||
and queried_dist[sample_idx] < orig_dist[sample_idx]:
|
||||
outputs.append(self.vectoridx_to_videoid[sample])
|
||||
else:
|
||||
outputs.append(None)
|
||||
return outputs
|
||||
|
||||
def search_by_video_ids(
|
||||
self,
|
||||
video_ids,
|
||||
retri_factor=8
|
||||
):
|
||||
if len(self.videoid_to_vectoridx) != self.db.ntotal:
|
||||
raise ValueError(
|
||||
len(self.videoid_to_vectoridx),
|
||||
self.db.ntotal
|
||||
)
|
||||
|
||||
if not self.make_direct_maps_done:
|
||||
self.make_direct_maps()
|
||||
if self.vectoridx_to_videoid is None:
|
||||
self.vectoridx_to_videoid = {
|
||||
self.videoid_to_vectoridx[videoid]: videoid
|
||||
for videoid in self.videoid_to_vectoridx
|
||||
}
|
||||
assert len(self.vectoridx_to_videoid) \
|
||||
== len(self.videoid_to_vectoridx)
|
||||
|
||||
query_hidden_states = []
|
||||
vector_ids = []
|
||||
for video_id in video_ids:
|
||||
vector_id = self.videoid_to_vectoridx[video_id]
|
||||
vector_ids.append(vector_id)
|
||||
query_hidden_state = self.db.reconstruct(vector_id)
|
||||
query_hidden_states.append(query_hidden_state)
|
||||
query_hidden_states = np.stack(query_hidden_states)
|
||||
|
||||
# MultilingualFaissDataset uses the following; not sure the reason.
|
||||
# faiss.ParameterSpace().set_index_parameter(self.db, "nprobe", 10)
|
||||
_, index = self.db.search(query_hidden_states, retri_factor)
|
||||
outputs = []
|
||||
for sample_idx, sample in enumerate(index):
|
||||
# the first video_id is always the video itself.
|
||||
cands = [video_ids[sample_idx]]
|
||||
for vector_idx in sample:
|
||||
if vector_idx >= 0 \
|
||||
and vector_ids[sample_idx] != vector_idx:
|
||||
cands.append(
|
||||
self.vectoridx_to_videoid[vector_idx]
|
||||
)
|
||||
outputs.append(cands)
|
||||
return outputs
|
||||
|
||||
|
||||
class MMVectorRetriever(VectorRetrieverDM):
|
||||
"""
|
||||
multimodal vector retriver:
|
||||
text retrieve video or video retrieve text.
|
||||
"""
|
||||
|
||||
def __init__(self, hidden_size, cent, db_type, examples_per_cent_to_train):
|
||||
super().__init__(
|
||||
hidden_size, cent, db_type, examples_per_cent_to_train)
|
||||
video_db = self.db
|
||||
super().__init__(
|
||||
hidden_size, cent, db_type, examples_per_cent_to_train)
|
||||
text_db = self.db
|
||||
self.db = {"video": video_db, "text": text_db}
|
||||
self.video_to_videoid = defaultdict(list)
|
||||
|
||||
def __len__(self):
|
||||
assert self.db["video"].ntotal == self.db["text"].ntotal
|
||||
return self.db["video"].ntotal
|
||||
|
||||
def make_direct_maps(self):
|
||||
faiss.downcast_index(self.db["video"]).make_direct_map()
|
||||
faiss.downcast_index(self.db["text"]).make_direct_map()
|
||||
|
||||
def save(self, out_dir):
|
||||
faiss.write_index(
|
||||
self.db["video"],
|
||||
os.path.join(out_dir, "video_faiss_idx")
|
||||
)
|
||||
faiss.write_index(
|
||||
self.db["text"],
|
||||
os.path.join(out_dir, "text_faiss_idx")
|
||||
)
|
||||
|
||||
with open(
|
||||
os.path.join(
|
||||
out_dir, "videoid_to_vectoridx.pkl"),
|
||||
"wb") as fw:
|
||||
pickle.dump(
|
||||
self.videoid_to_vectoridx, fw,
|
||||
protocol=pickle.HIGHEST_PROTOCOL
|
||||
)
|
||||
|
||||
def load(self, out_dir):
|
||||
fn = os.path.join(out_dir, "video_faiss_idx")
|
||||
video_db = faiss.read_index(fn)
|
||||
fn = os.path.join(out_dir, "text_faiss_idx")
|
||||
text_db = faiss.read_index(fn)
|
||||
self.db = {"video": video_db, "text": text_db}
|
||||
with open(
|
||||
os.path.join(out_dir, "videoid_to_vectoridx.pkl"), "rb") as fr:
|
||||
self.videoid_to_vectoridx = pickle.load(fr)
|
||||
self.video_to_videoid = defaultdict(list)
|
||||
|
||||
def add(self, hidden_states, video_ids):
|
||||
"""hidden_states is a pair `(video, text)`"""
|
||||
assert len(hidden_states) == len(video_ids), "{}, {}".format(
|
||||
str(len(hidden_states)), str(len(video_ids)))
|
||||
assert len(hidden_states.shape) == 3
|
||||
assert len(self.video_to_videoid) == 0
|
||||
|
||||
valid_idx = []
|
||||
for idx, video_id in enumerate(video_ids):
|
||||
if video_id not in self.videoid_to_vectoridx:
|
||||
valid_idx.append(idx)
|
||||
self.videoid_to_vectoridx[video_id] = \
|
||||
len(self.videoid_to_vectoridx)
|
||||
|
||||
batch_size = hidden_states.shape[0]
|
||||
hidden_states = hidden_states[valid_idx]
|
||||
|
||||
hidden_states = np.transpose(hidden_states, (1, 0, 2)).copy()
|
||||
if not self.db["video"].is_trained:
|
||||
self.train_cache.append(hidden_states)
|
||||
train_len = batch_size * len(self.train_cache)
|
||||
if train_len < self.train_thres:
|
||||
return
|
||||
|
||||
hidden_states = np.concatenate(self.train_cache, axis=1)
|
||||
del self.train_cache
|
||||
self.db["video"].train(hidden_states[0, :self.train_thres])
|
||||
self.db["text"].train(hidden_states[1, :self.train_thres])
|
||||
self.db["video"].add(hidden_states[0])
|
||||
self.db["text"].add(hidden_states[1])
|
||||
|
||||
def get_clips_by_video_id(self, video_id):
|
||||
if not self.video_to_videoid:
|
||||
for video_id, video_clip, text_clip in self.videoid_to_vectoridx:
|
||||
self.video_to_videoid[video_id].append(
|
||||
(video_id, video_clip, text_clip))
|
||||
return self.video_to_videoid[video_id]
|
||||
|
||||
def search(
|
||||
self,
|
||||
video_ids,
|
||||
target_modality,
|
||||
retri_factor=8
|
||||
):
|
||||
if len(self.videoid_to_vectoridx) != len(self):
|
||||
raise ValueError(
|
||||
len(self.videoid_to_vectoridx),
|
||||
len(self)
|
||||
)
|
||||
|
||||
if not self.make_direct_maps_done:
|
||||
self.make_direct_maps()
|
||||
if self.vectoridx_to_videoid is None:
|
||||
self.vectoridx_to_videoid = {
|
||||
self.videoid_to_vectoridx[videoid]: videoid
|
||||
for videoid in self.videoid_to_vectoridx
|
||||
}
|
||||
assert len(self.vectoridx_to_videoid) \
|
||||
== len(self.videoid_to_vectoridx)
|
||||
|
||||
src_modality = "text" if target_modality == "video" else "video"
|
||||
|
||||
query_hidden_states = []
|
||||
vector_ids = []
|
||||
for video_id in video_ids:
|
||||
vector_id = self.videoid_to_vectoridx[video_id]
|
||||
vector_ids.append(vector_id)
|
||||
query_hidden_state = self.db[src_modality].reconstruct(vector_id)
|
||||
query_hidden_states.append(query_hidden_state)
|
||||
query_hidden_states = np.stack(query_hidden_states)
|
||||
|
||||
# MultilingualFaissDataset uses the following; not sure the reason.
|
||||
# faiss.ParameterSpace().set_index_parameter(self.db, "nprobe", 10)
|
||||
_, index = self.db[target_modality].search(
|
||||
query_hidden_states, retri_factor)
|
||||
outputs = []
|
||||
for sample_idx, sample in enumerate(index):
|
||||
cands = []
|
||||
for vector_idx in sample:
|
||||
if vector_idx >= 0:
|
||||
cands.append(
|
||||
self.vectoridx_to_videoid[vector_idx]
|
||||
)
|
||||
outputs.append(cands)
|
||||
return outputs
|
246
examples/MMPT/mmpt/modules/vectorpool.py
Normal file
246
examples/MMPT/mmpt/modules/vectorpool.py
Normal file
@ -0,0 +1,246 @@
|
||||
# Copyright (c) Facebook, Inc. All Rights Reserved
|
||||
|
||||
import torch
|
||||
import os
|
||||
import numpy as np
|
||||
import pickle
|
||||
|
||||
from . import retri
|
||||
from ..utils import get_local_rank
|
||||
|
||||
|
||||
class VectorPool(object):
|
||||
"""
|
||||
Base class of retrieval space.
|
||||
"""
|
||||
|
||||
def __init__(self, config):
|
||||
from transformers import AutoConfig
|
||||
self.hidden_size = AutoConfig.from_pretrained(
|
||||
config.dataset.bert_name).hidden_size
|
||||
self.retriever_cls = getattr(retri, config.retriever_cls)
|
||||
|
||||
def __call__(self, sample, **kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
def build_retriver(
|
||||
self,
|
||||
retriever_cls=None,
|
||||
hidden_size=None,
|
||||
centroids=512,
|
||||
db_type="flatl2",
|
||||
examples_per_cent_to_train=48
|
||||
):
|
||||
|
||||
"""merge results from multiple gpus and return a retriver.."""
|
||||
self.retriver = retriever_cls(
|
||||
hidden_size, centroids, db_type, examples_per_cent_to_train)
|
||||
return self.retriver
|
||||
|
||||
def __repr__(self):
|
||||
if hasattr(self, "retriver"):
|
||||
retriver_name = str(len(self.retriver))
|
||||
else:
|
||||
retriver_name = "no retriver field yet"
|
||||
return self.__class__.__name__ \
|
||||
+ "(" + retriver_name + ")"
|
||||
|
||||
|
||||
class VideoVectorPool(VectorPool):
|
||||
"""
|
||||
average clips of a video as video representation.
|
||||
"""
|
||||
def __init__(self, config):
|
||||
super().__init__(config)
|
||||
self.build_retriver(self.retriever_cls, self.hidden_size)
|
||||
|
||||
def __call__(self, sample, subsampling, **kwargs):
|
||||
hidden_states = (
|
||||
sample["pooled_video"] + sample["pooled_text"]) / 2.
|
||||
hidden_states = hidden_states.view(
|
||||
-1, subsampling,
|
||||
hidden_states.size(-1))
|
||||
hidden_states = torch.mean(hidden_states, dim=1)
|
||||
hidden_states = hidden_states.cpu().detach().numpy()
|
||||
video_ids = []
|
||||
for offset_idx, video_id in enumerate(sample["video_id"]):
|
||||
if isinstance(video_id, tuple) and len(video_id) == 3:
|
||||
# a sharded video_id.
|
||||
video_id = video_id[0]
|
||||
video_ids.append(video_id)
|
||||
assert len(video_ids) == len(hidden_states)
|
||||
self.retriver.add(
|
||||
hidden_states.astype("float32"),
|
||||
video_ids
|
||||
)
|
||||
|
||||
|
||||
class DistributedVectorPool(VectorPool):
|
||||
"""
|
||||
support sync of multiple gpus/nodes.
|
||||
"""
|
||||
def __init__(self, config):
|
||||
super().__init__(config)
|
||||
self.out_dir = os.path.join(
|
||||
config.fairseq.checkpoint.save_dir,
|
||||
"retri")
|
||||
os.makedirs(self.out_dir, exist_ok=True)
|
||||
self.hidden_states = []
|
||||
self.video_ids = []
|
||||
|
||||
def build_retriver(
|
||||
self,
|
||||
retriever_cls=None,
|
||||
hidden_size=None,
|
||||
centroids=4096,
|
||||
db_type="flatl2",
|
||||
examples_per_cent_to_train=48
|
||||
):
|
||||
if retriever_cls is None:
|
||||
retriever_cls = self.retriever_cls
|
||||
if hidden_size is None:
|
||||
hidden_size = self.hidden_size
|
||||
"""merge results from multiple gpus and return a retriver.."""
|
||||
if torch.distributed.is_initialized():
|
||||
self.save()
|
||||
# sync saving.
|
||||
torch.distributed.barrier()
|
||||
world_size = torch.distributed.get_world_size()
|
||||
else:
|
||||
world_size = 1
|
||||
self.retriver = retriever_cls(
|
||||
hidden_size, centroids, db_type, examples_per_cent_to_train)
|
||||
# each gpu process has its own retriever.
|
||||
for local_rank in range(world_size):
|
||||
if get_local_rank() == 0:
|
||||
print("load local_rank", local_rank)
|
||||
hidden_states, video_ids = self.load(local_rank)
|
||||
hidden_states = hidden_states.astype("float32")
|
||||
self.retriver.add(hidden_states, video_ids)
|
||||
return self.retriver
|
||||
|
||||
def load(self, local_rank):
|
||||
hidden_states = np.load(
|
||||
os.path.join(
|
||||
self.out_dir,
|
||||
"hidden_state" + str(local_rank) + ".npy"
|
||||
)
|
||||
)
|
||||
|
||||
with open(
|
||||
os.path.join(
|
||||
self.out_dir, "video_id" + str(local_rank) + ".pkl"),
|
||||
"rb") as fr:
|
||||
video_ids = pickle.load(fr)
|
||||
return hidden_states, video_ids
|
||||
|
||||
def save(self):
|
||||
hidden_states = np.vstack(self.hidden_states)
|
||||
assert len(hidden_states) == len(self.video_ids), "{}, {}".format(
|
||||
len(hidden_states),
|
||||
len(self.video_ids)
|
||||
)
|
||||
local_rank = torch.distributed.get_rank() \
|
||||
if torch.distributed.is_initialized() else 0
|
||||
|
||||
np.save(
|
||||
os.path.join(
|
||||
self.out_dir,
|
||||
"hidden_state" + str(local_rank) + ".npy"),
|
||||
hidden_states)
|
||||
|
||||
with open(
|
||||
os.path.join(
|
||||
self.out_dir,
|
||||
"video_id" + str(local_rank) + ".pkl"),
|
||||
"wb") as fw:
|
||||
pickle.dump(
|
||||
self.video_ids,
|
||||
fw,
|
||||
protocol=pickle.HIGHEST_PROTOCOL
|
||||
)
|
||||
|
||||
|
||||
class DistributedVideoVectorPool(DistributedVectorPool):
|
||||
"""
|
||||
average clips of a video as video representation.
|
||||
"""
|
||||
def __call__(self, sample, subsampling, **kwargs):
|
||||
hidden_states = (
|
||||
sample["pooled_video"] + sample["pooled_text"]) / 2.
|
||||
hidden_states = hidden_states.view(
|
||||
-1, subsampling,
|
||||
hidden_states.size(-1))
|
||||
hidden_states = torch.mean(hidden_states, dim=1)
|
||||
hidden_states = hidden_states.cpu().detach().numpy()
|
||||
video_ids = []
|
||||
for offset_idx, video_id in enumerate(sample["video_id"]):
|
||||
if isinstance(video_id, tuple) and len(video_id) == 3:
|
||||
# a sharded video_id.
|
||||
video_id = video_id[0]
|
||||
video_ids.append(video_id)
|
||||
assert len(video_ids) == len(hidden_states)
|
||||
self.hidden_states.append(hidden_states)
|
||||
self.video_ids.extend(video_ids)
|
||||
|
||||
|
||||
# ------------ the following are deprecated --------------
|
||||
|
||||
class TextClipVectorPool(VectorPool):
|
||||
def __init__(self, config):
|
||||
from transformers import AutoConfig
|
||||
hidden_size = AutoConfig.from_pretrained(
|
||||
config.dataset.bert_name).hidden_size
|
||||
retriever_cls = getattr(retri, config.retriever_cls)
|
||||
self.build_retriver(retriever_cls, hidden_size)
|
||||
|
||||
def __call__(self, sample, **kwargs):
|
||||
clip_meta = sample["clip_meta"].cpu()
|
||||
assert torch.all(torch.le(clip_meta[:, 4], clip_meta[:, 5]))
|
||||
text_meta = [tuple(item.tolist()) for item in clip_meta[:, 3:]]
|
||||
|
||||
if hasattr(self, "retriver"):
|
||||
# build_retriver is called.
|
||||
self.retriver.add(
|
||||
sample["pooled_text"].cpu().numpy().astype("float32"),
|
||||
text_meta
|
||||
)
|
||||
else:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class MMClipVectorPool(VectorPool):
|
||||
"""
|
||||
Multimodal Clip-level vector pool.
|
||||
"""
|
||||
def __init__(self, out_dir):
|
||||
"""use hidden_states to store `(video, text)`."""
|
||||
"""use video_ids to store `(video_id, start, end)`."""
|
||||
super().__init__(out_dir)
|
||||
|
||||
def __call__(self, sample, **kwargs):
|
||||
pooled_video = sample["pooled_video"].cpu().unsqueeze(1).numpy()
|
||||
pooled_text = sample["pooled_text"].cpu().unsqueeze(1).numpy()
|
||||
|
||||
self.hidden_states.append(
|
||||
np.concatenate([pooled_video, pooled_text], axis=1)
|
||||
)
|
||||
|
||||
video_starts = sample["video_start"].cpu()
|
||||
video_ends = sample["video_end"].cpu()
|
||||
assert torch.all(torch.le(video_starts, video_ends))
|
||||
|
||||
text_starts = sample["text_start"].cpu()
|
||||
text_ends = sample["text_end"].cpu()
|
||||
assert torch.all(torch.le(text_starts, text_ends))
|
||||
subsample_size = sample["pooled_video"].size(0) // len(sample["video_id"])
|
||||
video_ids = [video_id for video_id in sample["video_id"]
|
||||
for _ in range(subsample_size)
|
||||
]
|
||||
for video_id, video_start, video_end, text_start, text_end in zip(
|
||||
video_ids, video_starts, video_ends, text_starts, text_ends):
|
||||
self.video_ids.append((
|
||||
video_id,
|
||||
(int(video_start), int(video_end)),
|
||||
(int(text_start), int(text_end))
|
||||
))
|
23
examples/MMPT/mmpt/processors/__init__.py
Normal file
23
examples/MMPT/mmpt/processors/__init__.py
Normal file
@ -0,0 +1,23 @@
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
#
|
||||
# This source code is licensed under the MIT license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
from .processor import *
|
||||
|
||||
from .how2processor import *
|
||||
from .how2retriprocessor import *
|
||||
|
||||
from .dsprocessor import *
|
||||
|
||||
try:
|
||||
from .rawvideoprocessor import *
|
||||
from .codecprocessor import *
|
||||
from .webvidprocessor import *
|
||||
from .expprocessor import *
|
||||
from .exphow2processor import *
|
||||
from .exphow2retriprocessor import *
|
||||
from .expcodecprocessor import *
|
||||
from .expfeatureencoder import *
|
||||
from .expdsprocessor import *
|
||||
except ImportError:
|
||||
pass
|
242
examples/MMPT/mmpt/processors/dedupprocessor.py
Normal file
242
examples/MMPT/mmpt/processors/dedupprocessor.py
Normal file
@ -0,0 +1,242 @@
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
#
|
||||
# This source code is licensed under the MIT license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
|
||||
import random
|
||||
import json
|
||||
import pickle
|
||||
from tqdm import tqdm
|
||||
import os
|
||||
import numpy as np
|
||||
|
||||
|
||||
class CaptionDedupProcessor(object):
|
||||
"""remove overlapping of caption sentences(clip).
|
||||
Some statistics:
|
||||
caption:
|
||||
{'t_clip_len': 246.6448431320854,
|
||||
'video_len': 281.09174795676245,
|
||||
'clip_tps': 0.8841283727427481,
|
||||
'video_tps': 0.7821156477732097,
|
||||
'min_clip_len': 0.0,
|
||||
'max_clip_len': 398.3,
|
||||
'mean_clip_len': 3.196580003006861,
|
||||
'num_clip': 77.15897706301081}
|
||||
|
||||
raw_caption:
|
||||
{'t_clip_len': 238.95908778424115,
|
||||
'video_len': 267.5914859862507,
|
||||
'clip_tps': 2.4941363624267963,
|
||||
'video_tps': 2.258989769647173,
|
||||
'min_clip_len': 0.0,
|
||||
'max_clip_len': 398.3,
|
||||
'mean_clip_len': 3.0537954186814265,
|
||||
'num_clip': 78.24986779481756}
|
||||
"""
|
||||
|
||||
def __init__(self, pkl_file):
|
||||
with open(pkl_file, "rb") as fd:
|
||||
self.data = pickle.load(fd)
|
||||
self.stat = {
|
||||
"t_clip_len": [],
|
||||
"video_len": [],
|
||||
"clip_tps": [],
|
||||
"video_tps": [],
|
||||
"clip_len": [],
|
||||
}
|
||||
|
||||
def __call__(self):
|
||||
for idx, video_id in enumerate(tqdm(self.data)):
|
||||
caption = json.loads(self.data[video_id])
|
||||
caption = self._dedup(caption)
|
||||
if idx < 4096: # for the first 4096 examples, compute the statistics.
|
||||
self.save_stat(video_id, caption)
|
||||
self.data[video_id] = json.dumps(caption)
|
||||
self.print_stat()
|
||||
|
||||
def single(self, video_id):
|
||||
caption = json.loads(self.data[video_id])
|
||||
for clip_idx, (start, end, text) in enumerate(
|
||||
zip(caption["start"], caption["end"], caption["text"])
|
||||
):
|
||||
print(start, end, text)
|
||||
print("@" * 100)
|
||||
caption = self._dedup(caption)
|
||||
for clip_idx, (start, end, text) in enumerate(
|
||||
zip(caption["start"], caption["end"], caption["text"])
|
||||
):
|
||||
print(start, end, text)
|
||||
print("#" * 100)
|
||||
self.save_stat(video_id, caption)
|
||||
self.print_stat()
|
||||
|
||||
def finalize(self, tgt_fn):
|
||||
with open(tgt_fn, "wb") as fw:
|
||||
pickle.dump(self.data, fw, pickle.HIGHEST_PROTOCOL)
|
||||
|
||||
def save_stat(self, video_id, caption):
|
||||
video_fn = os.path.join(
|
||||
"data/feat/feat_how2_s3d", video_id + ".npy"
|
||||
)
|
||||
if os.path.isfile(video_fn):
|
||||
with open(video_fn, "rb", 1) as fr: # 24 is the buffer size. buffered
|
||||
version = np.lib.format.read_magic(fr)
|
||||
shape, fortran, dtype = np.lib.format._read_array_header(fr, version)
|
||||
video_len = shape[0]
|
||||
|
||||
t_clip_len = 0.0
|
||||
t_tokens = 0
|
||||
for idx, (start, end, text) in enumerate(
|
||||
zip(caption["start"], caption["end"], caption["text"])
|
||||
):
|
||||
clip_len = (
|
||||
(end - max(caption["end"][idx - 1], start))
|
||||
if idx > 0
|
||||
else end - start
|
||||
)
|
||||
t_clip_len += clip_len
|
||||
t_tokens += len(text.split(" "))
|
||||
self.stat["clip_len"].append(clip_len)
|
||||
self.stat["t_clip_len"].append(t_clip_len)
|
||||
self.stat["video_len"].append(video_len)
|
||||
self.stat["clip_tps"].append(t_tokens / t_clip_len)
|
||||
self.stat["video_tps"].append(t_tokens / video_len)
|
||||
|
||||
def print_stat(self):
|
||||
result = {
|
||||
"t_clip_len": np.mean(self.stat["t_clip_len"]),
|
||||
"video_len": np.mean(self.stat["video_len"]),
|
||||
"clip_tps": np.mean(self.stat["clip_tps"]),
|
||||
"video_tps": np.mean(self.stat["video_tps"]),
|
||||
"min_clip_len": min(self.stat["clip_len"]),
|
||||
"max_clip_len": max(self.stat["clip_len"]),
|
||||
"mean_clip_len": np.mean(self.stat["clip_len"]),
|
||||
"num_clip": len(self.stat["clip_len"]) / len(self.stat["video_tps"]),
|
||||
}
|
||||
print(result)
|
||||
|
||||
def _dedup(self, caption):
|
||||
def random_merge(end_idx, start, end, text, starts, ends, texts):
|
||||
if random.random() > 0.5:
|
||||
# print(clip_idx, "[PARTIAL INTO PREV]", end_idx)
|
||||
# overlapped part goes to the end of previous.
|
||||
ends[-1] = max(ends[-1], start) # ?
|
||||
rest_text = text[end_idx:].strip()
|
||||
if rest_text:
|
||||
starts.append(max(ends[-1], start))
|
||||
ends.append(max(end, starts[-1]))
|
||||
texts.append(rest_text)
|
||||
else: # goes to the beginning of the current.
|
||||
# strip the previous.
|
||||
left_text = texts[-1][:-end_idx].strip()
|
||||
if left_text:
|
||||
# print(clip_idx, "[PREV PARTIAL INTO CUR]", end_idx)
|
||||
ends[-1] = min(ends[-1], start)
|
||||
texts[-1] = left_text
|
||||
else:
|
||||
# print(clip_idx, "[PREV LEFT NOTHING ALL INTO CUR]", end_idx)
|
||||
starts.pop(-1)
|
||||
ends.pop(-1)
|
||||
texts.pop(-1)
|
||||
starts.append(start)
|
||||
ends.append(end)
|
||||
texts.append(text)
|
||||
|
||||
starts, ends, texts = [], [], []
|
||||
for clip_idx, (start, end, text) in enumerate(
|
||||
zip(caption["start"], caption["end"], caption["text"])
|
||||
):
|
||||
if not isinstance(text, str):
|
||||
continue
|
||||
text = text.replace("\n", " ").strip()
|
||||
if len(text) == 0:
|
||||
continue
|
||||
starts.append(start)
|
||||
ends.append(end)
|
||||
texts.append(text)
|
||||
break
|
||||
|
||||
for clip_idx, (start, end, text) in enumerate(
|
||||
zip(
|
||||
caption["start"][clip_idx + 1:],
|
||||
caption["end"][clip_idx + 1:],
|
||||
caption["text"][clip_idx + 1:],
|
||||
)
|
||||
):
|
||||
if not isinstance(text, str):
|
||||
continue
|
||||
text = text.replace("\n", " ").strip()
|
||||
if len(text) == 0:
|
||||
continue
|
||||
|
||||
# print(clip_idx, texts[-5:])
|
||||
# print(clip_idx, start, end, text)
|
||||
if texts[-1].endswith(text): # subset of prev caption -> merge
|
||||
# print(clip_idx, "[MERGE INTO PREV]")
|
||||
ends[-1] = max(ends[-1], end)
|
||||
elif text.startswith(texts[-1]): # superset of prev caption -> merge
|
||||
# print(clip_idx, "[PREV MERGE INTO CUR]")
|
||||
texts[-1] = text
|
||||
starts[-1] = min(starts[-1], start)
|
||||
ends[-1] = max(ends[-1], end)
|
||||
else: # overlapping or non-overlapping.
|
||||
for end_idx in range(1, len(text) + 1):
|
||||
if texts[-1].endswith(text[:end_idx]):
|
||||
random_merge(end_idx, start, end, text, starts, ends, texts)
|
||||
break
|
||||
else:
|
||||
starts.append(start)
|
||||
ends.append(end)
|
||||
texts.append(text)
|
||||
|
||||
assert (ends[-1] + 0.001) >= starts[-1] and len(
|
||||
texts[-1]
|
||||
) > 0, "{} {} {} <- {} {} {}, {} {} {}".format(
|
||||
str(starts[-1]),
|
||||
str(ends[-1]),
|
||||
texts[-1],
|
||||
caption["start"][clip_idx - 1],
|
||||
caption["end"][clip_idx - 1],
|
||||
caption["text"][clip_idx - 1],
|
||||
str(start),
|
||||
str(end),
|
||||
text,
|
||||
)
|
||||
|
||||
return {"start": starts, "end": ends, "text": texts}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(description="dedup how2 caption")
|
||||
parser.add_argument('--how2dir', default="data/how2")
|
||||
args = parser.parse_args()
|
||||
|
||||
raw_caption_json = os.path.join(args.how2dir, "raw_caption.json")
|
||||
raw_caption_pickle = os.path.join(args.how2dir, "raw_caption.pkl")
|
||||
raw_caption_dedup_pickle = os.path.join(args.how2dir, "raw_caption_dedup.pkl")
|
||||
|
||||
def convert_to_pickle(src_fn, tgt_fn):
|
||||
with open(src_fn) as fd:
|
||||
captions = json.load(fd)
|
||||
|
||||
for video_id in captions:
|
||||
captions[video_id] = json.dumps(captions[video_id])
|
||||
|
||||
with open(tgt_fn, "wb") as fw:
|
||||
pickle.dump(captions, fw, pickle.HIGHEST_PROTOCOL)
|
||||
|
||||
if not os.path.isfile(raw_caption_pickle):
|
||||
convert_to_pickle(raw_caption_json, raw_caption_pickle)
|
||||
|
||||
deduper = CaptionDedupProcessor(raw_caption_pickle)
|
||||
deduper()
|
||||
deduper.finalize(raw_caption_dedup_pickle)
|
||||
|
||||
"""
|
||||
# demo
|
||||
deduper = CaptionDedupProcessor("data/how2/raw_caption.pkl")
|
||||
deduper.single("HfIeQ9pzL5U")
|
||||
"""
|
848
examples/MMPT/mmpt/processors/dsprocessor.py
Normal file
848
examples/MMPT/mmpt/processors/dsprocessor.py
Normal file
@ -0,0 +1,848 @@
|
||||
# Copyright (c) Facebook, Inc. All Rights Reserved
|
||||
|
||||
"""
|
||||
Processors for all downstream (ds) tasks.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import pickle
|
||||
import random
|
||||
import math
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from collections import defaultdict
|
||||
|
||||
from .processor import (
|
||||
MetaProcessor,
|
||||
VideoProcessor,
|
||||
TextProcessor,
|
||||
Aligner,
|
||||
MMAttentionMask2DProcessor,
|
||||
)
|
||||
|
||||
from .how2processor import TextGenerationProcessor
|
||||
|
||||
|
||||
# ------------- A General Aligner for all downstream tasks-----------------
|
||||
|
||||
|
||||
class DSAligner(Aligner):
|
||||
"""
|
||||
Downstream (DS) aligner shared by all datasets.
|
||||
"""
|
||||
|
||||
def __call__(self, video_id, video_feature, text_feature, wps=0.7):
|
||||
# random sample a starting sec for video.
|
||||
video_start = 0
|
||||
video_end = min(len(video_feature), self.max_video_len)
|
||||
# the whole sequence is a single clip.
|
||||
video_clips = {"start": [video_start], "end": [video_end]}
|
||||
|
||||
text_feature = {
|
||||
"cap": [text_feature],
|
||||
"start": [video_start],
|
||||
"end": [len(text_feature) / wps],
|
||||
}
|
||||
text_clip_indexs = [0]
|
||||
|
||||
vfeats, vmasks = self._build_video_seq(
|
||||
video_feature, video_clips
|
||||
)
|
||||
caps, cmasks = self._build_text_seq(
|
||||
text_feature, text_clip_indexs
|
||||
)
|
||||
|
||||
return {
|
||||
"caps": caps,
|
||||
"cmasks": cmasks,
|
||||
"vfeats": vfeats,
|
||||
"vmasks": vmasks,
|
||||
"video_id": video_id,
|
||||
}
|
||||
|
||||
|
||||
class NLGTextProcessor(TextProcessor):
|
||||
"""
|
||||
Also return the original text as ref.
|
||||
"""
|
||||
def __call__(self, text_id):
|
||||
return super().__call__(text_id), text_id
|
||||
|
||||
|
||||
class DSNLGAligner(DSAligner):
|
||||
"""extend with the capability of 2d mask for generation."""
|
||||
def __init__(self, config):
|
||||
super().__init__(config)
|
||||
self.attnmasker = MMAttentionMask2DProcessor()
|
||||
from transformers import AutoTokenizer
|
||||
tokenizer = AutoTokenizer.from_pretrained(
|
||||
self.bert_name, use_fast=self.use_fast,
|
||||
bos_token="[CLS]", eos_token="[SEP]"
|
||||
)
|
||||
self.tokenizer = tokenizer
|
||||
self.bos_token_id = tokenizer.bos_token_id
|
||||
self.eos_token_id = tokenizer.eos_token_id
|
||||
self.textgen = TextGenerationProcessor(tokenizer)
|
||||
|
||||
def __call__(self, video_id, video_feature, text_feature):
|
||||
output = super().__call__(video_id, video_feature, text_feature[0])
|
||||
if self.split == "test":
|
||||
# output.update({"ref": text_feature[1]})
|
||||
output.update({"ref": self.tokenizer.decode(
|
||||
output["caps"], skip_special_tokens=True)})
|
||||
text_label = output["caps"]
|
||||
cmasks = torch.BoolTensor([1] * text_label.size(0))
|
||||
caps = torch.LongTensor([
|
||||
self.cls_token_id,
|
||||
self.sep_token_id,
|
||||
self.bos_token_id])
|
||||
else:
|
||||
caps, text_label = self.textgen(output["caps"])
|
||||
cmasks = output["cmasks"]
|
||||
|
||||
attention_mask = self.attnmasker(
|
||||
output["vmasks"], cmasks, "textgen")
|
||||
|
||||
output.update({
|
||||
"caps": caps,
|
||||
"cmasks": cmasks,
|
||||
"text_label": text_label,
|
||||
"attention_mask": attention_mask,
|
||||
})
|
||||
return output
|
||||
|
||||
|
||||
# -------------------- MSRVTT ------------------------
|
||||
|
||||
|
||||
class MSRVTTMetaProcessor(MetaProcessor):
|
||||
"""MSRVTT dataset.
|
||||
reference: `howto100m/msrvtt_dataloader.py`
|
||||
"""
|
||||
|
||||
def __init__(self, config):
|
||||
super().__init__(config)
|
||||
import pandas as pd
|
||||
data = pd.read_csv(self._get_split_path(config))
|
||||
# TODO: add a text1ka flag.
|
||||
if config.split == "train" \
|
||||
and config.full_test_path is not None \
|
||||
and config.jsfusion_path is not None:
|
||||
# add testing videos from full_test_path not used by jfusion.
|
||||
additional_data = pd.read_csv(config.full_test_path)
|
||||
jsfusion_data = pd.read_csv(config.jsfusion_path)
|
||||
|
||||
for video_id in additional_data["video_id"]:
|
||||
if video_id not in jsfusion_data["video_id"].values:
|
||||
data = data.append(
|
||||
{"video_id": video_id}, ignore_index=True)
|
||||
|
||||
if config.dup is not None and config.split == "train":
|
||||
data = data.append([data] * (config.dup - 1), ignore_index=True)
|
||||
self.data = data
|
||||
|
||||
def __len__(self):
|
||||
return len(self.data)
|
||||
|
||||
def __getitem__(self, idx):
|
||||
"""slightly modify with if condition to combine train/test."""
|
||||
vid, sentence = None, None
|
||||
vid = self.data["video_id"].values[idx]
|
||||
if "sentence" in self.data: # for testing.
|
||||
sentence = self.data["sentence"].values[idx]
|
||||
else: # for training.
|
||||
sentence = vid
|
||||
return vid, sentence
|
||||
|
||||
|
||||
class MSRVTTTextProcessor(TextProcessor):
|
||||
"""MSRVTT dataset.
|
||||
reference: `msrvtt_dataloader.py` `MSRVTT_TrainDataLoader`.
|
||||
TODO (huxu): add max_words.
|
||||
"""
|
||||
|
||||
def __init__(self, config):
|
||||
super().__init__(config)
|
||||
self.sentences = None
|
||||
if config.json_path is not None and config.split == "train":
|
||||
with open(config.json_path) as fd:
|
||||
self.data = json.load(fd)
|
||||
self.sentences = defaultdict(list)
|
||||
for s in self.data["sentences"]:
|
||||
self.sentences[s["video_id"]].append(s["caption"])
|
||||
|
||||
def __call__(self, text_id):
|
||||
if self.sentences is not None:
|
||||
rind = random.randint(0, len(self.sentences[text_id]) - 1)
|
||||
sentence = self.sentences[text_id][rind]
|
||||
else:
|
||||
sentence = text_id
|
||||
caption = self.tokenizer(sentence, add_special_tokens=False)
|
||||
return caption["input_ids"]
|
||||
|
||||
|
||||
class MSRVTTNLGTextProcessor(MSRVTTTextProcessor):
|
||||
"""TODO: change dsaligner and merge to avoid any NLG text processor."""
|
||||
def __call__(self, text_id):
|
||||
if self.sentences is not None:
|
||||
rind = random.randint(0, len(self.sentences[text_id]) - 1)
|
||||
sentence = self.sentences[text_id][rind]
|
||||
else:
|
||||
sentence = text_id
|
||||
caption = self.tokenizer(sentence, add_special_tokens=False)
|
||||
return caption["input_ids"], sentence
|
||||
|
||||
|
||||
class MSRVTTQAMetaProcessor(MetaProcessor):
|
||||
"""MSRVTT-QA: retrieval-based multi-choice QA from JSFusion dataset.
|
||||
For simplicity, we use the train retrieval model.
|
||||
reference: `https://github.com/yj-yu/lsmdc`
|
||||
"""
|
||||
|
||||
def __init__(self, config):
|
||||
super().__init__(config)
|
||||
import pandas as pd
|
||||
csv_data = pd.read_csv(self._get_split_path(config), sep="\t")
|
||||
data = []
|
||||
for video_id, a1, a2, a3, a4, a5, answer in zip(
|
||||
csv_data["vid_key"].values,
|
||||
csv_data["a1"].values,
|
||||
csv_data["a2"].values,
|
||||
csv_data["a3"].values,
|
||||
csv_data["a4"].values,
|
||||
csv_data["a5"].values,
|
||||
csv_data["answer"].values):
|
||||
video_id = video_id.replace("msr", "video")
|
||||
data.append((video_id, (answer, [a1, a2, a3, a4, a5])))
|
||||
self.data = data
|
||||
|
||||
def __len__(self):
|
||||
return len(self.data)
|
||||
|
||||
def __getitem__(self, idx):
|
||||
return self.data[idx]
|
||||
|
||||
|
||||
class MSRVTTQATextProcessor(TextProcessor):
|
||||
"""MSRVTT-QA dataset.
|
||||
text_ans is of format `(answer, [a1, a2, a3, a4, a5])`.
|
||||
"""
|
||||
|
||||
def __call__(self, text_ans):
|
||||
for ans_idx, ans in enumerate(text_ans[1]):
|
||||
if isinstance(ans, str):
|
||||
text_ans[1][ans_idx] = self.tokenizer(ans, add_special_tokens=False)["input_ids"]
|
||||
return text_ans
|
||||
|
||||
|
||||
class MSRVTTQAAligner(DSAligner):
|
||||
"""MSRVTT dataset.
|
||||
similar to sample in how2.
|
||||
we call __call__ multiple times.
|
||||
"""
|
||||
|
||||
def __call__(self, video_id, video_feature, text_feature, wps=0.7):
|
||||
caps = []
|
||||
cmasks = []
|
||||
answer = text_feature[0]
|
||||
for ans_idx, _text_feature in enumerate(text_feature[1]):
|
||||
output = super().__call__(
|
||||
video_id, video_feature, _text_feature, wps)
|
||||
caps.append(output["caps"])
|
||||
cmasks.append(output["cmasks"])
|
||||
output.update({
|
||||
"caps": torch.stack(caps),
|
||||
"cmasks": torch.stack(cmasks),
|
||||
"answers": torch.LongTensor([answer]),
|
||||
})
|
||||
return output
|
||||
|
||||
|
||||
# -------------------- Youcook -----------------------
|
||||
|
||||
|
||||
class YoucookMetaProcessor(MetaProcessor):
|
||||
"""Youcook dataset.
|
||||
reference: `howto100m/youcook_dataloader.py`
|
||||
note that the data can be different as the
|
||||
(1) some videos already in Howto100m are removed.
|
||||
(2) stop words are removed from caption
|
||||
TODO (huxu): make a flag to load the original caption.
|
||||
(see youcookii_annotations_trainval.json).
|
||||
|
||||
The max_video_len can be 264 and text can be 64 tokens.
|
||||
In reality we may not need that long. see projects/task/youcook.yaml
|
||||
"""
|
||||
|
||||
def __init__(self, config):
|
||||
super().__init__(config)
|
||||
vfeat_dir = config.vfeat_dir
|
||||
print(self._get_split_path(config))
|
||||
with open(self._get_split_path(config), "rb") as fd:
|
||||
data = pickle.load(fd)
|
||||
all_valid_video_ids = set(
|
||||
[os.path.splitext(fn)[0] for fn in os.listdir(vfeat_dir)]
|
||||
)
|
||||
recs = []
|
||||
video_ids = set()
|
||||
valid_video_ids = set()
|
||||
for rec in data: # filter videos not available.
|
||||
udl_idx = rec["id"].rindex("_")
|
||||
video_id = rec["id"][:udl_idx]
|
||||
video_ids.add(video_id)
|
||||
if video_id in all_valid_video_ids:
|
||||
valid_video_ids.add(video_id)
|
||||
recs.append(rec)
|
||||
print("total video_ids in .pkl", len(video_ids))
|
||||
print("valid video_ids in .pkl", len(valid_video_ids))
|
||||
print("please verify {train,val}_list.txt")
|
||||
data = recs
|
||||
self.data = data
|
||||
|
||||
with open(config.trainval_annotation) as fd:
|
||||
self.youcook_annotation = json.load(fd)["database"]
|
||||
if config.use_annotation_text is True:
|
||||
print("using text in annotation.")
|
||||
self.use_annotation_caption = True
|
||||
else:
|
||||
self.use_annotation_caption = False
|
||||
|
||||
def __getitem__(self, idx):
|
||||
def _get_video_and_caption(rec):
|
||||
vid = rec["id"]
|
||||
udl_idx = vid.rindex("_")
|
||||
video_id, clip_id = vid[:udl_idx], int(vid[udl_idx + 1:])
|
||||
clip = self.youcook_annotation[video_id]["annotations"][clip_id]
|
||||
start, end = clip["segment"]
|
||||
if self.use_annotation_caption:
|
||||
caption = clip["sentence"]
|
||||
else:
|
||||
caption = rec["caption"]
|
||||
return (video_id, start, end), caption
|
||||
|
||||
rec = self.data[idx]
|
||||
video_info, text_info = _get_video_and_caption(rec)
|
||||
return video_info, text_info
|
||||
|
||||
|
||||
class YoucookVideoProcessor(VideoProcessor):
|
||||
"""video_fn is a tuple of (video_id, start, end) now."""
|
||||
|
||||
def __call__(self, video_fn):
|
||||
video_id, start, end = video_fn
|
||||
feat = np.load(os.path.join(self.vfeat_dir, video_id + ".npy"))
|
||||
return feat[start:end]
|
||||
|
||||
|
||||
class YoucookNLGMetaProcessor(MetaProcessor):
|
||||
"""NLG uses the original split:
|
||||
`train_list.txt` and `val_list.txt`
|
||||
"""
|
||||
|
||||
def __init__(self, config):
|
||||
super().__init__(config)
|
||||
vfeat_dir = config.vfeat_dir
|
||||
print(self._get_split_path(config))
|
||||
with open(self._get_split_path(config)) as fd:
|
||||
video_ids = [
|
||||
line.strip().split("/")[1] for line in fd.readlines()]
|
||||
print("total video_ids in train/val_list.txt", len(video_ids))
|
||||
|
||||
all_valid_video_ids = set(
|
||||
[os.path.splitext(fn)[0] for fn in os.listdir(vfeat_dir)]
|
||||
)
|
||||
video_ids = [
|
||||
video_id for video_id in video_ids
|
||||
if video_id in all_valid_video_ids]
|
||||
|
||||
print("valid video_ids in train/val_list.txt", len(video_ids))
|
||||
with open(config.trainval_annotation) as fd:
|
||||
self.youcook_annotation = json.load(fd)["database"]
|
||||
|
||||
data = []
|
||||
for video_id in video_ids:
|
||||
for clip in self.youcook_annotation[video_id]["annotations"]:
|
||||
start, end = clip["segment"]
|
||||
caption = clip["sentence"]
|
||||
data.append(((video_id, start, end), caption))
|
||||
self.data = data
|
||||
|
||||
def __getitem__(self, idx):
|
||||
return self.data[idx]
|
||||
|
||||
|
||||
# --------------------- CrossTask -------------------------
|
||||
|
||||
class CrossTaskMetaProcessor(MetaProcessor):
|
||||
def __init__(self, config):
|
||||
super().__init__(config)
|
||||
np.random.seed(0) # deterministic random split.
|
||||
task_vids = self._get_vids(
|
||||
config.train_csv_path,
|
||||
config.vfeat_dir,
|
||||
config.annotation_path)
|
||||
|
||||
val_vids = self._get_vids(
|
||||
config.val_csv_path,
|
||||
config.vfeat_dir,
|
||||
config.annotation_path)
|
||||
|
||||
# filter out those task and vids appear in val_vids.
|
||||
task_vids = {
|
||||
task: [
|
||||
vid for vid in vids
|
||||
if task not in val_vids or vid not in val_vids[task]]
|
||||
for task, vids in task_vids.items()}
|
||||
|
||||
primary_info = self._read_task_info(config.primary_path)
|
||||
test_tasks = set(primary_info['steps'].keys())
|
||||
|
||||
# if args.use_related:
|
||||
related_info = self._read_task_info(config.related_path)
|
||||
task_steps = {**primary_info['steps'], **related_info['steps']}
|
||||
n_steps = {**primary_info['n_steps'], **related_info['n_steps']}
|
||||
# else:
|
||||
# task_steps = primary_info['steps']
|
||||
# n_steps = primary_info['n_steps']
|
||||
all_tasks = set(n_steps.keys())
|
||||
# filter and keep task in primary or related.
|
||||
task_vids = {
|
||||
task: vids for task, vids in task_vids.items()
|
||||
if task in all_tasks}
|
||||
# vocab-by-step matrix (A) and vocab (M)
|
||||
# (huxu): we do not use BoW.
|
||||
# A, M = self._get_A(task_steps, share="words")
|
||||
|
||||
train_vids, test_vids = self._random_split(
|
||||
task_vids, test_tasks, config.n_train)
|
||||
print("train_num_videos", sum(len(vids) for vids in train_vids.values()))
|
||||
print("test_num_videos", sum(len(vids) for vids in test_vids.values()))
|
||||
# added by huxu to automatically determine the split.
|
||||
split_map = {
|
||||
"train": train_vids,
|
||||
"valid": test_vids,
|
||||
"test": test_vids
|
||||
}
|
||||
task_vids = split_map[config.split]
|
||||
|
||||
self.vids = []
|
||||
for task, vids in task_vids.items():
|
||||
self.vids.extend([(task, vid) for vid in vids])
|
||||
self.task_steps = task_steps
|
||||
self.n_steps = n_steps
|
||||
|
||||
def __getitem__(self, idx):
|
||||
task, vid = self.vids[idx]
|
||||
n_steps = self.n_steps[task]
|
||||
steps = self.task_steps[task]
|
||||
assert len(steps) == n_steps
|
||||
return (task, vid, steps, n_steps), (task, vid, steps, n_steps)
|
||||
|
||||
def __len__(self):
|
||||
return len(self.vids)
|
||||
|
||||
def _random_split(self, task_vids, test_tasks, n_train):
|
||||
train_vids = {}
|
||||
test_vids = {}
|
||||
for task, vids in task_vids.items():
|
||||
if task in test_tasks and len(vids) > n_train:
|
||||
train_vids[task] = np.random.choice(
|
||||
vids, n_train, replace=False).tolist()
|
||||
test_vids[task] = [
|
||||
vid for vid in vids if vid not in train_vids[task]]
|
||||
else:
|
||||
train_vids[task] = vids
|
||||
return train_vids, test_vids
|
||||
|
||||
def _get_vids(self, path, vfeat_dir, annotation_path):
|
||||
"""refactored from
|
||||
https://github.com/DmZhukov/CrossTask/blob/master/data.py
|
||||
changes: add `vfeat_dir` to check if the video is available.
|
||||
add `annotation_path` to check if the video is available.
|
||||
"""
|
||||
|
||||
task_vids = {}
|
||||
with open(path, 'r') as f:
|
||||
for line in f:
|
||||
task, vid, url = line.strip().split(',')
|
||||
# double check the video is available.
|
||||
if not os.path.exists(
|
||||
os.path.join(vfeat_dir, vid + ".npy")):
|
||||
continue
|
||||
# double check the annotation is available.
|
||||
if not os.path.exists(os.path.join(
|
||||
annotation_path,
|
||||
task + "_" + vid + ".csv")):
|
||||
continue
|
||||
if task not in task_vids:
|
||||
task_vids[task] = []
|
||||
task_vids[task].append(vid)
|
||||
return task_vids
|
||||
|
||||
def _read_task_info(self, path):
|
||||
titles = {}
|
||||
urls = {}
|
||||
n_steps = {}
|
||||
steps = {}
|
||||
with open(path, 'r') as f:
|
||||
idx = f.readline()
|
||||
while idx != '':
|
||||
idx = idx.strip()
|
||||
titles[idx] = f.readline().strip()
|
||||
urls[idx] = f.readline().strip()
|
||||
n_steps[idx] = int(f.readline().strip())
|
||||
steps[idx] = f.readline().strip().split(',')
|
||||
next(f)
|
||||
idx = f.readline()
|
||||
return {
|
||||
'title': titles,
|
||||
'url': urls,
|
||||
'n_steps': n_steps,
|
||||
'steps': steps
|
||||
}
|
||||
|
||||
def _get_A(self, task_steps, share="words"):
|
||||
raise ValueError("running get_A is not allowed for BERT.")
|
||||
"""Step-to-component matrices."""
|
||||
if share == 'words':
|
||||
# share words
|
||||
task_step_comps = {
|
||||
task: [step.split(' ') for step in steps]
|
||||
for task, steps in task_steps.items()}
|
||||
elif share == 'task_words':
|
||||
# share words within same task
|
||||
task_step_comps = {
|
||||
task: [[task+'_'+tok for tok in step.split(' ')] for step in steps]
|
||||
for task, steps in task_steps.items()}
|
||||
elif share == 'steps':
|
||||
# share whole step descriptions
|
||||
task_step_comps = {
|
||||
task: [[step] for step in steps] for task, steps in task_steps.items()}
|
||||
else:
|
||||
# no sharing
|
||||
task_step_comps = {
|
||||
task: [[task+'_'+step] for step in steps]
|
||||
for task, steps in task_steps.items()}
|
||||
# BERT tokenizer here?
|
||||
vocab = []
|
||||
for task, steps in task_step_comps.items():
|
||||
for step in steps:
|
||||
vocab.extend(step)
|
||||
vocab = {comp: m for m, comp in enumerate(set(vocab))}
|
||||
M = len(vocab)
|
||||
A = {}
|
||||
for task, steps in task_step_comps.items():
|
||||
K = len(steps)
|
||||
a = torch.zeros(M, K)
|
||||
for k, step in enumerate(steps):
|
||||
a[[vocab[comp] for comp in step], k] = 1
|
||||
a /= a.sum(dim=0)
|
||||
A[task] = a
|
||||
return A, M
|
||||
|
||||
|
||||
class CrossTaskVideoProcessor(VideoProcessor):
|
||||
def __call__(self, video_fn):
|
||||
task, vid, steps, n_steps = video_fn
|
||||
video_fn = os.path.join(self.vfeat_dir, vid + ".npy")
|
||||
feat = np.load(video_fn)
|
||||
return feat
|
||||
|
||||
|
||||
class CrossTaskTextProcessor(TextProcessor):
|
||||
def __call__(self, text_id):
|
||||
task, vid, steps, n_steps = text_id
|
||||
step_ids = []
|
||||
for step_str in steps:
|
||||
step_ids.append(
|
||||
self.tokenizer(step_str, add_special_tokens=False)["input_ids"]
|
||||
)
|
||||
return step_ids
|
||||
|
||||
|
||||
class CrossTaskAligner(Aligner):
|
||||
"""
|
||||
TODO: it's not clear yet the formulation of the task; finish this later.
|
||||
"""
|
||||
def __init__(self, config):
|
||||
super().__init__(config)
|
||||
self.annotation_path = config.annotation_path
|
||||
self.sliding_window = config.sliding_window
|
||||
self.sliding_window_size = config.sliding_window_size
|
||||
|
||||
def __call__(self, video_id, video_feature, text_feature):
|
||||
task, vid, steps, n_steps = video_id
|
||||
annot_path = os.path.join(
|
||||
self.annotation_path, task + '_' + vid + '.csv')
|
||||
video_len = len(video_feature)
|
||||
|
||||
labels = torch.from_numpy(self._read_assignment(
|
||||
video_len, n_steps, annot_path)).float()
|
||||
|
||||
vfeats, vmasks, targets = [], [], []
|
||||
# sliding window on video features and targets.
|
||||
for window_start in range(0, video_len, self.sliding_window):
|
||||
video_start = 0
|
||||
video_end = min(video_len - window_start, self.sliding_window_size)
|
||||
video_clip = {"start": [video_start], "end": [video_end]}
|
||||
|
||||
vfeat, vmask = self._build_video_seq(
|
||||
video_feature[window_start: window_start + video_end],
|
||||
video_clip
|
||||
)
|
||||
|
||||
target = labels[window_start: window_start + video_end]
|
||||
assert len(vfeat) >= len(target), "{},{}".format(len(vfeat), len(target))
|
||||
# TODO: randomly drop all zero targets for training ?
|
||||
# if self.split == "train" and target.sum() == 0:
|
||||
# continue
|
||||
vfeats.append(vfeat)
|
||||
vmasks.append(vmask)
|
||||
targets.append(target)
|
||||
|
||||
if (video_len - window_start) <= self.sliding_window_size:
|
||||
break
|
||||
|
||||
vfeats = torch.stack(vfeats)
|
||||
vmasks = torch.stack(vmasks)
|
||||
targets = torch.cat(targets, dim=0)
|
||||
|
||||
caps, cmasks = [], []
|
||||
for step in text_feature:
|
||||
step_text_feature = {"start": [0], "end": [1], "cap": [step]}
|
||||
step_text_clip_index = [0]
|
||||
cap, cmask = self._build_text_seq(
|
||||
step_text_feature, step_text_clip_index
|
||||
)
|
||||
caps.append(cap)
|
||||
cmasks.append(cmask)
|
||||
caps = torch.stack(caps)
|
||||
cmasks = torch.stack(cmasks)
|
||||
|
||||
return {
|
||||
"caps": caps,
|
||||
"cmasks": cmasks,
|
||||
"vfeats": vfeats, # X for original code.
|
||||
"vmasks": vmasks,
|
||||
"targets": targets,
|
||||
"video_id": vid,
|
||||
"task": task,
|
||||
"video_len": video_len # for later checking.
|
||||
}
|
||||
|
||||
def _read_assignment(self, T, K, path):
|
||||
"""
|
||||
refactored from https://github.com/DmZhukov/CrossTask/blob/master/data.py
|
||||
Howto interpret contraints on loss that is going to be minimized:
|
||||
lambd is a big number;
|
||||
self.lambd * C is a big number for all valid position (csv stores invalids)
|
||||
|
||||
def forward(self, O, Y, C):
|
||||
return (Y*(self.lambd * C - self.lsm(O))).mean(dim=0).sum()
|
||||
|
||||
This will load the csv file and fill-in the step col from start to end rows.
|
||||
"""
|
||||
|
||||
Y = np.zeros([T, K], dtype=np.uint8)
|
||||
with open(path, 'r') as f:
|
||||
for line in f:
|
||||
step, start, end = line.strip().split(',')
|
||||
start = int(math.floor(float(start)))
|
||||
end = int(math.ceil(float(end)))
|
||||
step = int(step) - 1
|
||||
Y[start:end, step] = 1
|
||||
return Y
|
||||
|
||||
|
||||
# --------------------- COIN -------------------------
|
||||
|
||||
class MetaTextBinarizer(Aligner):
|
||||
def __call__(self, text_feature):
|
||||
text_feature = {
|
||||
"cap": [text_feature],
|
||||
"start": [0.],
|
||||
"end": [100.],
|
||||
}
|
||||
text_clip_indexs = [0]
|
||||
|
||||
caps, cmasks = self._build_text_seq(
|
||||
text_feature, text_clip_indexs
|
||||
)
|
||||
return {"caps": caps, "cmasks": cmasks}
|
||||
|
||||
|
||||
class COINActionSegmentationMetaProcessor(MetaProcessor):
|
||||
split_map = {
|
||||
"train": "training",
|
||||
"valid": "testing",
|
||||
"test": "testing",
|
||||
}
|
||||
|
||||
def __init__(self, config):
|
||||
super().__init__(config)
|
||||
with open(self._get_split_path(config)) as fr:
|
||||
database = json.load(fr)["database"]
|
||||
id2label = {}
|
||||
data = []
|
||||
# filter the data by split.
|
||||
for video_id, rec in database.items():
|
||||
# always use testing to determine label_set
|
||||
if rec["subset"] == "testing":
|
||||
for segment in rec["annotation"]:
|
||||
id2label[int(segment["id"])] = segment["label"]
|
||||
# text_labels is used for ZS setting
|
||||
self.text_labels = ["none"] * len(id2label)
|
||||
for label_id in id2label:
|
||||
self.text_labels[label_id-1] = id2label[label_id]
|
||||
|
||||
id2label[0] = "O"
|
||||
print("num of labels", len(id2label))
|
||||
|
||||
for video_id, rec in database.items():
|
||||
if not os.path.isfile(os.path.join(config.vfeat_dir, video_id + ".npy")):
|
||||
continue
|
||||
if rec["subset"] == COINActionSegmentationMetaProcessor.split_map[self.split]:
|
||||
starts, ends, labels = [], [], []
|
||||
for segment in rec["annotation"]:
|
||||
start, end = segment["segment"]
|
||||
label = int(segment["id"])
|
||||
starts.append(start)
|
||||
ends.append(end)
|
||||
labels.append(label)
|
||||
data.append(
|
||||
(video_id, {"start": starts, "end": ends, "label": labels}))
|
||||
self.data = data
|
||||
|
||||
def meta_text_labels(self, config):
|
||||
from transformers import default_data_collator
|
||||
from ..utils import get_local_rank
|
||||
|
||||
text_processor = TextProcessor(config)
|
||||
binarizer = MetaTextBinarizer(config)
|
||||
# TODO: add prompts to .yaml.
|
||||
text_labels = [label for label in self.text_labels]
|
||||
|
||||
if get_local_rank() == 0:
|
||||
print(text_labels)
|
||||
|
||||
outputs = []
|
||||
for text_label in text_labels:
|
||||
text_feature = text_processor(text_label)
|
||||
outputs.append(binarizer(text_feature))
|
||||
return default_data_collator(outputs)
|
||||
|
||||
def __getitem__(self, idx):
|
||||
return self.data[idx]
|
||||
|
||||
|
||||
class COINActionSegmentationTextProcessor(TextProcessor):
|
||||
def __call__(self, text_label):
|
||||
return text_label
|
||||
|
||||
|
||||
class COINActionSegmentationAligner(Aligner):
|
||||
def __init__(self, config):
|
||||
super().__init__(config)
|
||||
self.sliding_window = config.sliding_window
|
||||
self.sliding_window_size = config.sliding_window_size
|
||||
|
||||
def __call__(self, video_id, video_feature, text_feature):
|
||||
starts, ends, label_ids = text_feature["start"], text_feature["end"], text_feature["label"]
|
||||
# sliding window.
|
||||
video_len = len(video_feature)
|
||||
|
||||
vfeats, vmasks, targets = [], [], []
|
||||
# sliding window on video features and targets.
|
||||
for window_start in range(0, video_len, self.sliding_window):
|
||||
video_start = 0
|
||||
video_end = min(video_len - window_start, self.sliding_window_size)
|
||||
video_clip = {"start": [video_start], "end": [video_end]}
|
||||
vfeat, vmask = self._build_video_seq(
|
||||
video_feature[window_start: window_start + video_end],
|
||||
video_clip
|
||||
)
|
||||
# covers video length only.
|
||||
target = torch.full_like(vmask, -100, dtype=torch.long)
|
||||
target[vmask] = 0
|
||||
for start, end, label_id in zip(starts, ends, label_ids):
|
||||
if (window_start < end) and (start < (window_start + video_end)):
|
||||
start_offset = max(0, math.floor(start) - window_start)
|
||||
end_offset = min(video_end, math.ceil(end) - window_start)
|
||||
target[start_offset:end_offset] = label_id
|
||||
vfeats.append(vfeat)
|
||||
vmasks.append(vmask)
|
||||
targets.append(target)
|
||||
if (video_len - window_start) <= self.sliding_window_size:
|
||||
break
|
||||
|
||||
vfeats = torch.stack(vfeats)
|
||||
vmasks = torch.stack(vmasks)
|
||||
targets = torch.stack(targets)
|
||||
video_targets = torch.full((video_len,), 0)
|
||||
for start, end, label_id in zip(starts, ends, label_ids):
|
||||
start_offset = max(0, math.floor(start))
|
||||
end_offset = min(video_len, math.ceil(end))
|
||||
video_targets[start_offset:end_offset] = label_id
|
||||
|
||||
caps = torch.LongTensor(
|
||||
[[self.cls_token_id, self.sep_token_id,
|
||||
self.pad_token_id, self.sep_token_id]],
|
||||
).repeat(vfeats.size(0), 1)
|
||||
cmasks = torch.BoolTensor(
|
||||
[[0, 1, 0, 1]] # pad are valid for attention.
|
||||
).repeat(vfeats.size(0), 1)
|
||||
return {
|
||||
"caps": caps,
|
||||
"cmasks": cmasks,
|
||||
"vfeats": vfeats, # X for original code.
|
||||
"vmasks": vmasks,
|
||||
"targets": targets,
|
||||
"video_id": video_id,
|
||||
"video_len": video_len, # for later checking.
|
||||
"video_targets": video_targets
|
||||
}
|
||||
|
||||
|
||||
class DiDeMoMetaProcessor(MetaProcessor):
|
||||
"""reference: https://github.com/LisaAnne/LocalizingMoments/blob/master/utils/eval.py
|
||||
https://github.com/LisaAnne/LocalizingMoments/blob/master/utils/data_processing.py
|
||||
"""
|
||||
def __init__(self, config):
|
||||
super().__init__(config)
|
||||
|
||||
assert "test" in self._get_split_path(config), "DiDeMo only supports zero-shot testing for now."
|
||||
|
||||
with open(self._get_split_path(config)) as data_file:
|
||||
json_data = json.load(data_file)
|
||||
|
||||
data = []
|
||||
for record in json_data:
|
||||
data.append((record["video"], record["description"]))
|
||||
self.data = data
|
||||
|
||||
def __len__(self):
|
||||
return len(self.data)
|
||||
|
||||
def __getitem__(self, idx):
|
||||
return self.data[idx]
|
||||
|
||||
|
||||
class DiDeMoTextProcessor(TextProcessor):
|
||||
"""reference: https://github.com/LisaAnne/LocalizingMoments/blob/master/utils/eval.py
|
||||
https://github.com/LisaAnne/LocalizingMoments/blob/master/utils/data_processing.py
|
||||
"""
|
||||
|
||||
def __call__(self, text):
|
||||
return self.tokenizer(text, add_special_tokens=False)["input_ids"]
|
||||
|
||||
|
||||
class DiDeMoAligner(DSAligner):
|
||||
"""
|
||||
check video length.
|
||||
"""
|
||||
|
||||
def __call__(self, video_id, video_feature, text_feature):
|
||||
# print(video_feature.shape[0])
|
||||
return super().__call__(video_id, video_feature, text_feature)
|
887
examples/MMPT/mmpt/processors/how2processor.py
Normal file
887
examples/MMPT/mmpt/processors/how2processor.py
Normal file
@ -0,0 +1,887 @@
|
||||
# coding=utf-8
|
||||
# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team.
|
||||
# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# Copyright (c) Facebook, Inc. All Rights Reserved
|
||||
|
||||
|
||||
import torch
|
||||
import math
|
||||
import pickle
|
||||
import random
|
||||
import os
|
||||
import numpy as np
|
||||
|
||||
from collections import deque
|
||||
from typing import Optional, Tuple, List
|
||||
from .processor import (
|
||||
Processor,
|
||||
MetaProcessor,
|
||||
TextProcessor,
|
||||
Aligner,
|
||||
MMAttentionMask2DProcessor
|
||||
)
|
||||
|
||||
from ..utils import ShardedTensor
|
||||
|
||||
|
||||
class How2MetaProcessor(MetaProcessor):
|
||||
def __init__(self, config):
|
||||
super().__init__(config)
|
||||
path = self._get_split_path(config)
|
||||
with open(path) as fd:
|
||||
self.data = [line.strip() for line in fd]
|
||||
|
||||
def __getitem__(self, idx):
|
||||
video_id = self.data[idx]
|
||||
return video_id, video_id
|
||||
|
||||
|
||||
class ShardedHow2MetaProcessor(How2MetaProcessor):
|
||||
def __init__(self, config):
|
||||
super().__init__(config)
|
||||
self.split = str(config.split)
|
||||
self.vfeat_dir = config.vfeat_dir
|
||||
self._init_shard()
|
||||
|
||||
def _init_shard(self):
|
||||
if self.split == "train":
|
||||
meta_fn = os.path.join(self.vfeat_dir, "train" + "_meta.pkl")
|
||||
with open(meta_fn, "rb") as fr:
|
||||
meta = pickle.load(fr)
|
||||
elif self.split == "valid":
|
||||
meta_fn = os.path.join(self.vfeat_dir, "val" + "_meta.pkl")
|
||||
with open(meta_fn, "rb") as fr:
|
||||
meta = pickle.load(fr)
|
||||
elif self.split == "test":
|
||||
print("use how2 val as test.")
|
||||
meta_fn = os.path.join(self.vfeat_dir, "val" + "_meta.pkl")
|
||||
with open(meta_fn, "rb") as fr:
|
||||
meta = pickle.load(fr)
|
||||
else:
|
||||
raise ValueError("unsupported for MetaProcessor:", self.split)
|
||||
video_id_to_shard = {}
|
||||
for shard_id in meta:
|
||||
for video_idx, video_id in enumerate(meta[shard_id]):
|
||||
video_id_to_shard[video_id] = (shard_id, video_idx)
|
||||
self.video_id_to_shard = video_id_to_shard
|
||||
|
||||
def __getitem__(self, idx):
|
||||
video_id, video_id = super().__getitem__(idx)
|
||||
shard_id, shard_idx = self.video_id_to_shard[video_id]
|
||||
meta = (video_id, idx, shard_id, shard_idx)
|
||||
return meta, meta
|
||||
|
||||
|
||||
class ShardedVideoProcessor(Processor):
|
||||
"""
|
||||
mmaped shards of numpy video features.
|
||||
"""
|
||||
|
||||
def __init__(self, config):
|
||||
self.split = str(config.split)
|
||||
self.vfeat_dir = config.vfeat_dir
|
||||
|
||||
def __call__(self, video_id):
|
||||
_, _, shard_id, video_idx = video_id
|
||||
if self.split == "train":
|
||||
shard = ShardedTensor.load(
|
||||
os.path.join(self.vfeat_dir, "train" + "_" + str(shard_id)),
|
||||
"r"
|
||||
)
|
||||
elif self.split == "valid":
|
||||
shard = ShardedTensor.load(
|
||||
os.path.join(self.vfeat_dir, "val" + "_" + str(shard_id)),
|
||||
"r"
|
||||
)
|
||||
elif self.split == "test":
|
||||
shard = ShardedTensor.load(
|
||||
os.path.join(self.vfeat_dir, "val" + "_" + str(shard_id)),
|
||||
"r"
|
||||
)
|
||||
else:
|
||||
raise ValueError("unknown split", self.split)
|
||||
feat = shard[video_idx]
|
||||
return feat
|
||||
|
||||
|
||||
class ShardedTextProcessor(Processor):
|
||||
def __init__(self, config):
|
||||
self.tfeat_dir = str(config.tfeat_dir)
|
||||
self.split = str(config.split)
|
||||
|
||||
def __call__(self, video_id):
|
||||
_, _, shard_id, shard_idx = video_id
|
||||
if self.split == "train":
|
||||
target_path = self.tfeat_dir + "train" + "_" + str(shard_id)
|
||||
elif self.split == "valid":
|
||||
target_path = self.tfeat_dir + "val" + "_" + str(shard_id)
|
||||
elif self.split == "test":
|
||||
target_path = self.tfeat_dir + "val" + "_" + str(shard_id)
|
||||
else:
|
||||
raise ValueError("unknown split", self.split)
|
||||
|
||||
startend = ShardedTensor.load(
|
||||
target_path + ".startends", "r")[shard_idx]
|
||||
cap_ids = ShardedTensor.load(
|
||||
target_path + ".caps_ids", "r")[shard_idx]
|
||||
cap = []
|
||||
for clip_idx in range(len(cap_ids)):
|
||||
clip = cap_ids[clip_idx]
|
||||
cap.append(clip[clip != -1].tolist())
|
||||
start, end = startend[:, 0].tolist(), startend[:, 1].tolist()
|
||||
return {"start": start, "end": end, "cap": cap}
|
||||
|
||||
|
||||
class FixedLenAligner(Aligner):
|
||||
"""
|
||||
In the model we assume text is on the left (closer to BERT formulation)
|
||||
and video is on the right.
|
||||
We fix the total length of text + video.
|
||||
max_video_len is in number of secs.
|
||||
max_text_len is in number of tokens.
|
||||
|
||||
special tokens formats:
|
||||
we use the format [CLS] [SEP] text tokens [SEP] [PAD] ...
|
||||
[CLS] will be splitted out into:
|
||||
[CLS] video tokens [SEP] text tokens [SEP] [PAD] ...
|
||||
token_type_ids will be generated by the model (for now).
|
||||
0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1
|
||||
| first sequence | second sequence |
|
||||
so each sequence owns a [SEP] token for no-ops.
|
||||
"""
|
||||
|
||||
def __init__(self, config):
|
||||
super().__init__(config)
|
||||
self.text_clip_sampler = TextClipSamplingProcessor(
|
||||
self.max_len - self.max_video_len - 3
|
||||
)
|
||||
"""
|
||||
decide subsampling:
|
||||
`config.subsampling` will change batch_size in trainer.
|
||||
`config.clip_per_video` (used by RetriTask) doesn't
|
||||
change batch_size in trainer.
|
||||
"""
|
||||
subsampling = config.subsampling \
|
||||
if config.subsampling is not None else None
|
||||
if config.clip_per_video is not None:
|
||||
subsampling = config.clip_per_video
|
||||
self.subsampling = subsampling
|
||||
|
||||
def _get_text_maxlen(self):
|
||||
# use max text len
|
||||
return self.text_clip_sampler.max_text_len
|
||||
|
||||
def __call__(self, video_id, video_feature, text_feature):
|
||||
from transformers import default_data_collator
|
||||
video_idx = video_id[1]
|
||||
if self.subsampling is not None and self.subsampling >= 1:
|
||||
batch = []
|
||||
for _ in range(self.subsampling):
|
||||
centerclip_idx = random.randint(
|
||||
0, len(text_feature["start"]) - 1)
|
||||
batch.append(
|
||||
self.sampling(
|
||||
video_idx,
|
||||
video_feature,
|
||||
text_feature,
|
||||
centerclip_idx,
|
||||
self._get_text_maxlen()
|
||||
))
|
||||
batch = self.batch_post_processing(batch, video_feature)
|
||||
batch = default_data_collator(batch)
|
||||
else:
|
||||
raise ValueError(
|
||||
"dataset.subsampling must be >= 1 for efficient video loading.")
|
||||
batch = self.sampling(video_idx, video_feature, text_feature)
|
||||
batch = self.batch_post_processing(batch, video_feature)
|
||||
|
||||
batch["video_id"] = video_id if isinstance(video_id, str) \
|
||||
else video_id[0]
|
||||
# e2e: make sure frame ids is into tensor.
|
||||
assert torch.is_tensor(batch["vfeats"])
|
||||
return batch
|
||||
|
||||
def sampling(
|
||||
self,
|
||||
video_idx,
|
||||
video_feature,
|
||||
text_feature,
|
||||
centerclip_idx=None,
|
||||
sampled_max_text_len=None,
|
||||
):
|
||||
text_clip_indexs = self.text_clip_sampler(
|
||||
text_feature, centerclip_idx,
|
||||
sampled_max_text_len
|
||||
)
|
||||
if isinstance(video_feature, np.ndarray):
|
||||
video_len = len(video_feature)
|
||||
else:
|
||||
video_len = math.ceil(text_feature["end"][-1])
|
||||
|
||||
video_end = min(
|
||||
math.ceil(text_feature["end"][text_clip_indexs[-1]]),
|
||||
video_len
|
||||
)
|
||||
video_start = max(
|
||||
min(
|
||||
math.floor(text_feature["start"][text_clip_indexs[0]]),
|
||||
video_end),
|
||||
0
|
||||
)
|
||||
|
||||
video_clips = {"start": [video_start], "end": [video_end]}
|
||||
|
||||
# tensorize.
|
||||
vfeats, vmasks = self._build_video_seq(
|
||||
video_feature, video_clips
|
||||
)
|
||||
caps, cmasks = self._build_text_seq(
|
||||
text_feature, text_clip_indexs
|
||||
)
|
||||
|
||||
text_start = text_clip_indexs[0]
|
||||
text_end = text_clip_indexs[-1] + 1
|
||||
|
||||
return {
|
||||
"caps": caps,
|
||||
"cmasks": cmasks,
|
||||
"vfeats": vfeats,
|
||||
"vmasks": vmasks,
|
||||
"video_start": video_start,
|
||||
"video_end": video_end,
|
||||
"text_start": text_start,
|
||||
"text_end": text_end,
|
||||
}
|
||||
|
||||
|
||||
class VariedLenAligner(FixedLenAligner):
|
||||
def __init__(self, config):
|
||||
super().__init__(config)
|
||||
self.sampled_min_len = config.sampled_min_len
|
||||
self.sampled_max_len = config.sampled_max_len
|
||||
|
||||
def _get_text_maxlen(self):
|
||||
return random.randint(self.sampled_min_len, self.sampled_max_len)
|
||||
|
||||
|
||||
class StartClipAligner(VariedLenAligner):
|
||||
def sampling(
|
||||
self,
|
||||
video_idx,
|
||||
video_feature,
|
||||
text_feature,
|
||||
centerclip_idx=None,
|
||||
sampled_max_text_len=None,
|
||||
):
|
||||
return super().sampling(
|
||||
video_idx, video_feature, text_feature, 0)
|
||||
|
||||
|
||||
class OverlappedAligner(VariedLenAligner):
|
||||
"""video clip and text clip has overlappings
|
||||
but may not be the same start/end."""
|
||||
def __init__(self, config):
|
||||
super().__init__(config)
|
||||
self.sampled_video_min_len = config.sampled_video_min_len
|
||||
self.sampled_video_max_len = config.sampled_video_max_len
|
||||
|
||||
self.video_clip_sampler = VideoClipSamplingProcessor()
|
||||
|
||||
def _get_video_maxlen(self):
|
||||
return random.randint(
|
||||
self.sampled_video_min_len, self.sampled_video_max_len)
|
||||
|
||||
def sampling(
|
||||
self,
|
||||
video_idx,
|
||||
video_feature,
|
||||
text_feature,
|
||||
centerclip_idx=None,
|
||||
sampled_max_text_len=None,
|
||||
):
|
||||
text_clip_indexs = self.text_clip_sampler(
|
||||
text_feature, centerclip_idx,
|
||||
sampled_max_text_len
|
||||
)
|
||||
if isinstance(video_feature, np.ndarray):
|
||||
video_len = len(video_feature)
|
||||
else:
|
||||
video_len = math.ceil(text_feature["end"][-1])
|
||||
low = math.floor(text_feature["start"][text_clip_indexs[0]])
|
||||
high = math.ceil(text_feature["end"][text_clip_indexs[-1]])
|
||||
if low < high:
|
||||
center = random.randint(low, high)
|
||||
else:
|
||||
center = int((low + high) // 2)
|
||||
center = max(0, min(video_feature.shape[0] - 1, center))
|
||||
|
||||
assert 0 <= center < video_feature.shape[0]
|
||||
|
||||
video_clips = self.video_clip_sampler(
|
||||
video_len, self._get_video_maxlen(), center
|
||||
)
|
||||
video_start = video_clips["start"][0]
|
||||
video_end = video_clips["end"][0]
|
||||
|
||||
# tensorize.
|
||||
vfeats, vmasks = self._build_video_seq(
|
||||
video_feature, video_clips
|
||||
)
|
||||
caps, cmasks = self._build_text_seq(
|
||||
text_feature, text_clip_indexs
|
||||
)
|
||||
|
||||
text_start = text_clip_indexs[0]
|
||||
text_end = text_clip_indexs[-1] + 1
|
||||
|
||||
return {
|
||||
"caps": caps,
|
||||
"cmasks": cmasks,
|
||||
"vfeats": vfeats,
|
||||
"vmasks": vmasks,
|
||||
"video_start": video_start,
|
||||
"video_end": video_end,
|
||||
"text_start": text_start,
|
||||
"text_end": text_end,
|
||||
}
|
||||
|
||||
|
||||
class MFMMLMAligner(FixedLenAligner):
|
||||
"""
|
||||
`FixedLenAligner` with Masked Language Model and Masked Frame Model.
|
||||
"""
|
||||
|
||||
def __init__(self, config):
|
||||
super().__init__(config)
|
||||
keep_prob = config.keep_prob if config.keep_prob is not None else 1.0
|
||||
self.text_clip_sampler = TextClipSamplingProcessor(
|
||||
self.max_len - self.max_video_len - 3, keep_prob
|
||||
)
|
||||
self.sampled_min_len = config.sampled_min_len
|
||||
self.sampled_max_len = config.sampled_max_len
|
||||
self.masked_token_sampler = TextMaskingProcessor(config)
|
||||
self.mm_type = config.mm_type \
|
||||
if config.mm_type is not None else "full"
|
||||
self.attnmasker = MMAttentionMask2DProcessor() \
|
||||
if self.mm_type == "textgen" else None
|
||||
self.masked_frame_sampler = FrameMaskingProcessor(config)
|
||||
self.lazy_vfeat_mask = (
|
||||
False if config.lazy_vfeat_mask is None else config.lazy_vfeat_mask
|
||||
)
|
||||
self.mm_prob = config.mm_prob if config.mm_prob is not None else 0.
|
||||
|
||||
def __call__(self, video_id, video_feature, text_feature):
|
||||
from transformers import default_data_collator
|
||||
if self.subsampling is not None and self.subsampling > 1:
|
||||
batch = []
|
||||
for _ in range(self.subsampling):
|
||||
centerclip_idx = random.randint(
|
||||
0, len(text_feature["start"]) - 1)
|
||||
sampled_max_text_len = random.randint(
|
||||
self.sampled_min_len, self.sampled_max_len
|
||||
)
|
||||
batch.append(
|
||||
self.sampling(
|
||||
video_id,
|
||||
video_feature,
|
||||
text_feature,
|
||||
centerclip_idx,
|
||||
sampled_max_text_len,
|
||||
)
|
||||
)
|
||||
batch = self.batch_post_processing(batch, video_feature)
|
||||
batch = default_data_collator(batch)
|
||||
else:
|
||||
batch = self.sampling(video_id, video_feature, text_feature)
|
||||
batch = self.batch_post_processing(batch, video_feature)
|
||||
batch["video_id"] = video_id if isinstance(video_id, str) \
|
||||
else video_id[0]
|
||||
return batch
|
||||
|
||||
def sampling(
|
||||
self,
|
||||
video_id,
|
||||
video_feature,
|
||||
text_feature,
|
||||
centerclip_idx=None,
|
||||
sampled_max_text_len=None,
|
||||
):
|
||||
output = FixedLenAligner.sampling(self,
|
||||
video_id, video_feature, text_feature,
|
||||
centerclip_idx, sampled_max_text_len)
|
||||
|
||||
masking_text, masking_video = None, None
|
||||
if random.random() < self.mm_prob:
|
||||
if random.random() > 0.5:
|
||||
masking_text, masking_video = self.mm_type, "no"
|
||||
else:
|
||||
masking_text, masking_video = "no", "full"
|
||||
video_feats = output["vfeats"] if not self.lazy_vfeat_mask else None
|
||||
video_label = self.masked_frame_sampler(
|
||||
output["vmasks"], masking_video, vfeats=video_feats)
|
||||
caps, text_label = self.masked_token_sampler(
|
||||
output["caps"], masking_text)
|
||||
|
||||
output.update({
|
||||
"caps": caps,
|
||||
"video_label": video_label,
|
||||
"text_label": text_label,
|
||||
})
|
||||
|
||||
if self.attnmasker is not None:
|
||||
attention_mask = self.attnmasker(
|
||||
output["vmasks"], output["cmasks"], masking_text)
|
||||
output.update({
|
||||
"attention_mask": attention_mask
|
||||
})
|
||||
return output
|
||||
|
||||
|
||||
class FrameMaskingProcessor(Processor):
|
||||
def __init__(self, config):
|
||||
self.mfm_probability = 0.15
|
||||
if config.mfm_probability is not None:
|
||||
self.mfm_probability = config.mfm_probability
|
||||
|
||||
def __call__(self, vmasks, modality_masking=None, vfeats=None):
|
||||
"""
|
||||
We perform lazy masking to save data transfer time.
|
||||
It only generates video_labels by default and MFM model
|
||||
will do actualy masking.
|
||||
Return: `video_label` is a binary mask.
|
||||
"""
|
||||
video_label = vmasks.clone()
|
||||
if modality_masking is not None:
|
||||
if modality_masking == "full":
|
||||
probability_matrix = torch.full(video_label.shape, 1.)
|
||||
elif modality_masking == "no":
|
||||
probability_matrix = torch.full(video_label.shape, 0.)
|
||||
elif modality_masking == "inverse":
|
||||
probability_matrix = torch.full(
|
||||
video_label.shape, 1. - self.mfm_probability)
|
||||
else:
|
||||
raise ValueError("unknown modality masking.", modality_masking)
|
||||
else:
|
||||
probability_matrix = torch.full(
|
||||
video_label.shape, self.mfm_probability)
|
||||
masked_indices = torch.bernoulli(probability_matrix).bool()
|
||||
# We only compute loss on masked tokens
|
||||
video_label[~masked_indices] = 0
|
||||
if vfeats is not None:
|
||||
vfeats[video_label, :] = 0.0
|
||||
return video_label
|
||||
|
||||
|
||||
class TextGenerationProcessor(Processor):
|
||||
def __init__(self, tokenizer):
|
||||
self.bos_token_id = tokenizer.bos_token_id
|
||||
self.pad_token_id = tokenizer.pad_token_id
|
||||
|
||||
def __call__(self, inputs):
|
||||
labels = inputs.clone()
|
||||
# [CLS] [SEP] for video
|
||||
labels[:2] = -100
|
||||
# keep [SEP] for text.
|
||||
pad_mask = labels == self.pad_token_id
|
||||
labels[pad_mask] = -100
|
||||
inputs[2:] = torch.cat([
|
||||
torch.LongTensor([self.bos_token_id]),
|
||||
inputs[2:-1]])
|
||||
inputs[pad_mask] = self.pad_token_id
|
||||
assert len(inputs) == len(labels)
|
||||
return inputs, labels
|
||||
|
||||
|
||||
class TextMaskingProcessor(Processor):
|
||||
def __init__(self, config):
|
||||
"""this function is borrowed from
|
||||
`transformers/data/data_collator.DataCollatorForLanguageModeling`"""
|
||||
self.mlm_probability = 0.15
|
||||
if config.mlm_probability is not None:
|
||||
self.mlm_probability = config.mlm_probability
|
||||
self.bert_name = config.bert_name
|
||||
# [CLS] is used as bos_token and [SEP] is used as eos_token.
|
||||
# https://huggingface.co/transformers/master/model_doc/bertgeneration.html
|
||||
from transformers import AutoTokenizer
|
||||
self.tokenizer = AutoTokenizer.from_pretrained(
|
||||
self.bert_name, bos_token="[CLS]", eos_token="[SEP]")
|
||||
self.textgen = TextGenerationProcessor(self.tokenizer)
|
||||
|
||||
def __call__(
|
||||
self, inputs: torch.Tensor,
|
||||
modality_masking=None,
|
||||
special_tokens_mask: Optional[torch.Tensor] = None
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
expand modality_masking into
|
||||
None: traditional bert masking.
|
||||
"no": no masking.
|
||||
"full": all [MASK] token for generation.
|
||||
"gen": autoregressive generation.
|
||||
"""
|
||||
"""
|
||||
Prepare masked tokens inputs/labels for masked language modeling:
|
||||
80% MASK, 10% random, 10% original.
|
||||
"""
|
||||
labels = inputs.clone()
|
||||
# We sample a few tokens in each sequence for MLM training
|
||||
# (with probability `self.mlm_probability`)
|
||||
if modality_masking is not None:
|
||||
if modality_masking == "full":
|
||||
probability_matrix = torch.full(labels.shape, 1.)
|
||||
elif modality_masking == "no":
|
||||
probability_matrix = torch.full(labels.shape, 0.)
|
||||
elif modality_masking.startswith("textgen"):
|
||||
# [CLS] [SEP] <s> ...
|
||||
inputs, labels = self.textgen(inputs)
|
||||
if "mask" not in modality_masking:
|
||||
return inputs, labels
|
||||
inputs = self.mask_input(inputs, special_tokens_mask)
|
||||
return inputs, labels
|
||||
elif modality_masking == "mask":
|
||||
inputs = self.mask_input(inputs, special_tokens_mask)
|
||||
labels = torch.full(inputs.shape, -100)
|
||||
return inputs, labels
|
||||
elif modality_masking == "inverse":
|
||||
probability_matrix = torch.full(labels.shape, 1. - self.mlm_probability)
|
||||
else:
|
||||
raise ValueError("unknown modality masking.", modality_masking)
|
||||
else:
|
||||
probability_matrix = torch.full(labels.shape, self.mlm_probability)
|
||||
|
||||
if special_tokens_mask is None:
|
||||
special_tokens_mask = self.get_special_tokens_mask(
|
||||
labels.tolist(), already_has_special_tokens=True
|
||||
)
|
||||
special_tokens_mask = torch.tensor(
|
||||
special_tokens_mask, dtype=torch.bool)
|
||||
else:
|
||||
special_tokens_mask = special_tokens_mask.bool()
|
||||
|
||||
probability_matrix.masked_fill_(special_tokens_mask, value=0.0)
|
||||
masked_indices = torch.bernoulli(probability_matrix).bool()
|
||||
labels[~masked_indices] = -100 # We only compute loss on masked tokens
|
||||
|
||||
# 80% of the time,
|
||||
# we replace masked input tokens with tokenizer.mask_token ([MASK])
|
||||
indices_replaced = (
|
||||
torch.bernoulli(
|
||||
torch.full(labels.shape, 0.8)).bool() & masked_indices
|
||||
)
|
||||
inputs[indices_replaced] = self.tokenizer.convert_tokens_to_ids(
|
||||
self.tokenizer.mask_token
|
||||
)
|
||||
|
||||
# 10% of the time, we replace masked input tokens with random word
|
||||
indices_random = (
|
||||
torch.bernoulli(torch.full(labels.shape, 0.5)).bool()
|
||||
& masked_indices
|
||||
& ~indices_replaced
|
||||
)
|
||||
random_words = torch.randint(
|
||||
len(self.tokenizer), labels.shape, dtype=torch.long
|
||||
)
|
||||
inputs[indices_random] = random_words[indices_random]
|
||||
|
||||
# The rest of the time (10% of the time) we keep the masked input
|
||||
# tokens unchanged
|
||||
return inputs, labels
|
||||
|
||||
def mask_input(self, inputs, special_tokens_mask=None):
|
||||
# the following is new with masked autoregressive.
|
||||
probability_matrix = torch.full(
|
||||
inputs.shape, self.mlm_probability)
|
||||
if special_tokens_mask is None:
|
||||
special_tokens_mask = self.get_special_tokens_mask(
|
||||
inputs.tolist(), already_has_special_tokens=True
|
||||
)
|
||||
special_tokens_mask = torch.tensor(
|
||||
special_tokens_mask, dtype=torch.bool)
|
||||
else:
|
||||
special_tokens_mask = special_tokens_mask.bool()
|
||||
probability_matrix.masked_fill_(special_tokens_mask, value=0.0)
|
||||
masked_indices = torch.bernoulli(probability_matrix).bool()
|
||||
indices_replaced = (
|
||||
torch.bernoulli(
|
||||
torch.full(inputs.shape, 0.8)).bool() & masked_indices
|
||||
)
|
||||
inputs[indices_replaced] = self.tokenizer.convert_tokens_to_ids(
|
||||
self.tokenizer.mask_token
|
||||
)
|
||||
|
||||
# 10% of the time, we replace masked input tokens with random word
|
||||
indices_random = (
|
||||
torch.bernoulli(torch.full(inputs.shape, 0.5)).bool()
|
||||
& masked_indices
|
||||
& ~indices_replaced
|
||||
)
|
||||
random_words = torch.randint(
|
||||
len(self.tokenizer), inputs.shape, dtype=torch.long
|
||||
)
|
||||
inputs[indices_random] = random_words[indices_random]
|
||||
return inputs
|
||||
|
||||
def get_special_tokens_mask(
|
||||
self, token_ids_0: List[int],
|
||||
token_ids_1: Optional[List[int]] = None,
|
||||
already_has_special_tokens: bool = False
|
||||
) -> List[int]:
|
||||
"""
|
||||
Note: the version from transformers do not consider pad
|
||||
as special tokens.
|
||||
"""
|
||||
|
||||
if already_has_special_tokens:
|
||||
if token_ids_1 is not None:
|
||||
raise ValueError(
|
||||
"You should not supply a second sequence if"
|
||||
"the provided sequence of "
|
||||
"ids is already formated with special tokens "
|
||||
"for the model."
|
||||
)
|
||||
return list(map(lambda x: 1 if x in [
|
||||
self.tokenizer.sep_token_id,
|
||||
self.tokenizer.cls_token_id,
|
||||
self.tokenizer.pad_token_id] else 0, token_ids_0))
|
||||
|
||||
if token_ids_1 is not None:
|
||||
return [1] + ([0] * len(token_ids_0)) + [1] + ([0] * len(token_ids_1)) + [1]
|
||||
return [1] + ([0] * len(token_ids_0)) + [1]
|
||||
|
||||
|
||||
class TextClipSamplingProcessor(Processor):
|
||||
def __init__(self, max_text_len, keep_prob=1.0):
|
||||
self.max_text_len = max_text_len
|
||||
self.max_video_len = 256 # always hold.
|
||||
self.keep_prob = keep_prob
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
text_feature,
|
||||
centerclip_idx=None,
|
||||
sampled_max_text_len=None,
|
||||
sampled_max_video_len=None,
|
||||
):
|
||||
# Let's use all caps for now and see if 256 can cover all of them.
|
||||
if sampled_max_text_len is not None:
|
||||
max_text_len = sampled_max_text_len
|
||||
else:
|
||||
max_text_len = self.max_text_len
|
||||
if sampled_max_video_len is not None:
|
||||
max_video_len = sampled_max_video_len
|
||||
else:
|
||||
max_video_len = self.max_video_len
|
||||
|
||||
t_num_clips = len(text_feature["start"])
|
||||
|
||||
if centerclip_idx is None:
|
||||
centerclip_idx = random.randint(0, t_num_clips - 1)
|
||||
|
||||
start_idx, end_idx = centerclip_idx, centerclip_idx + 1
|
||||
text_clip_indexs = deque()
|
||||
text_clip_indexs.append(start_idx)
|
||||
text_len = len(text_feature["cap"][start_idx])
|
||||
|
||||
video_len = max(
|
||||
0,
|
||||
text_feature["end"][start_idx]
|
||||
- text_feature["start"][start_idx],
|
||||
)
|
||||
|
||||
while (
|
||||
(start_idx > 0 or end_idx < t_num_clips)
|
||||
and text_len < max_text_len
|
||||
and video_len < max_video_len
|
||||
):
|
||||
if random.random() > 0.5 and end_idx < t_num_clips:
|
||||
# skip the next one?
|
||||
if random.random() > self.keep_prob and (end_idx + 1) < t_num_clips:
|
||||
end_idx = end_idx + 1
|
||||
text_clip_indexs.append(end_idx)
|
||||
text_len += len(text_feature["cap"][end_idx])
|
||||
end_idx += 1
|
||||
elif start_idx > 0:
|
||||
if random.random() > self.keep_prob and (start_idx - 1) > 0:
|
||||
start_idx = start_idx - 1
|
||||
start_idx -= 1
|
||||
text_clip_indexs.insert(0, start_idx)
|
||||
text_len += len(text_feature["cap"][start_idx])
|
||||
else:
|
||||
if end_idx < t_num_clips:
|
||||
if random.random() > self.keep_prob and (end_idx + 1) < t_num_clips:
|
||||
end_idx = end_idx + 1
|
||||
text_clip_indexs.append(end_idx)
|
||||
text_len += len(text_feature["cap"][end_idx])
|
||||
end_idx += 1
|
||||
else:
|
||||
return text_clip_indexs
|
||||
video_len = max(
|
||||
0,
|
||||
text_feature["end"][text_clip_indexs[-1]]
|
||||
- text_feature["start"][text_clip_indexs[0]],
|
||||
)
|
||||
return text_clip_indexs
|
||||
|
||||
|
||||
class VideoClipSamplingProcessor(Processor):
|
||||
def __call__(self, video_len, max_video_len, center):
|
||||
"""
|
||||
`video_len`: length of the video.
|
||||
`max_video_len`: maximum video tokens allowd in a sequence.
|
||||
`center`: initial starting index.
|
||||
"""
|
||||
assert center >= 0 and center < video_len
|
||||
t_clip_len = 0
|
||||
start, end = center, center
|
||||
while (start > 0 or end < video_len) and t_clip_len < max_video_len:
|
||||
# decide the direction to grow.
|
||||
if start <= 0:
|
||||
end += 1
|
||||
elif end >= video_len:
|
||||
start -= 1
|
||||
elif random.random() > 0.5:
|
||||
end += 1
|
||||
else:
|
||||
start -= 1
|
||||
t_clip_len += 1
|
||||
return {"start": [start], "end": [end]}
|
||||
|
||||
|
||||
class How2MILNCEAligner(FixedLenAligner):
|
||||
"""reference: `antoine77340/MIL-NCE_HowTo100M/video_loader.py`"""
|
||||
|
||||
def __init__(self, config):
|
||||
super().__init__(config)
|
||||
self.num_candidates = 4
|
||||
self.min_time = 5.0
|
||||
self.num_sec = 3.2
|
||||
# self.num_sec = self.num_frames / float(self.fps) num_frames=16 / fps = 5
|
||||
# self.num_frames = 16
|
||||
|
||||
def sampling(
|
||||
self,
|
||||
video_id,
|
||||
video_feature,
|
||||
text_feature,
|
||||
centerclip_idx=None, # will be ignored.
|
||||
sampled_max_text_len=None # will be ignored.
|
||||
):
|
||||
text, start, end = self._get_text(text_feature)
|
||||
video = self._get_video(video_feature, start, end)
|
||||
|
||||
vfeats = torch.zeros((self.max_video_len, video_feature.shape[1]))
|
||||
vmasks = torch.zeros((self.max_video_len,), dtype=torch.bool)
|
||||
vfeats[: video.shape[0]] = torch.from_numpy(np.array(video))
|
||||
vmasks[: video.shape[0]] = 1
|
||||
|
||||
caps, cmasks = [], []
|
||||
for words in text:
|
||||
cap, cmask = self._build_text_seq(text_feature, words)
|
||||
caps.append(cap)
|
||||
cmasks.append(cmask)
|
||||
caps = torch.stack(caps)
|
||||
cmasks = torch.stack(cmasks)
|
||||
# video of shape: (video_len)
|
||||
# text of shape (num_candidates, max_text_len)
|
||||
|
||||
return {
|
||||
"caps": caps,
|
||||
"cmasks": cmasks,
|
||||
"vfeats": vfeats,
|
||||
"vmasks": vmasks,
|
||||
# "video_id": video_id,
|
||||
}
|
||||
|
||||
def _get_video(self, video_feature, start, end):
|
||||
start_seek = random.randint(start, int(max(start, end - self.num_sec)))
|
||||
# duration = self.num_sec + 0.1
|
||||
return video_feature[start_seek : int(start_seek + self.num_sec)]
|
||||
|
||||
def _get_text(self, cap):
|
||||
ind = random.randint(0, len(cap["start"]) - 1)
|
||||
if self.num_candidates == 1:
|
||||
words = [ind]
|
||||
else:
|
||||
words = []
|
||||
cap_start = self._find_nearest_candidates(cap, ind)
|
||||
for i in range(self.num_candidates):
|
||||
words.append([max(0, min(len(cap["cap"]) - 1, cap_start + i))])
|
||||
|
||||
start, end = cap["start"][ind], cap["end"][ind]
|
||||
# TODO: May need to be improved for edge cases.
|
||||
# expand the min time.
|
||||
if end - start < self.min_time:
|
||||
diff = self.min_time - end + start
|
||||
start = max(0, start - diff / 2)
|
||||
end = start + self.min_time
|
||||
return words, int(start), int(end)
|
||||
|
||||
def _find_nearest_candidates(self, caption, ind):
|
||||
"""find the range of the clips."""
|
||||
start, end = ind, ind
|
||||
#diff = caption["end"][end] - caption["start"][start]
|
||||
n_candidate = 1
|
||||
while n_candidate < self.num_candidates:
|
||||
# the first clip
|
||||
if start == 0:
|
||||
return 0
|
||||
# we add () in the following condition to fix the bug.
|
||||
elif end == (len(caption["start"]) - 1):
|
||||
return start - (self.num_candidates - n_candidate)
|
||||
elif (caption["end"][end] - caption["start"][start - 1]) < (
|
||||
caption["end"][end + 1] - caption["start"][start]
|
||||
):
|
||||
start -= 1
|
||||
else:
|
||||
end += 1
|
||||
n_candidate += 1
|
||||
return start
|
||||
|
||||
|
||||
class PKLJSONStrTextProcessor(TextProcessor):
|
||||
"""`caption.json` from howto100m are preprocessed as a
|
||||
dict `[video_id, json_str]`.
|
||||
Json parsing tokenization are conducted on-the-fly and cached into dict.
|
||||
"""
|
||||
|
||||
def __init__(self, config, max_clip_text_len=96):
|
||||
print("[Warning] PKLJSONStrTextProcessor is slow for num_workers > 0.")
|
||||
self.caption_pkl_path = str(config.caption_pkl_path)
|
||||
with open(self.caption_pkl_path, "rb") as fd:
|
||||
self.data = pickle.load(fd)
|
||||
self.max_clip_text_len = max_clip_text_len
|
||||
from transformers import AutoTokenizer
|
||||
self.tokenizer = AutoTokenizer.from_pretrained(
|
||||
str(config.bert_name), use_fast=config.use_fast
|
||||
)
|
||||
|
||||
def __call__(self, video_id):
|
||||
caption = self.data[video_id]
|
||||
if isinstance(caption, str):
|
||||
import json
|
||||
caption = json.loads(caption)
|
||||
cap = []
|
||||
for clip_idx, text_clip in enumerate(caption["text"]):
|
||||
clip_ids = []
|
||||
if isinstance(text_clip, str):
|
||||
clip_ids = self.tokenizer(
|
||||
text_clip[: self.max_clip_text_len],
|
||||
add_special_tokens=False
|
||||
)["input_ids"]
|
||||
cap.append(clip_ids)
|
||||
caption["cap"] = cap
|
||||
caption.pop("text") # save space.
|
||||
self.data[video_id] = caption
|
||||
return caption
|
100
examples/MMPT/mmpt/processors/how2retriprocessor.py
Normal file
100
examples/MMPT/mmpt/processors/how2retriprocessor.py
Normal file
@ -0,0 +1,100 @@
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
#
|
||||
# This source code is licensed under the MIT license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
|
||||
from .how2processor import (
|
||||
ShardedHow2MetaProcessor,
|
||||
ShardedVideoProcessor,
|
||||
ShardedTextProcessor,
|
||||
VariedLenAligner,
|
||||
OverlappedAligner
|
||||
)
|
||||
|
||||
|
||||
class ShardedHow2VideoRetriMetaProcessor(ShardedHow2MetaProcessor):
|
||||
def __init__(self, config):
|
||||
super().__init__(config)
|
||||
self.num_video_per_batch = config.num_video_per_batch
|
||||
self.cands = [
|
||||
self.data[batch_offset:batch_offset + self.num_video_per_batch]
|
||||
for batch_offset in
|
||||
range(0, (len(self.data) // (8 * self.num_video_per_batch)) * 8 * self.num_video_per_batch, self.num_video_per_batch)]
|
||||
|
||||
def __len__(self):
|
||||
return len(self.cands)
|
||||
|
||||
def set_candidates(self, cands):
|
||||
# no changes on num of batches.
|
||||
print(len(self.cands), "->", len(cands))
|
||||
# assert len(self.cands) == len(cands)
|
||||
self.cands = cands
|
||||
|
||||
def __getitem__(self, idx):
|
||||
video_ids = self.cands[idx]
|
||||
assert isinstance(video_ids, list)
|
||||
sharded_video_idxs = []
|
||||
for video_id in video_ids:
|
||||
shard_id, video_idx = self.video_id_to_shard[video_id]
|
||||
sharded_video_idxs.append((video_id, -1, shard_id, video_idx))
|
||||
return sharded_video_idxs, sharded_video_idxs
|
||||
|
||||
|
||||
class ShardedVideoRetriVideoProcessor(ShardedVideoProcessor):
|
||||
"""In retrival case the video_id
|
||||
is a list of tuples: `(shard_id, video_idx)` ."""
|
||||
|
||||
def __call__(self, sharded_video_idxs):
|
||||
assert isinstance(sharded_video_idxs, list)
|
||||
cand_feats = []
|
||||
for shared_video_idx in sharded_video_idxs:
|
||||
feat = super().__call__(shared_video_idx)
|
||||
cand_feats.append(feat)
|
||||
return cand_feats
|
||||
|
||||
|
||||
class ShardedVideoRetriTextProcessor(ShardedTextProcessor):
|
||||
"""In retrival case the video_id
|
||||
is a list of tuples: `(shard_id, video_idx)` ."""
|
||||
|
||||
def __call__(self, sharded_video_idxs):
|
||||
assert isinstance(sharded_video_idxs, list)
|
||||
cand_caps = []
|
||||
for shared_video_idx in sharded_video_idxs:
|
||||
caps = super().__call__(shared_video_idx)
|
||||
cand_caps.append(caps)
|
||||
return cand_caps
|
||||
|
||||
|
||||
class VideoRetriAligner(VariedLenAligner):
|
||||
# Retritask will trim dim-0.
|
||||
def __call__(self, sharded_video_idxs, video_features, text_features):
|
||||
from transformers import default_data_collator
|
||||
batch, video_ids = [], []
|
||||
for video_id, video_feature, text_feature in \
|
||||
zip(sharded_video_idxs, video_features, text_features):
|
||||
sub_batch = super().__call__(video_id, video_feature, text_feature)
|
||||
batch.append(sub_batch)
|
||||
if isinstance(video_id, tuple):
|
||||
video_id = video_id[0]
|
||||
video_ids.append(video_id)
|
||||
batch = default_data_collator(batch)
|
||||
batch["video_id"] = video_ids
|
||||
return batch
|
||||
|
||||
|
||||
class VideoRetriOverlappedAligner(OverlappedAligner):
|
||||
# Retritask will trim dim-0.
|
||||
def __call__(self, sharded_video_idxs, video_features, text_features):
|
||||
from transformers import default_data_collator
|
||||
batch, video_ids = [], []
|
||||
for video_id, video_feature, text_feature in \
|
||||
zip(sharded_video_idxs, video_features, text_features):
|
||||
sub_batch = super().__call__(video_id, video_feature, text_feature)
|
||||
batch.append(sub_batch)
|
||||
if isinstance(video_id, tuple):
|
||||
video_id = video_id[0]
|
||||
video_ids.append(video_id)
|
||||
batch = default_data_collator(batch)
|
||||
batch["video_id"] = video_ids
|
||||
return batch
|
336
examples/MMPT/mmpt/processors/models/s3dg.py
Normal file
336
examples/MMPT/mmpt/processors/models/s3dg.py
Normal file
@ -0,0 +1,336 @@
|
||||
# This source code is licensed under the MIT license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
|
||||
"""Contains a PyTorch definition for Gated Separable 3D network (S3D-G)
|
||||
with a text module for computing joint text-video embedding from raw text
|
||||
and video input. The following code will enable you to load the HowTo100M
|
||||
pretrained S3D Text-Video model from:
|
||||
A. Miech, J.-B. Alayrac, L. Smaira, I. Laptev, J. Sivic and A. Zisserman,
|
||||
End-to-End Learning of Visual Representations from Uncurated Instructional Videos.
|
||||
https://arxiv.org/abs/1912.06430.
|
||||
|
||||
S3D-G was proposed by:
|
||||
S. Xie, C. Sun, J. Huang, Z. Tu and K. Murphy,
|
||||
Rethinking Spatiotemporal Feature Learning For Video Understanding.
|
||||
https://arxiv.org/abs/1712.04851.
|
||||
Tensorflow code: https://github.com/tensorflow/models/blob/master/research/slim/nets/s3dg.py
|
||||
|
||||
The S3D architecture was slightly modified with a space to depth trick for TPU
|
||||
optimization.
|
||||
"""
|
||||
|
||||
import torch as th
|
||||
import torch.nn.functional as F
|
||||
import torch.nn as nn
|
||||
import os
|
||||
import numpy as np
|
||||
import re
|
||||
|
||||
|
||||
class InceptionBlock(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
input_dim,
|
||||
num_outputs_0_0a,
|
||||
num_outputs_1_0a,
|
||||
num_outputs_1_0b,
|
||||
num_outputs_2_0a,
|
||||
num_outputs_2_0b,
|
||||
num_outputs_3_0b,
|
||||
gating=True,
|
||||
):
|
||||
super(InceptionBlock, self).__init__()
|
||||
self.conv_b0 = STConv3D(input_dim, num_outputs_0_0a, [1, 1, 1])
|
||||
self.conv_b1_a = STConv3D(input_dim, num_outputs_1_0a, [1, 1, 1])
|
||||
self.conv_b1_b = STConv3D(
|
||||
num_outputs_1_0a, num_outputs_1_0b, [3, 3, 3], padding=1, separable=True
|
||||
)
|
||||
self.conv_b2_a = STConv3D(input_dim, num_outputs_2_0a, [1, 1, 1])
|
||||
self.conv_b2_b = STConv3D(
|
||||
num_outputs_2_0a, num_outputs_2_0b, [3, 3, 3], padding=1, separable=True
|
||||
)
|
||||
self.maxpool_b3 = th.nn.MaxPool3d((3, 3, 3), stride=1, padding=1)
|
||||
self.conv_b3_b = STConv3D(input_dim, num_outputs_3_0b, [1, 1, 1])
|
||||
self.gating = gating
|
||||
self.output_dim = (
|
||||
num_outputs_0_0a + num_outputs_1_0b + num_outputs_2_0b + num_outputs_3_0b
|
||||
)
|
||||
if gating:
|
||||
self.gating_b0 = SelfGating(num_outputs_0_0a)
|
||||
self.gating_b1 = SelfGating(num_outputs_1_0b)
|
||||
self.gating_b2 = SelfGating(num_outputs_2_0b)
|
||||
self.gating_b3 = SelfGating(num_outputs_3_0b)
|
||||
|
||||
def forward(self, input):
|
||||
"""Inception block
|
||||
"""
|
||||
b0 = self.conv_b0(input)
|
||||
b1 = self.conv_b1_a(input)
|
||||
b1 = self.conv_b1_b(b1)
|
||||
b2 = self.conv_b2_a(input)
|
||||
b2 = self.conv_b2_b(b2)
|
||||
b3 = self.maxpool_b3(input)
|
||||
b3 = self.conv_b3_b(b3)
|
||||
if self.gating:
|
||||
b0 = self.gating_b0(b0)
|
||||
b1 = self.gating_b1(b1)
|
||||
b2 = self.gating_b2(b2)
|
||||
b3 = self.gating_b3(b3)
|
||||
return th.cat((b0, b1, b2, b3), dim=1)
|
||||
|
||||
|
||||
class SelfGating(nn.Module):
|
||||
def __init__(self, input_dim):
|
||||
super(SelfGating, self).__init__()
|
||||
self.fc = nn.Linear(input_dim, input_dim)
|
||||
|
||||
def forward(self, input_tensor):
|
||||
"""Feature gating as used in S3D-G.
|
||||
"""
|
||||
spatiotemporal_average = th.mean(input_tensor, dim=[2, 3, 4])
|
||||
weights = self.fc(spatiotemporal_average)
|
||||
weights = th.sigmoid(weights)
|
||||
return weights[:, :, None, None, None] * input_tensor
|
||||
|
||||
|
||||
class STConv3D(nn.Module):
|
||||
def __init__(
|
||||
self, input_dim, output_dim, kernel_size, stride=1, padding=0, separable=False
|
||||
):
|
||||
super(STConv3D, self).__init__()
|
||||
self.separable = separable
|
||||
self.relu = nn.ReLU(inplace=True)
|
||||
assert len(kernel_size) == 3
|
||||
if separable and kernel_size[0] != 1:
|
||||
spatial_kernel_size = [1, kernel_size[1], kernel_size[2]]
|
||||
temporal_kernel_size = [kernel_size[0], 1, 1]
|
||||
if isinstance(stride, list) and len(stride) == 3:
|
||||
spatial_stride = [1, stride[1], stride[2]]
|
||||
temporal_stride = [stride[0], 1, 1]
|
||||
else:
|
||||
spatial_stride = [1, stride, stride]
|
||||
temporal_stride = [stride, 1, 1]
|
||||
if isinstance(padding, list) and len(padding) == 3:
|
||||
spatial_padding = [0, padding[1], padding[2]]
|
||||
temporal_padding = [padding[0], 0, 0]
|
||||
else:
|
||||
spatial_padding = [0, padding, padding]
|
||||
temporal_padding = [padding, 0, 0]
|
||||
if separable:
|
||||
self.conv1 = nn.Conv3d(
|
||||
input_dim,
|
||||
output_dim,
|
||||
kernel_size=spatial_kernel_size,
|
||||
stride=spatial_stride,
|
||||
padding=spatial_padding,
|
||||
bias=False,
|
||||
)
|
||||
self.bn1 = nn.BatchNorm3d(output_dim)
|
||||
self.conv2 = nn.Conv3d(
|
||||
output_dim,
|
||||
output_dim,
|
||||
kernel_size=temporal_kernel_size,
|
||||
stride=temporal_stride,
|
||||
padding=temporal_padding,
|
||||
bias=False,
|
||||
)
|
||||
self.bn2 = nn.BatchNorm3d(output_dim)
|
||||
else:
|
||||
self.conv1 = nn.Conv3d(
|
||||
input_dim,
|
||||
output_dim,
|
||||
kernel_size=kernel_size,
|
||||
stride=stride,
|
||||
padding=padding,
|
||||
bias=False,
|
||||
)
|
||||
self.bn1 = nn.BatchNorm3d(output_dim)
|
||||
|
||||
def forward(self, input):
|
||||
out = self.relu(self.bn1(self.conv1(input)))
|
||||
if self.separable:
|
||||
out = self.relu(self.bn2(self.conv2(out)))
|
||||
return out
|
||||
|
||||
|
||||
class MaxPool3dTFPadding(th.nn.Module):
|
||||
def __init__(self, kernel_size, stride=None, padding="SAME"):
|
||||
super(MaxPool3dTFPadding, self).__init__()
|
||||
if padding == "SAME":
|
||||
padding_shape = self._get_padding_shape(kernel_size, stride)
|
||||
self.padding_shape = padding_shape
|
||||
self.pad = th.nn.ConstantPad3d(padding_shape, 0)
|
||||
self.pool = th.nn.MaxPool3d(kernel_size, stride, ceil_mode=True)
|
||||
|
||||
def _get_padding_shape(self, filter_shape, stride):
|
||||
def _pad_top_bottom(filter_dim, stride_val):
|
||||
pad_along = max(filter_dim - stride_val, 0)
|
||||
pad_top = pad_along // 2
|
||||
pad_bottom = pad_along - pad_top
|
||||
return pad_top, pad_bottom
|
||||
|
||||
padding_shape = []
|
||||
for filter_dim, stride_val in zip(filter_shape, stride):
|
||||
pad_top, pad_bottom = _pad_top_bottom(filter_dim, stride_val)
|
||||
padding_shape.append(pad_top)
|
||||
padding_shape.append(pad_bottom)
|
||||
depth_top = padding_shape.pop(0)
|
||||
depth_bottom = padding_shape.pop(0)
|
||||
padding_shape.append(depth_top)
|
||||
padding_shape.append(depth_bottom)
|
||||
return tuple(padding_shape)
|
||||
|
||||
def forward(self, inp):
|
||||
inp = self.pad(inp)
|
||||
out = self.pool(inp)
|
||||
return out
|
||||
|
||||
|
||||
class Sentence_Embedding(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
embd_dim,
|
||||
num_embeddings=66250,
|
||||
word_embedding_dim=300,
|
||||
token_to_word_path="dict.npy",
|
||||
max_words=16,
|
||||
output_dim=2048,
|
||||
):
|
||||
super(Sentence_Embedding, self).__init__()
|
||||
self.word_embd = nn.Embedding(num_embeddings, word_embedding_dim)
|
||||
self.fc1 = nn.Linear(word_embedding_dim, output_dim)
|
||||
self.fc2 = nn.Linear(output_dim, embd_dim)
|
||||
self.word_to_token = {}
|
||||
self.max_words = max_words
|
||||
token_to_word = np.load(token_to_word_path)
|
||||
for i, t in enumerate(token_to_word):
|
||||
self.word_to_token[t] = i + 1
|
||||
|
||||
def _zero_pad_tensor_token(self, tensor, size):
|
||||
if len(tensor) >= size:
|
||||
return tensor[:size]
|
||||
else:
|
||||
zero = th.zeros(size - len(tensor)).long()
|
||||
return th.cat((tensor, zero), dim=0)
|
||||
|
||||
def _split_text(self, sentence):
|
||||
w = re.findall(r"[\w']+", str(sentence))
|
||||
return w
|
||||
|
||||
def _words_to_token(self, words):
|
||||
words = [
|
||||
self.word_to_token[word] for word in words if word in self.word_to_token
|
||||
]
|
||||
if words:
|
||||
we = self._zero_pad_tensor_token(th.LongTensor(words), self.max_words)
|
||||
return we
|
||||
else:
|
||||
return th.zeros(self.max_words).long()
|
||||
|
||||
def _words_to_ids(self, x):
|
||||
split_x = [self._words_to_token(self._split_text(sent.lower())) for sent in x]
|
||||
return th.stack(split_x, dim=0)
|
||||
|
||||
def forward(self, x):
|
||||
x = self._words_to_ids(x)
|
||||
x = self.word_embd(x)
|
||||
x = F.relu(self.fc1(x))
|
||||
x = th.max(x, dim=1)[0]
|
||||
x = self.fc2(x)
|
||||
return {'text_embedding': x}
|
||||
|
||||
|
||||
class S3D(nn.Module):
|
||||
def __init__(self, dict_path, num_classes=512, gating=True, space_to_depth=True):
|
||||
super(S3D, self).__init__()
|
||||
self.num_classes = num_classes
|
||||
self.gating = gating
|
||||
self.space_to_depth = space_to_depth
|
||||
if space_to_depth:
|
||||
self.conv1 = STConv3D(
|
||||
24, 64, [2, 4, 4], stride=1, padding=(1, 2, 2), separable=False
|
||||
)
|
||||
else:
|
||||
self.conv1 = STConv3D(
|
||||
3, 64, [3, 7, 7], stride=2, padding=(1, 3, 3), separable=False
|
||||
)
|
||||
self.conv_2b = STConv3D(64, 64, [1, 1, 1], separable=False)
|
||||
self.conv_2c = STConv3D(64, 192, [3, 3, 3], padding=1, separable=True)
|
||||
self.gating = SelfGating(192)
|
||||
self.maxpool_2a = MaxPool3dTFPadding(
|
||||
kernel_size=(1, 3, 3), stride=(1, 2, 2), padding="SAME"
|
||||
)
|
||||
self.maxpool_3a = MaxPool3dTFPadding(
|
||||
kernel_size=(1, 3, 3), stride=(1, 2, 2), padding="SAME"
|
||||
)
|
||||
self.mixed_3b = InceptionBlock(192, 64, 96, 128, 16, 32, 32)
|
||||
self.mixed_3c = InceptionBlock(
|
||||
self.mixed_3b.output_dim, 128, 128, 192, 32, 96, 64
|
||||
)
|
||||
self.maxpool_4a = MaxPool3dTFPadding(
|
||||
kernel_size=(3, 3, 3), stride=(2, 2, 2), padding="SAME"
|
||||
)
|
||||
self.mixed_4b = InceptionBlock(
|
||||
self.mixed_3c.output_dim, 192, 96, 208, 16, 48, 64
|
||||
)
|
||||
self.mixed_4c = InceptionBlock(
|
||||
self.mixed_4b.output_dim, 160, 112, 224, 24, 64, 64
|
||||
)
|
||||
self.mixed_4d = InceptionBlock(
|
||||
self.mixed_4c.output_dim, 128, 128, 256, 24, 64, 64
|
||||
)
|
||||
self.mixed_4e = InceptionBlock(
|
||||
self.mixed_4d.output_dim, 112, 144, 288, 32, 64, 64
|
||||
)
|
||||
self.mixed_4f = InceptionBlock(
|
||||
self.mixed_4e.output_dim, 256, 160, 320, 32, 128, 128
|
||||
)
|
||||
self.maxpool_5a = self.maxPool3d_5a_2x2 = MaxPool3dTFPadding(
|
||||
kernel_size=(2, 2, 2), stride=(2, 2, 2), padding="SAME"
|
||||
)
|
||||
self.mixed_5b = InceptionBlock(
|
||||
self.mixed_4f.output_dim, 256, 160, 320, 32, 128, 128
|
||||
)
|
||||
self.mixed_5c = InceptionBlock(
|
||||
self.mixed_5b.output_dim, 384, 192, 384, 48, 128, 128
|
||||
)
|
||||
self.fc = nn.Linear(self.mixed_5c.output_dim, num_classes)
|
||||
self.text_module = Sentence_Embedding(num_classes,
|
||||
token_to_word_path=dict_path)
|
||||
|
||||
def _space_to_depth(self, input):
|
||||
"""3D space to depth trick for TPU optimization.
|
||||
"""
|
||||
B, C, T, H, W = input.shape
|
||||
input = input.view(B, C, T // 2, 2, H // 2, 2, W // 2, 2)
|
||||
input = input.permute(0, 3, 5, 7, 1, 2, 4, 6)
|
||||
input = input.contiguous().view(B, 8 * C, T // 2, H // 2, W // 2)
|
||||
return input
|
||||
|
||||
def forward(self, inputs):
|
||||
"""Defines the S3DG base architecture."""
|
||||
if self.space_to_depth:
|
||||
inputs = self._space_to_depth(inputs)
|
||||
net = self.conv1(inputs)
|
||||
if self.space_to_depth:
|
||||
# we need to replicate 'SAME' tensorflow padding
|
||||
net = net[:, :, 1:, 1:, 1:]
|
||||
net = self.maxpool_2a(net)
|
||||
net = self.conv_2b(net)
|
||||
net = self.conv_2c(net)
|
||||
if self.gating:
|
||||
net = self.gating(net)
|
||||
net = self.maxpool_3a(net)
|
||||
net = self.mixed_3b(net)
|
||||
net = self.mixed_3c(net)
|
||||
net = self.maxpool_4a(net)
|
||||
net = self.mixed_4b(net)
|
||||
net = self.mixed_4c(net)
|
||||
net = self.mixed_4d(net)
|
||||
net = self.mixed_4e(net)
|
||||
net = self.mixed_4f(net)
|
||||
net = self.maxpool_5a(net)
|
||||
net = self.mixed_5b(net)
|
||||
net = self.mixed_5c(net)
|
||||
net = th.mean(net, dim=[2, 3, 4])
|
||||
return {'video_embedding': self.fc(net), 'mixed_5c': net}
|
274
examples/MMPT/mmpt/processors/processor.py
Normal file
274
examples/MMPT/mmpt/processors/processor.py
Normal file
@ -0,0 +1,274 @@
|
||||
# Copyright (c) Facebook, Inc. All Rights Reserved
|
||||
|
||||
import numpy as np
|
||||
import os
|
||||
import torch
|
||||
|
||||
|
||||
class Processor(object):
|
||||
"""
|
||||
A generic processor for video (codec, feature etc.) and text.
|
||||
"""
|
||||
|
||||
def __call__(self, **kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class MetaProcessor(Processor):
|
||||
"""
|
||||
A meta processor is expected to load the metadata of a dataset:
|
||||
(e.g., video_ids, or captions).
|
||||
You must implement the `__getitem__` (meta datasets are rather diverse.).
|
||||
"""
|
||||
|
||||
def __init__(self, config):
|
||||
self.split = config.split
|
||||
|
||||
def __len__(self):
|
||||
return len(self.data)
|
||||
|
||||
def __getitem__(self, idx):
|
||||
raise NotImplementedError
|
||||
|
||||
def _get_split_path(self, config):
|
||||
splits = {
|
||||
"train": config.train_path,
|
||||
"valid": config.val_path,
|
||||
"test": config.test_path,
|
||||
}
|
||||
if config.split is not None:
|
||||
return splits[config.split]
|
||||
return config.train_path
|
||||
|
||||
|
||||
class TextProcessor(Processor):
|
||||
"""
|
||||
A generic Text processor: rename this as `withTokenizer`.
|
||||
tokenize a string of text on-the-fly.
|
||||
Warning: mostly used for end tasks.
|
||||
(on-the-fly tokenization is slow for how2.)
|
||||
TODO(huxu): move this class as a subclass.
|
||||
"""
|
||||
|
||||
def __init__(self, config):
|
||||
self.bert_name = str(config.bert_name)
|
||||
self.use_fast = config.use_fast
|
||||
from transformers import AutoTokenizer
|
||||
self.tokenizer = AutoTokenizer.from_pretrained(
|
||||
self.bert_name, use_fast=self.use_fast
|
||||
)
|
||||
|
||||
def __call__(self, text_id):
|
||||
caption = self.tokenizer(text_id, add_special_tokens=False)
|
||||
return caption["input_ids"]
|
||||
|
||||
|
||||
class VideoProcessor(Processor):
|
||||
"""
|
||||
A generic video processor: load a numpy video tokens by default.
|
||||
"""
|
||||
|
||||
def __init__(self, config):
|
||||
self.vfeat_dir = config.vfeat_dir
|
||||
|
||||
def __call__(self, video_fn):
|
||||
if isinstance(video_fn, tuple):
|
||||
video_fn = video_fn[0]
|
||||
assert isinstance(video_fn, str)
|
||||
video_fn = os.path.join(self.vfeat_dir, video_fn + ".npy")
|
||||
feat = np.load(video_fn)
|
||||
return feat
|
||||
|
||||
|
||||
class Aligner(object):
|
||||
"""
|
||||
An alignprocessor align video and text and output a dict of tensors (for a model).
|
||||
"""
|
||||
def __init__(self, config):
|
||||
"""__init__ needs to be light weight for more workers/threads."""
|
||||
self.split = config.split
|
||||
self.max_video_len = config.max_video_len
|
||||
self.max_len = config.max_len
|
||||
from transformers import AutoTokenizer
|
||||
tokenizer = AutoTokenizer.from_pretrained(
|
||||
str(config.bert_name), use_fast=config.use_fast
|
||||
)
|
||||
self.cls_token_id = tokenizer.cls_token_id
|
||||
self.sep_token_id = tokenizer.sep_token_id
|
||||
self.pad_token_id = tokenizer.pad_token_id
|
||||
self.mask_token_id = tokenizer.mask_token_id
|
||||
|
||||
def __call__(self, video_id, video_feature, text_feature):
|
||||
raise NotImplementedError
|
||||
|
||||
def _build_video_seq(self, video_feature, video_clips=None):
|
||||
"""
|
||||
`video_feature`: available video tokens.
|
||||
`video_clips`: video clip sequence to build.
|
||||
"""
|
||||
if not isinstance(video_feature, np.ndarray):
|
||||
raise ValueError(
|
||||
"unsupported type of video_feature", type(video_feature)
|
||||
)
|
||||
|
||||
if video_clips is None:
|
||||
# this is borrowed from DSAligner
|
||||
video_start = 0
|
||||
video_end = min(len(video_feature), self.max_video_len)
|
||||
# the whole sequence is a single clip.
|
||||
video_clips = {"start": [video_start], "end": [video_end]}
|
||||
|
||||
vfeats = np.zeros(
|
||||
(self.max_video_len, video_feature.shape[1]), dtype=np.float32
|
||||
)
|
||||
vmasks = torch.zeros((self.max_video_len,), dtype=torch.bool)
|
||||
video_len = 0
|
||||
for start, end in zip(video_clips["start"], video_clips["end"]):
|
||||
clip_len = min(self.max_video_len - video_len, (end - start))
|
||||
if clip_len > 0:
|
||||
vfeats[video_len: video_len + clip_len] = video_feature[
|
||||
start: start + clip_len
|
||||
]
|
||||
vmasks[video_len: video_len + clip_len] = 1
|
||||
video_len += clip_len
|
||||
vfeats = torch.from_numpy(vfeats)
|
||||
|
||||
return vfeats, vmasks
|
||||
|
||||
def _build_text_seq(self, text_feature, text_clip_indexs=None):
|
||||
"""
|
||||
`text_feature`: all available clips.
|
||||
`text_clip_indexes`: clip sequence to build.
|
||||
"""
|
||||
if text_clip_indexs is None:
|
||||
text_clip_indexs = [0]
|
||||
|
||||
full_caps = []
|
||||
if isinstance(text_feature, dict):
|
||||
for clip_idx in text_clip_indexs:
|
||||
full_caps.extend(text_feature["cap"][clip_idx])
|
||||
else:
|
||||
full_caps = text_feature
|
||||
max_text_len = self.max_len - self.max_video_len - 3
|
||||
full_caps = full_caps[:max_text_len]
|
||||
full_caps = (
|
||||
[self.cls_token_id, self.sep_token_id] + full_caps + [self.sep_token_id]
|
||||
)
|
||||
text_pad_len = self.max_len - len(full_caps) - self.max_video_len
|
||||
padded_full_caps = full_caps + [self.pad_token_id] * text_pad_len
|
||||
caps = torch.LongTensor(padded_full_caps)
|
||||
cmasks = torch.zeros((len(padded_full_caps),), dtype=torch.bool)
|
||||
cmasks[: len(full_caps)] = 1
|
||||
|
||||
return caps, cmasks
|
||||
|
||||
def batch_post_processing(self, batch, video_feature):
|
||||
return batch
|
||||
|
||||
|
||||
class MMAttentionMask2DProcessor(Processor):
|
||||
"""text generation requires 2d mask
|
||||
that is harder to generate by GPU at this stage."""
|
||||
|
||||
def __call__(self, vmask, cmask, mtype):
|
||||
if mtype == "textgen":
|
||||
return self._build_textgeneration_mask(vmask, cmask)
|
||||
elif mtype == "videogen":
|
||||
return self._build_videogeneration_mask(vmask, cmask)
|
||||
else:
|
||||
return self._build_mm_mask(vmask, cmask)
|
||||
|
||||
def _build_mm_mask(self, vmask, cmask):
|
||||
mask_1d = torch.cat([cmask[:1], vmask, cmask[1:]], dim=0)
|
||||
return mask_1d[None, :].repeat(mask_1d.size(0), 1)
|
||||
|
||||
def _build_videogeneration_mask(self, vmask, cmask):
|
||||
# cls_mask is only about text otherwise it will leak generation.
|
||||
cls_text_mask = torch.cat([
|
||||
# [CLS]
|
||||
torch.ones(
|
||||
(1,), dtype=torch.bool, device=cmask.device),
|
||||
# video tokens and [SEP] for video.
|
||||
torch.zeros(
|
||||
(vmask.size(0) + 1,), dtype=torch.bool, device=cmask.device),
|
||||
cmask[2:]
|
||||
], dim=0)
|
||||
|
||||
# concat horizontially.
|
||||
video_len = int(vmask.sum())
|
||||
video_masks = torch.cat([
|
||||
# [CLS]
|
||||
torch.ones(
|
||||
(video_len, 1), dtype=torch.bool, device=cmask.device
|
||||
),
|
||||
torch.tril(
|
||||
torch.ones(
|
||||
(video_len, video_len),
|
||||
dtype=torch.bool, device=cmask.device)),
|
||||
# video_padding
|
||||
torch.zeros(
|
||||
(video_len, vmask.size(0) - video_len),
|
||||
dtype=torch.bool, device=cmask.device
|
||||
),
|
||||
# [SEP] for video (unused).
|
||||
torch.zeros(
|
||||
(video_len, 1), dtype=torch.bool, device=cmask.device
|
||||
),
|
||||
cmask[2:].unsqueeze(0).repeat(video_len, 1)
|
||||
], dim=1)
|
||||
|
||||
text_masks = cls_text_mask[None, :].repeat(
|
||||
cmask.size(0) - 2, 1)
|
||||
video_padding_masks = cls_text_mask[None, :].repeat(
|
||||
vmask.size(0) - video_len, 1)
|
||||
|
||||
return torch.cat([
|
||||
cls_text_mask[None, :],
|
||||
video_masks,
|
||||
video_padding_masks,
|
||||
torch.cat([cmask[:1], vmask, cmask[1:]], dim=0)[None,:],
|
||||
text_masks
|
||||
], dim=0)
|
||||
|
||||
def _build_textgeneration_mask(self, vmask, cmask):
|
||||
# cls_mask is only about video otherwise it will leak generation.
|
||||
cls_video_mask = torch.cat([
|
||||
# [CLS]
|
||||
torch.ones(
|
||||
(1,), dtype=torch.bool, device=cmask.device),
|
||||
vmask,
|
||||
# [SEP]
|
||||
torch.ones((1,), dtype=torch.bool, device=cmask.device),
|
||||
torch.zeros(
|
||||
(cmask.size(0)-2,), dtype=torch.bool, device=cmask.device)
|
||||
], dim=0)
|
||||
|
||||
# concat horizontially.
|
||||
text_len = int(cmask[2:].sum())
|
||||
text_masks = torch.cat([
|
||||
# [CLS]
|
||||
torch.ones(
|
||||
(text_len, 1), dtype=torch.bool, device=cmask.device
|
||||
),
|
||||
vmask.unsqueeze(0).repeat(text_len, 1),
|
||||
# [SEP] for video.
|
||||
torch.ones(
|
||||
(text_len, 1), dtype=torch.bool, device=cmask.device
|
||||
),
|
||||
torch.tril(
|
||||
torch.ones(
|
||||
(text_len, text_len),
|
||||
dtype=torch.bool, device=cmask.device)),
|
||||
# padding.
|
||||
torch.zeros(
|
||||
(text_len, cmask.size(0) - text_len - 2),
|
||||
dtype=torch.bool, device=cmask.device
|
||||
)
|
||||
], dim=1)
|
||||
|
||||
cls_video_masks = cls_video_mask[None, :].repeat(
|
||||
vmask.size(0) + 2, 1)
|
||||
text_padding_masks = cls_video_mask[None, :].repeat(
|
||||
cmask.size(0) - text_len - 2, 1)
|
||||
return torch.cat([
|
||||
cls_video_masks, text_masks, text_padding_masks], dim=0)
|
22
examples/MMPT/mmpt/tasks/__init__.py
Normal file
22
examples/MMPT/mmpt/tasks/__init__.py
Normal file
@ -0,0 +1,22 @@
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
#
|
||||
# This source code is licensed under the MIT license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
from .task import *
|
||||
from .vlmtask import *
|
||||
from .retritask import *
|
||||
|
||||
try:
|
||||
from .fairseqmmtask import *
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
try:
|
||||
from .milncetask import *
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
try:
|
||||
from .expretritask import *
|
||||
except ImportError:
|
||||
pass
|
104
examples/MMPT/mmpt/tasks/fairseqmmtask.py
Normal file
104
examples/MMPT/mmpt/tasks/fairseqmmtask.py
Normal file
@ -0,0 +1,104 @@
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
#
|
||||
# This source code is licensed under the MIT license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
"""
|
||||
make a general fairseq task for MM pretraining.
|
||||
"""
|
||||
|
||||
import random
|
||||
|
||||
from fairseq.tasks import LegacyFairseqTask, register_task
|
||||
|
||||
from .task import Task
|
||||
from .retritask import RetriTask
|
||||
from ..datasets import FairseqMMDataset
|
||||
from .. import utils
|
||||
|
||||
|
||||
@register_task("mmtask")
|
||||
class FairseqMMTask(LegacyFairseqTask):
|
||||
@staticmethod
|
||||
def add_args(parser):
|
||||
# Add some command-line arguments for specifying where the data is
|
||||
# located and the maximum supported input length.
|
||||
parser.add_argument(
|
||||
"taskconfig",
|
||||
metavar="FILE",
|
||||
help=("taskconfig to load all configurations" "outside fairseq parser."),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def setup_task(cls, args, **kwargs):
|
||||
return FairseqMMTask(args)
|
||||
|
||||
def __init__(self, args):
|
||||
super().__init__(args)
|
||||
config = utils.load_config(args)
|
||||
self.mmtask = Task.config_task(config)
|
||||
self.mmtask.build_dataset()
|
||||
self.mmtask.build_model()
|
||||
self.mmtask.build_loss()
|
||||
|
||||
def load_dataset(self, split, **kwargs):
|
||||
split_map = {
|
||||
"train": self.mmtask.train_data,
|
||||
"valid": self.mmtask.val_data,
|
||||
"test": self.mmtask.test_data,
|
||||
}
|
||||
if split not in split_map:
|
||||
raise ValueError("unknown split type.")
|
||||
if split_map[split] is not None:
|
||||
self.datasets[split] = FairseqMMDataset(split_map[split])
|
||||
|
||||
def get_batch_iterator(
|
||||
self,
|
||||
dataset,
|
||||
max_tokens=None,
|
||||
max_sentences=None,
|
||||
max_positions=None,
|
||||
ignore_invalid_inputs=False,
|
||||
required_batch_size_multiple=1,
|
||||
seed=1,
|
||||
num_shards=1,
|
||||
shard_id=0,
|
||||
num_workers=0,
|
||||
epoch=1,
|
||||
data_buffer_size=0,
|
||||
disable_iterator_cache=False,
|
||||
skip_remainder_batch=False,
|
||||
grouped_shuffling=False,
|
||||
update_epoch_batch_itr=False,
|
||||
):
|
||||
random.seed(epoch)
|
||||
if dataset.mmdataset.split == "train" and isinstance(self.mmtask, RetriTask):
|
||||
if epoch >= self.mmtask.config.retri_epoch:
|
||||
if not hasattr(self.mmtask, "retri_dataloader"):
|
||||
self.mmtask.build_dataloader()
|
||||
self.mmtask.retrive_candidates(epoch)
|
||||
|
||||
return super().get_batch_iterator(
|
||||
dataset,
|
||||
max_tokens,
|
||||
max_sentences,
|
||||
max_positions,
|
||||
ignore_invalid_inputs,
|
||||
required_batch_size_multiple,
|
||||
seed,
|
||||
num_shards,
|
||||
shard_id,
|
||||
num_workers,
|
||||
epoch,
|
||||
data_buffer_size,
|
||||
disable_iterator_cache,
|
||||
grouped_shuffling,
|
||||
update_epoch_batch_itr,
|
||||
)
|
||||
|
||||
@property
|
||||
def source_dictionary(self):
|
||||
return None
|
||||
|
||||
@property
|
||||
def target_dictionary(self):
|
||||
return None
|
27
examples/MMPT/mmpt/tasks/milncetask.py
Normal file
27
examples/MMPT/mmpt/tasks/milncetask.py
Normal file
@ -0,0 +1,27 @@
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
#
|
||||
# This source code is licensed under the MIT license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
|
||||
import torch
|
||||
|
||||
from .task import Task
|
||||
|
||||
|
||||
class MILNCETask(Task):
|
||||
def reshape_subsample(self, sample):
|
||||
if (
|
||||
hasattr(self.config.dataset, "subsampling")
|
||||
and self.config.dataset.subsampling is not None
|
||||
and self.config.dataset.subsampling > 1
|
||||
):
|
||||
for key in sample:
|
||||
if torch.is_tensor(sample[key]):
|
||||
tensor = self.flat_subsample(sample[key])
|
||||
if key in ["caps", "cmasks"]:
|
||||
size = tensor.size()
|
||||
batch_size = size[0] * size[1]
|
||||
expanded_size = (batch_size,) + size[2:]
|
||||
tensor = tensor.view(expanded_size)
|
||||
sample[key] = tensor
|
||||
return sample
|
253
examples/MMPT/mmpt/tasks/retritask.py
Normal file
253
examples/MMPT/mmpt/tasks/retritask.py
Normal file
@ -0,0 +1,253 @@
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
#
|
||||
# This source code is licensed under the MIT license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
import os
|
||||
import torch
|
||||
import pickle
|
||||
import random
|
||||
|
||||
from tqdm import tqdm
|
||||
from torch.utils.data import DataLoader
|
||||
from torch.utils.data.distributed import DistributedSampler
|
||||
|
||||
from ..processors import (
|
||||
ShardedHow2MetaProcessor,
|
||||
ShardedVideoProcessor,
|
||||
ShardedTextProcessor,
|
||||
VariedLenAligner,
|
||||
)
|
||||
|
||||
from ..datasets import MMDataset
|
||||
from .task import Task
|
||||
from ..modules import vectorpool
|
||||
from ..evaluators.predictor import Predictor
|
||||
from ..utils import set_seed, get_local_rank, get_world_size
|
||||
|
||||
|
||||
class RetriTask(Task):
|
||||
"""abstract class for task with retrival."""
|
||||
|
||||
def reshape_subsample(self, sample):
|
||||
for key in sample:
|
||||
if torch.is_tensor(sample[key]):
|
||||
sample[key] = self.flat_subsample(sample[key])
|
||||
return sample
|
||||
|
||||
def flat_subsample(self, tensor):
|
||||
if tensor.size(0) == 1:
|
||||
tensor = tensor.squeeze(0)
|
||||
return tensor
|
||||
|
||||
def build_dataloader(self):
|
||||
"""called by `get_batch_iterator` in fairseqmmtask. """
|
||||
# TODO: hard-code dataloader for retri for now and configurable in .yaml.
|
||||
# reuse the `train.lst`.
|
||||
self.config.dataset.split = "train"
|
||||
meta_processor = ShardedHow2MetaProcessor(self.config.dataset)
|
||||
video_processor = ShardedVideoProcessor(self.config.dataset)
|
||||
text_processor = ShardedTextProcessor(self.config.dataset)
|
||||
|
||||
aligner = VariedLenAligner(self.config.dataset)
|
||||
aligner.subsampling = self.config.dataset.clip_per_video
|
||||
|
||||
self.retri_data = MMDataset(
|
||||
meta_processor, video_processor, text_processor, aligner
|
||||
)
|
||||
|
||||
retri_sampler = DistributedSampler(self.retri_data)
|
||||
infer_scale = 16
|
||||
batch_size = self.config.dataset.num_video_per_batch \
|
||||
* infer_scale
|
||||
|
||||
self.retri_dataloader = DataLoader(
|
||||
self.retri_data,
|
||||
collate_fn=self.retri_data.collater,
|
||||
batch_size=batch_size,
|
||||
shuffle=False,
|
||||
sampler=retri_sampler,
|
||||
num_workers=self.config.fairseq.dataset.num_workers
|
||||
)
|
||||
return self.retri_dataloader
|
||||
|
||||
def retrive_candidates(self, epoch, dataloader=None):
|
||||
if get_local_rank() == 0:
|
||||
print("running retrieval model.")
|
||||
out_dir = os.path.join(
|
||||
self.config.fairseq.checkpoint.save_dir, "retri")
|
||||
os.makedirs(out_dir, exist_ok=True)
|
||||
|
||||
if not os.path.isfile(
|
||||
os.path.join(
|
||||
out_dir, "batched_e" + str(epoch) + "_videos0.pkl")
|
||||
):
|
||||
if dataloader is None:
|
||||
dataloader = self.retri_dataloader
|
||||
|
||||
self.model.eval()
|
||||
self.model.is_train = False
|
||||
|
||||
assert self.retri_data.meta_processor.data == \
|
||||
self.train_data.meta_processor.data # video_ids not mutated.
|
||||
|
||||
self._retri_predict(epoch, dataloader)
|
||||
|
||||
self.model.train()
|
||||
self.model.is_train = True
|
||||
|
||||
torch.distributed.barrier()
|
||||
output = self._retri_sync(epoch, out_dir)
|
||||
torch.distributed.barrier()
|
||||
self.train_data.meta_processor.set_candidates(output)
|
||||
return output
|
||||
|
||||
|
||||
class VideoRetriTask(RetriTask):
|
||||
"""RetriTask on video level."""
|
||||
|
||||
def reshape_subsample(self, sample):
|
||||
if (
|
||||
hasattr(self.config.dataset, "clip_per_video")
|
||||
and self.config.dataset.clip_per_video is not None
|
||||
and self.config.dataset.clip_per_video > 1
|
||||
):
|
||||
for key in sample:
|
||||
if torch.is_tensor(sample[key]):
|
||||
sample[key] = self.flat_subsample(sample[key])
|
||||
return sample
|
||||
|
||||
def flat_subsample(self, tensor):
|
||||
if tensor.size(0) == 1:
|
||||
tensor = tensor.squeeze(0)
|
||||
return Task.flat_subsample(self, tensor)
|
||||
|
||||
def _retri_predict(self, epoch, dataloader):
|
||||
set_seed(epoch)
|
||||
# save for retrival.
|
||||
predictor = VideoPredictor(self.config)
|
||||
predictor.predict_loop(
|
||||
self.model, dataloader)
|
||||
set_seed(epoch) # get the same text clips.
|
||||
# retrival.
|
||||
retri_predictor = VideoRetriPredictor(
|
||||
self.config)
|
||||
retri_predictor.predict_loop(
|
||||
self.model, predictor.vecpool.retriver, epoch)
|
||||
del predictor
|
||||
del retri_predictor
|
||||
|
||||
def _retri_sync(self, epoch, out_dir):
|
||||
# gpu do the same merge.
|
||||
batched_videos = []
|
||||
for local_rank in range(get_world_size()):
|
||||
fn = os.path.join(
|
||||
out_dir,
|
||||
"batched_e" + str(epoch) + "_videos" + str(local_rank) + ".pkl")
|
||||
with open(fn, "rb") as fr:
|
||||
batched_videos.extend(pickle.load(fr))
|
||||
print(
|
||||
"[INFO] batched_videos",
|
||||
len(batched_videos), len(batched_videos[0]))
|
||||
return batched_videos
|
||||
|
||||
|
||||
class VideoPredictor(Predictor):
|
||||
def __init__(self, config):
|
||||
vectorpool_cls = getattr(vectorpool, config.vectorpool_cls)
|
||||
self.vecpool = vectorpool_cls(config)
|
||||
|
||||
def predict_loop(
|
||||
self,
|
||||
model,
|
||||
dataloader,
|
||||
early_stop=-1,
|
||||
):
|
||||
with torch.no_grad():
|
||||
if get_local_rank() == 0:
|
||||
dataloader = tqdm(dataloader)
|
||||
for batch_idx, batch in enumerate(dataloader):
|
||||
if batch_idx == early_stop:
|
||||
break
|
||||
self(batch, model)
|
||||
return self.finalize()
|
||||
|
||||
def __call__(self, sample, model, **kwargs):
|
||||
param = next(model.parameters())
|
||||
dtype = param.dtype
|
||||
device = param.device
|
||||
subsample = sample["vfeats"].size(1)
|
||||
sample = self.to_ctx(sample, device, dtype)
|
||||
for key in sample:
|
||||
if torch.is_tensor(sample[key]):
|
||||
size = sample[key].size()
|
||||
if len(size) >= 2:
|
||||
batch_size = size[0] * size[1]
|
||||
expanded_size = (
|
||||
(batch_size,) + size[2:] if len(size) > 2
|
||||
else (batch_size,)
|
||||
)
|
||||
sample[key] = sample[key].view(expanded_size)
|
||||
|
||||
outputs = model(**sample)
|
||||
sample.update(outputs)
|
||||
self.vecpool(sample, subsample)
|
||||
|
||||
def finalize(self):
|
||||
print("[INFO]", self.vecpool)
|
||||
if not self.vecpool.retriver.db.is_trained:
|
||||
self.vecpool.retriver.finalize_training()
|
||||
return self.vecpool.retriver
|
||||
|
||||
|
||||
class VideoRetriPredictor(Predictor):
|
||||
"""
|
||||
Online Retrieval Predictor for Clips (used by RetriTask).
|
||||
TODO: merge this with VisPredictor?
|
||||
"""
|
||||
|
||||
def __init__(self, config):
|
||||
self.pred_dir = os.path.join(
|
||||
config.fairseq.checkpoint.save_dir,
|
||||
"retri")
|
||||
self.num_cands = config.num_cands
|
||||
self.num_video_per_batch = config.dataset.num_video_per_batch
|
||||
|
||||
def predict_loop(
|
||||
self,
|
||||
model,
|
||||
retriver,
|
||||
epoch,
|
||||
early_stop=-1
|
||||
):
|
||||
# a fake loop that only try to recover video vector
|
||||
# from video_id.
|
||||
batched_videos = []
|
||||
# obtain available video_ids.
|
||||
video_ids = list(retriver.videoid_to_vectoridx.keys())
|
||||
|
||||
dataloader = random.sample(
|
||||
video_ids,
|
||||
len(video_ids) // self.num_video_per_batch
|
||||
)
|
||||
|
||||
if get_local_rank() == 0:
|
||||
dataloader = tqdm(dataloader)
|
||||
for batch_idx, batch in enumerate(dataloader):
|
||||
# batch is one video id.
|
||||
if batch_idx == early_stop:
|
||||
break
|
||||
video_ids = retriver.search_by_video_ids(
|
||||
[batch], self.num_cands)[0]
|
||||
if len(video_ids) > self.num_video_per_batch:
|
||||
# we moved the center to make cluster robust.
|
||||
video_ids = random.sample(video_ids, self.num_video_per_batch)
|
||||
batched_videos.append(video_ids)
|
||||
return self.finalize(batched_videos, epoch)
|
||||
|
||||
def finalize(self, batched_videos, epoch):
|
||||
fn = os.path.join(
|
||||
self.pred_dir,
|
||||
"batched_e" + str(epoch) + "_videos" + str(get_local_rank()) + ".pkl")
|
||||
with open(fn, "wb") as fw:
|
||||
pickle.dump(batched_videos, fw, pickle.HIGHEST_PROTOCOL)
|
||||
return batched_videos
|
184
examples/MMPT/mmpt/tasks/task.py
Normal file
184
examples/MMPT/mmpt/tasks/task.py
Normal file
@ -0,0 +1,184 @@
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
#
|
||||
# This source code is licensed under the MIT license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
import torch
|
||||
|
||||
from .. import tasks
|
||||
from .. import models
|
||||
from .. import losses
|
||||
from ..datasets import MMDataset
|
||||
from .. import processors
|
||||
|
||||
|
||||
class Task(object):
|
||||
"""
|
||||
A task refers to one generic training task (e.g., training one model).
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def config_task(cls, config):
|
||||
"""
|
||||
determine whether to load a hard-coded task or config from a generic one.
|
||||
via if a task string is available in config.
|
||||
"""
|
||||
if config.task is not None:
|
||||
# TODO (huxu): expand the search scope.
|
||||
task_cls = getattr(tasks, config.task)
|
||||
return task_cls(config)
|
||||
else:
|
||||
return Task(config)
|
||||
|
||||
def __init__(self, config):
|
||||
self.config = config
|
||||
self.train_data = None
|
||||
self.val_data = None
|
||||
self.test_data = None
|
||||
|
||||
self.model = None
|
||||
self.loss_fn = None
|
||||
self.eval_fn = None
|
||||
|
||||
def build_dataset(self):
|
||||
"""TODO (huxu): move processor breakdown to MMDataset."""
|
||||
"""fill-in `self.train_data`, `self.val_data` and `self.test_data`."""
|
||||
|
||||
meta_processor_cls = getattr(
|
||||
processors, self.config.dataset.meta_processor)
|
||||
video_processor_cls = getattr(
|
||||
processors, self.config.dataset.video_processor)
|
||||
text_processor_cls = getattr(
|
||||
processors, self.config.dataset.text_processor)
|
||||
aligner_cls = getattr(
|
||||
processors, self.config.dataset.aligner)
|
||||
|
||||
if self.config.dataset.train_path is not None:
|
||||
self.config.dataset.split = "train"
|
||||
# may be used by meta processor.
|
||||
# meta_processor controls different dataset.
|
||||
meta_processor = meta_processor_cls(self.config.dataset)
|
||||
video_processor = video_processor_cls(self.config.dataset)
|
||||
text_processor = text_processor_cls(self.config.dataset)
|
||||
aligner = aligner_cls(self.config.dataset)
|
||||
self.train_data = MMDataset(
|
||||
meta_processor, video_processor, text_processor, aligner
|
||||
)
|
||||
print("train_len", len(self.train_data))
|
||||
output = self.train_data[0]
|
||||
self.train_data.print_example(output)
|
||||
if self.config.dataset.val_path is not None:
|
||||
self.config.dataset.split = "valid"
|
||||
# may be used by meta processor.
|
||||
meta_processor = meta_processor_cls(self.config.dataset)
|
||||
video_processor = video_processor_cls(self.config.dataset)
|
||||
text_processor = text_processor_cls(self.config.dataset)
|
||||
aligner = aligner_cls(self.config.dataset)
|
||||
self.val_data = MMDataset(
|
||||
meta_processor, video_processor, text_processor, aligner
|
||||
)
|
||||
print("val_len", len(self.val_data))
|
||||
output = self.val_data[0]
|
||||
self.val_data.print_example(output)
|
||||
|
||||
if self.config.dataset.split == "test":
|
||||
# the following is run via lauching fairseq-validate.
|
||||
meta_processor = meta_processor_cls(self.config.dataset)
|
||||
video_processor = video_processor_cls(self.config.dataset)
|
||||
text_processor = text_processor_cls(self.config.dataset)
|
||||
|
||||
self.test_data = MMDataset(
|
||||
meta_processor, video_processor, text_processor, aligner
|
||||
)
|
||||
print("test_len", len(self.test_data))
|
||||
output = self.test_data[0]
|
||||
self.test_data.print_example(output)
|
||||
|
||||
def build_model(self, checkpoint=None):
|
||||
if self.model is None:
|
||||
model_cls = getattr(models, self.config.model.model_cls)
|
||||
self.model = model_cls(self.config)
|
||||
if checkpoint is not None:
|
||||
self.load_checkpoint(checkpoint)
|
||||
return self.model
|
||||
|
||||
def load_checkpoint(self, checkpoint):
|
||||
if self.model is None:
|
||||
raise ValueError("model is not initialized.")
|
||||
state_dict = torch.load(checkpoint)
|
||||
state_dict = self._trim_state_dict(state_dict)
|
||||
self.model.load_state_dict(state_dict, strict=False)
|
||||
# if it's a fp16 model, turn it back.
|
||||
if next(self.model.parameters()).dtype == torch.float16:
|
||||
self.model = self.model.float()
|
||||
return self.model
|
||||
|
||||
def _trim_state_dict(self, state_dict):
|
||||
from collections import OrderedDict
|
||||
|
||||
if "state_dict" in state_dict:
|
||||
state_dict = state_dict["state_dict"]
|
||||
if "model" in state_dict: # fairseq checkpoint format.
|
||||
state_dict = state_dict["model"]
|
||||
ret_state_dict = OrderedDict()
|
||||
for (
|
||||
key,
|
||||
value,
|
||||
) in state_dict.items():
|
||||
# remove fairseq wrapper since this is a task.
|
||||
if key.startswith("mmmodel"):
|
||||
key = key[len("mmmodel."):]
|
||||
ret_state_dict[key] = value
|
||||
return ret_state_dict
|
||||
|
||||
def build_loss(self):
|
||||
if self.loss_fn is None and self.config.loss is not None:
|
||||
loss_cls = getattr(losses, self.config.loss.loss_cls)
|
||||
self.loss_fn = loss_cls()
|
||||
return self.loss_fn
|
||||
|
||||
def flat_subsample(self, tensor):
|
||||
size = tensor.size()
|
||||
if len(size) >= 2:
|
||||
batch_size = size[0] * size[1]
|
||||
expanded_size = (
|
||||
(batch_size,) + size[2:] if len(size) > 2
|
||||
else (batch_size,)
|
||||
)
|
||||
tensor = tensor.view(expanded_size)
|
||||
return tensor
|
||||
|
||||
def reshape_subsample(self, sample):
|
||||
if (
|
||||
hasattr(self.config.dataset, "subsampling")
|
||||
and self.config.dataset.subsampling is not None
|
||||
and self.config.dataset.subsampling > 1
|
||||
):
|
||||
for key in sample:
|
||||
if torch.is_tensor(sample[key]):
|
||||
sample[key] = self.flat_subsample(sample[key])
|
||||
return sample
|
||||
|
||||
def __call__(self, model, sample):
|
||||
loss = None
|
||||
loss_scalar = float("inf")
|
||||
|
||||
sample = self.reshape_subsample(sample)
|
||||
outputs = self.model(**sample)
|
||||
sample.update(outputs)
|
||||
if self.loss_fn is not None:
|
||||
loss = self.loss_fn(**sample)
|
||||
loss_scalar = loss.item()
|
||||
|
||||
batch_size = sample["caps"].size(0)
|
||||
sample_size = 1
|
||||
return {
|
||||
"loss": loss,
|
||||
"loss_scalar": loss_scalar,
|
||||
"max_len": self.config.dataset.max_len,
|
||||
"batch_size": batch_size,
|
||||
"sample_size": sample_size,
|
||||
}
|
||||
|
||||
def build_dataloader(self):
|
||||
"""only used for trainer that lacks building loaders."""
|
||||
raise NotImplementedError
|
27
examples/MMPT/mmpt/tasks/vlmtask.py
Normal file
27
examples/MMPT/mmpt/tasks/vlmtask.py
Normal file
@ -0,0 +1,27 @@
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
#
|
||||
# This source code is licensed under the MIT license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
import torch
|
||||
|
||||
from .task import Task
|
||||
|
||||
|
||||
class VLMTask(Task):
|
||||
"""A VLM task for reproducibility.
|
||||
the collator split subsamples into two sub-batches.
|
||||
This has should have no logic changes.
|
||||
but changed the randomness in frame masking.
|
||||
"""
|
||||
|
||||
def flat_subsample(self, tensor):
|
||||
size = tensor.size()
|
||||
if len(size) >= 2:
|
||||
batch_size = size[0] * (size[1] // 2)
|
||||
expanded_size = (
|
||||
(batch_size, 2) + size[2:] if len(size) > 2
|
||||
else (batch_size, 2)
|
||||
)
|
||||
tensor = tensor.view(expanded_size)
|
||||
tensor = torch.cat([tensor[:, 0], tensor[:, 1]], dim=0)
|
||||
return tensor
|
68
examples/MMPT/mmpt/utils/__init__.py
Normal file
68
examples/MMPT/mmpt/utils/__init__.py
Normal file
@ -0,0 +1,68 @@
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
#
|
||||
# This source code is licensed under the MIT license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
import random
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from .shardedtensor import *
|
||||
from .load_config import *
|
||||
|
||||
|
||||
def set_seed(seed=43211):
|
||||
random.seed(seed)
|
||||
np.random.seed(seed)
|
||||
torch.manual_seed(seed)
|
||||
torch.cuda.manual_seed_all(seed)
|
||||
if torch.backends.cudnn.enabled:
|
||||
torch.backends.cudnn.benchmark = False
|
||||
torch.backends.cudnn.deterministic = True
|
||||
|
||||
|
||||
def get_world_size():
|
||||
if torch.distributed.is_initialized():
|
||||
world_size = torch.distributed.get_world_size()
|
||||
else:
|
||||
world_size = 1
|
||||
return world_size
|
||||
|
||||
|
||||
def get_local_rank():
|
||||
return torch.distributed.get_rank() \
|
||||
if torch.distributed.is_initialized() else 0
|
||||
|
||||
|
||||
def print_on_rank0(func):
|
||||
local_rank = get_local_rank()
|
||||
if local_rank == 0:
|
||||
print("[INFO]", func)
|
||||
|
||||
|
||||
class RetriMeter(object):
|
||||
"""
|
||||
Statistics on whether retrieval yields a better pair.
|
||||
"""
|
||||
def __init__(self, freq=1024):
|
||||
self.freq = freq
|
||||
self.total = 0
|
||||
self.replace = 0
|
||||
self.updates = 0
|
||||
|
||||
def __call__(self, data):
|
||||
if isinstance(data, np.ndarray):
|
||||
self.replace += data.shape[0] - int((data[:, 0] == -1).sum())
|
||||
self.total += data.shape[0]
|
||||
elif torch.is_tensor(data):
|
||||
self.replace += int(data.sum())
|
||||
self.total += data.size(0)
|
||||
else:
|
||||
raise ValueError("unsupported RetriMeter data type.", type(data))
|
||||
|
||||
self.updates += 1
|
||||
if get_local_rank() == 0 and self.updates % self.freq == 0:
|
||||
print("[INFO]", self)
|
||||
|
||||
def __repr__(self):
|
||||
return "RetriMeter (" + str(self.replace / self.total) \
|
||||
+ "/" + str(self.replace) + "/" + str(self.total) + ")"
|
81
examples/MMPT/mmpt/utils/load_config.py
Normal file
81
examples/MMPT/mmpt/utils/load_config.py
Normal file
@ -0,0 +1,81 @@
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
#
|
||||
# This source code is licensed under the MIT license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
import os
|
||||
import omegaconf
|
||||
from omegaconf import OmegaConf
|
||||
|
||||
|
||||
def load_config(args=None, config_file=None, overwrite_fairseq=False):
|
||||
"""TODO (huxu): move fairseq overwrite to another function."""
|
||||
if args is not None:
|
||||
config_file = args.taskconfig
|
||||
config = recursive_config(config_file)
|
||||
|
||||
if config.dataset.subsampling is not None:
|
||||
batch_size = config.fairseq.dataset.batch_size // config.dataset.subsampling
|
||||
print(
|
||||
"adjusting batch_size to {} due to subsampling {}.".format(
|
||||
batch_size, config.dataset.subsampling
|
||||
)
|
||||
)
|
||||
config.fairseq.dataset.batch_size = batch_size
|
||||
|
||||
is_test = config.dataset.split is not None and config.dataset.split == "test"
|
||||
if not is_test:
|
||||
if (
|
||||
config.fairseq.checkpoint is None
|
||||
or config.fairseq.checkpoint.save_dir is None
|
||||
):
|
||||
raise ValueError("fairseq save_dir or save_path must be specified.")
|
||||
|
||||
save_dir = config.fairseq.checkpoint.save_dir
|
||||
os.makedirs(save_dir, exist_ok=True)
|
||||
if config.fairseq.common.tensorboard_logdir is not None:
|
||||
tb_run_dir = suffix_rundir(
|
||||
save_dir, config.fairseq.common.tensorboard_logdir
|
||||
)
|
||||
config.fairseq.common.tensorboard_logdir = tb_run_dir
|
||||
print(
|
||||
"update tensorboard_logdir as", config.fairseq.common.tensorboard_logdir
|
||||
)
|
||||
os.makedirs(save_dir, exist_ok=True)
|
||||
OmegaConf.save(config=config, f=os.path.join(save_dir, "config.yaml"))
|
||||
|
||||
if overwrite_fairseq and config.fairseq is not None and args is not None:
|
||||
# flatten fields.
|
||||
for group in config.fairseq:
|
||||
for field in config.fairseq[group]:
|
||||
print("overwrite args." + field, "as", config.fairseq[group][field])
|
||||
setattr(args, field, config.fairseq[group][field])
|
||||
return config
|
||||
|
||||
|
||||
def recursive_config(config_path):
|
||||
"""allows for stacking of configs in any depth."""
|
||||
config = OmegaConf.load(config_path)
|
||||
if config.includes is not None:
|
||||
includes = config.includes
|
||||
config.pop("includes")
|
||||
base_config = recursive_config(includes)
|
||||
config = OmegaConf.merge(base_config, config)
|
||||
return config
|
||||
|
||||
|
||||
def suffix_rundir(save_dir, run_dir):
|
||||
max_id = -1
|
||||
for search_dir in os.listdir(save_dir):
|
||||
if search_dir.startswith(run_dir):
|
||||
splits = search_dir.split("_")
|
||||
cur_id = int(splits[1]) if len(splits) > 1 else 0
|
||||
max_id = max(max_id, cur_id)
|
||||
return os.path.join(save_dir, run_dir + "_" + str(max_id + 1))
|
||||
|
||||
|
||||
def overwrite_dir(config, replace, basedir):
|
||||
for key in config:
|
||||
if isinstance(config[key], str) and config[key].startswith(basedir):
|
||||
config[key] = config[key].replace(basedir, replace)
|
||||
if isinstance(config[key], omegaconf.dictconfig.DictConfig):
|
||||
overwrite_dir(config[key], replace, basedir)
|
46
examples/MMPT/mmpt/utils/shardedtensor.py
Normal file
46
examples/MMPT/mmpt/utils/shardedtensor.py
Normal file
@ -0,0 +1,46 @@
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
#
|
||||
# This source code is licensed under the MIT license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
import os
|
||||
import pickle
|
||||
import numpy as np
|
||||
|
||||
|
||||
class ShardedTensor(object):
|
||||
def __init__(self, data, starts):
|
||||
self.data = data
|
||||
self.starts = starts
|
||||
assert self.starts[0] == 0
|
||||
assert self.starts[-1] == len(self.data)
|
||||
assert (self.starts[1:] >= self.starts[:-1]).all()
|
||||
assert (self.starts > -1).all()
|
||||
|
||||
@staticmethod
|
||||
def from_list(xs):
|
||||
starts = np.full((len(xs) + 1,), -1, dtype=np.long)
|
||||
data = np.concatenate(xs, axis=0)
|
||||
starts[0] = 0
|
||||
for i, x in enumerate(xs):
|
||||
starts[i + 1] = starts[i] + x.shape[0]
|
||||
assert (starts > -1).all()
|
||||
return ShardedTensor(data, starts)
|
||||
|
||||
def __getitem__(self, i):
|
||||
return self.data[self.starts[i] : self.starts[i + 1]]
|
||||
|
||||
def __len__(self):
|
||||
return len(self.starts) - 1
|
||||
|
||||
def lengths(self):
|
||||
return self.starts[1:] - self.starts[:-1]
|
||||
|
||||
def save(self, path):
|
||||
np.save(path + "_starts", self.starts)
|
||||
np.save(path + "_data", self.data)
|
||||
|
||||
@staticmethod
|
||||
def load(path, mmap_mode=None):
|
||||
starts = np.load(path + "_starts.npy", mmap_mode)
|
||||
data = np.load(path + "_data.npy", mmap_mode)
|
||||
return ShardedTensor(data, starts)
|
117
examples/MMPT/mmpt_cli/localjob.py
Normal file
117
examples/MMPT/mmpt_cli/localjob.py
Normal file
@ -0,0 +1,117 @@
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
#
|
||||
# This source code is licensed under the MIT license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
import os
|
||||
|
||||
from mmpt.utils import recursive_config
|
||||
|
||||
|
||||
class BaseJob(object):
|
||||
def __init__(self, yaml_file, dryrun=False):
|
||||
self.yaml_file = yaml_file
|
||||
self.config = recursive_config(yaml_file)
|
||||
self.dryrun = dryrun
|
||||
|
||||
def submit(self, **kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
def _normalize_cmd(self, cmd_list):
|
||||
cmd_list = list(cmd_list)
|
||||
yaml_index = cmd_list.index("[yaml]")
|
||||
cmd_list[yaml_index] = self.yaml_file
|
||||
return cmd_list
|
||||
|
||||
|
||||
class LocalJob(BaseJob):
|
||||
|
||||
CMD_CONFIG = {
|
||||
"local_single": [
|
||||
"fairseq-train", "[yaml]", "--user-dir", "mmpt",
|
||||
"--task", "mmtask", "--arch", "mmarch",
|
||||
"--criterion", "mmloss",
|
||||
],
|
||||
"local_small": [
|
||||
"fairseq-train", "[yaml]", "--user-dir", "mmpt",
|
||||
"--task", "mmtask", "--arch", "mmarch",
|
||||
"--criterion", "mmloss",
|
||||
"--distributed-world-size", "2"
|
||||
],
|
||||
"local_big": [
|
||||
"fairseq-train", "[yaml]", "--user-dir", "mmpt",
|
||||
"--task", "mmtask", "--arch", "mmarch",
|
||||
"--criterion", "mmloss",
|
||||
"--distributed-world-size", "8"
|
||||
],
|
||||
"local_predict": ["python", "mmpt_cli/predict.py", "[yaml]"],
|
||||
}
|
||||
|
||||
def __init__(self, yaml_file, job_type=None, dryrun=False):
|
||||
super().__init__(yaml_file, dryrun)
|
||||
if job_type is None:
|
||||
self.job_type = "local_single"
|
||||
if self.config.task_type is not None:
|
||||
self.job_type = self.config.task_type
|
||||
else:
|
||||
self.job_type = job_type
|
||||
if self.job_type in ["local_single", "local_small"]:
|
||||
if self.config.fairseq.dataset.batch_size > 32:
|
||||
print("decreasing batch_size to 32 for local testing?")
|
||||
|
||||
def submit(self):
|
||||
cmd_list = self._normalize_cmd(LocalJob.CMD_CONFIG[self.job_type])
|
||||
if "predict" not in self.job_type:
|
||||
# append fairseq args.
|
||||
from mmpt.utils import load_config
|
||||
|
||||
config = load_config(config_file=self.yaml_file)
|
||||
for field in config.fairseq:
|
||||
for key in config.fairseq[field]:
|
||||
if key in ["fp16", "reset_optimizer", "reset_dataloader", "reset_meters"]: # a list of binary flag.
|
||||
param = ["--" + key.replace("_", "-")]
|
||||
else:
|
||||
if key == "lr":
|
||||
value = str(config.fairseq[field][key][0])
|
||||
elif key == "adam_betas":
|
||||
value = "'"+str(config.fairseq[field][key])+"'"
|
||||
else:
|
||||
value = str(config.fairseq[field][key])
|
||||
param = [
|
||||
"--" + key.replace("_", "-"),
|
||||
value
|
||||
]
|
||||
cmd_list.extend(param)
|
||||
|
||||
print("launching", " ".join(cmd_list))
|
||||
if not self.dryrun:
|
||||
os.system(" ".join(cmd_list))
|
||||
return JobStatus("12345678")
|
||||
|
||||
|
||||
class JobStatus(object):
|
||||
def __init__(self, job_id):
|
||||
self.job_id = job_id
|
||||
|
||||
def __repr__(self):
|
||||
return self.job_id
|
||||
|
||||
def __str__(self):
|
||||
return self.job_id
|
||||
|
||||
def done(self):
|
||||
return False
|
||||
|
||||
def running(self):
|
||||
return False
|
||||
|
||||
def result(self):
|
||||
if self.done():
|
||||
return "{} is done.".format(self.job_id)
|
||||
else:
|
||||
return "{} is running.".format(self.job_id)
|
||||
|
||||
def stderr(self):
|
||||
return self.result()
|
||||
|
||||
def stdout(self):
|
||||
return self.result()
|
113
examples/MMPT/mmpt_cli/predict.py
Normal file
113
examples/MMPT/mmpt_cli/predict.py
Normal file
@ -0,0 +1,113 @@
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
#
|
||||
# This source code is licensed under the MIT license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
import os
|
||||
import glob
|
||||
import argparse
|
||||
import pprint
|
||||
import omegaconf
|
||||
|
||||
from omegaconf import OmegaConf
|
||||
from torch.utils.data import DataLoader
|
||||
|
||||
from mmpt.utils import load_config, set_seed
|
||||
from mmpt.evaluators import Evaluator
|
||||
from mmpt.evaluators import predictor as predictor_path
|
||||
from mmpt.tasks import Task
|
||||
from mmpt import processors
|
||||
from mmpt.datasets import MMDataset
|
||||
|
||||
|
||||
def get_dataloader(config):
|
||||
meta_processor_cls = getattr(processors, config.dataset.meta_processor)
|
||||
video_processor_cls = getattr(processors, config.dataset.video_processor)
|
||||
text_processor_cls = getattr(processors, config.dataset.text_processor)
|
||||
aligner_cls = getattr(processors, config.dataset.aligner)
|
||||
|
||||
meta_processor = meta_processor_cls(config.dataset)
|
||||
video_processor = video_processor_cls(config.dataset)
|
||||
text_processor = text_processor_cls(config.dataset)
|
||||
aligner = aligner_cls(config.dataset)
|
||||
|
||||
test_data = MMDataset(
|
||||
meta_processor,
|
||||
video_processor,
|
||||
text_processor,
|
||||
aligner,
|
||||
)
|
||||
print("test_len", len(test_data))
|
||||
output = test_data[0]
|
||||
test_data.print_example(output)
|
||||
|
||||
test_dataloader = DataLoader(
|
||||
test_data,
|
||||
batch_size=config.fairseq.dataset.batch_size,
|
||||
shuffle=False,
|
||||
num_workers=6,
|
||||
collate_fn=test_data.collater,
|
||||
)
|
||||
return test_dataloader
|
||||
|
||||
|
||||
def main(args):
|
||||
config = load_config(args)
|
||||
|
||||
if isinstance(config, omegaconf.dictconfig.DictConfig):
|
||||
print(OmegaConf.to_yaml(config))
|
||||
else:
|
||||
pp = pprint.PrettyPrinter(indent=4)
|
||||
pp.print(config)
|
||||
|
||||
mmtask = Task.config_task(config)
|
||||
mmtask.build_model()
|
||||
|
||||
test_dataloader = get_dataloader(config)
|
||||
checkpoint_search_path = os.path.dirname(config.eval.save_path)
|
||||
results = []
|
||||
|
||||
prefix = os.path.basename(args.taskconfig)
|
||||
if prefix.startswith("test"):
|
||||
# loop all checkpoint for datasets without validation set.
|
||||
if "best" not in config.fairseq.common_eval.path:
|
||||
print("eval each epoch.")
|
||||
for checkpoint in glob.glob(checkpoint_search_path + "/checkpoint*"):
|
||||
model = mmtask.load_checkpoint(checkpoint)
|
||||
ckpt = os.path.basename(checkpoint)
|
||||
evaluator = Evaluator(config)
|
||||
output = evaluator.evaluate(
|
||||
model, test_dataloader, ckpt + "_merged")
|
||||
results.append((checkpoint, output))
|
||||
# use the one specified by the config lastly.
|
||||
model = mmtask.load_checkpoint(config.fairseq.common_eval.path)
|
||||
evaluator = Evaluator(config)
|
||||
output = evaluator.evaluate(model, test_dataloader)
|
||||
results.append((config.fairseq.common_eval.path, output))
|
||||
|
||||
best_result = None
|
||||
best_metric = 0.
|
||||
for checkpoint, result in results:
|
||||
print(checkpoint)
|
||||
evaluator.metric.print_computed_metrics(result)
|
||||
best_score = evaluator.metric.best_metric(result)
|
||||
if best_score > best_metric:
|
||||
best_result = (checkpoint, result)
|
||||
best_metric = best_score
|
||||
print("best results:")
|
||||
print(best_result[0])
|
||||
evaluator.metric.print_computed_metrics(best_result[1])
|
||||
|
||||
elif prefix.startswith("vis"):
|
||||
model = mmtask.load_checkpoint(config.fairseq.common_eval.path)
|
||||
predictor_cls = getattr(predictor_path, config.predictor)
|
||||
predictor = predictor_cls(config)
|
||||
predictor.predict_loop(model, test_dataloader, mmtask, None)
|
||||
else:
|
||||
raise ValueError("unknown prefix of the config file", args.taskconfig)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("taskconfig", type=str)
|
||||
args = parser.parse_args()
|
||||
main(args)
|
29
examples/MMPT/pretraining.md
Normal file
29
examples/MMPT/pretraining.md
Normal file
@ -0,0 +1,29 @@
|
||||
# Pretraining
|
||||
|
||||
(If you are new to the ideas of `mmpt.processors`, see [README](README.md) first.)
|
||||
We mostly use [howto100M](https://github.com/antoine77340/howto100m) dataset for pretraining (other datasets are coming). So you are less likely to write a new `MetaProcessor`, `VideoProcessor` or `TextProcessor` but only working on a new `Aligner`, a new model and loss.
|
||||
|
||||
### Data Sharding
|
||||
Pretraining on Howto100M is heavy on IO since we have millions of videos or captions on the hard disk that cannot be fit into the memory.
|
||||
It is desirable to have an optimized preprocessing step before the actual dataloading.
|
||||
|
||||
We support data sharding to pack multiple videos into a shards of training data for both videos and captions. (see [dataset](DATASET.md) for preprocessing).
|
||||
These shards will be mapped into memory to reduce the frequency of IO access on millions of files. See (processors starting with `Sharded*`).
|
||||
This will be the default config for a how2 dataset `projects/task/how2.yaml`.
|
||||
|
||||
Great thanks to Dmytro Okhonko for sharing the code from MARGE project.
|
||||
|
||||
### Training
|
||||
Pretraining on Howto100m is expected on one or multiple nodes, where each node has 8 GPUS with 32 GB mem.
|
||||
launching a pretraing on MFM+MLM can be done, via:
|
||||
```python locallaunch.py projects/mfmmlm/how2.yaml```
|
||||
|
||||
### Pre-training with a Retrieval Model (VideoCLIP)
|
||||
This projects now support alternatively run a retrieval model and pre-training.
|
||||
We implement a basic retrieval model that is built on the hidden states of a video and faiss.
|
||||
|
||||
You may need to install faiss via `conda install faiss-cpu -c pytorch`.
|
||||
|
||||
Right now, the hidden states of a video is computed as the average of 8 clips of their pooled visual/text hidden states.
|
||||
See `mmpt/tasks/retritask.py` for more details.
|
||||
The `.yaml` config for running pre-training with a retrieval model can be found at `projects/retri/videoretri.yaml`.
|
59
examples/MMPT/projects/mfmmlm.yaml
Normal file
59
examples/MMPT/projects/mfmmlm.yaml
Normal file
@ -0,0 +1,59 @@
|
||||
project_dir: mfmmlm
|
||||
run_task:
|
||||
- how2.yaml
|
||||
- [vtt.yaml, vttcap.yaml, vttqa.yaml, youcook.yaml, youcookcap.yaml, crosstask.yaml, coin.yaml]
|
||||
base_dir: task
|
||||
task_group:
|
||||
pretrain:
|
||||
task_list:
|
||||
- how2.yaml
|
||||
dataset:
|
||||
subsampling: 32
|
||||
sampled_min_len: 10
|
||||
sampled_max_len: 64
|
||||
max_video_len: 32
|
||||
max_len: 96
|
||||
aligner: MFMMLMAligner
|
||||
lazy_vfeat_mask: True
|
||||
mfm_probability: 0.15
|
||||
mlm_probability: 0.15
|
||||
mm_prob: 0.5
|
||||
model:
|
||||
model_cls: MMFusionMFMMLM
|
||||
mm_encoder_cls: MMFusionForMFMMLM
|
||||
loss:
|
||||
loss_cls: MFMMLM
|
||||
fairseq:
|
||||
common:
|
||||
fp16: true
|
||||
dataset:
|
||||
batch_size: 256
|
||||
optimization:
|
||||
max_epoch: 15
|
||||
finetune:
|
||||
task_list:
|
||||
- vtt.yaml
|
||||
- vttqa.yaml
|
||||
- youcook.yaml
|
||||
- youcookcap.yaml
|
||||
- crosstask.yaml
|
||||
- coin.yaml
|
||||
dataset:
|
||||
max_video_len: 32
|
||||
max_len: 96
|
||||
fairseq:
|
||||
common:
|
||||
fp16: true
|
||||
# do not write any model or loss here (they are expected to be fixed in mmfusion).
|
||||
test:
|
||||
task_list:
|
||||
- test_vtt.yaml
|
||||
- test_vttqa.yaml
|
||||
- test_youcook.yaml
|
||||
- test_youcookcap.yaml
|
||||
- test_crosstask.yaml
|
||||
- test_crosstask_zs.yaml
|
||||
- test_coin.yaml
|
||||
dataset:
|
||||
max_video_len: 32
|
||||
max_len: 96
|
19
examples/MMPT/projects/mtm/mmfusionmtm.yaml
Normal file
19
examples/MMPT/projects/mtm/mmfusionmtm.yaml
Normal file
@ -0,0 +1,19 @@
|
||||
includes: projects/mfmmlm.yaml
|
||||
project_dir: mtm/mmfusionmtm
|
||||
task_group:
|
||||
pretrain:
|
||||
task: VLMTask # reproducible
|
||||
dataset:
|
||||
aligner: MFMMLMAligner
|
||||
model:
|
||||
use_seg_emb: True # reproducible
|
||||
model_cls: MMFusionMTM
|
||||
mm_encoder_cls: MMBertForMFMMLM
|
||||
loss:
|
||||
loss_cls: MTM
|
||||
finetune:
|
||||
model:
|
||||
use_seg_emb: True # reproducible
|
||||
test:
|
||||
model:
|
||||
use_seg_emb: True # reproducible
|
8
examples/MMPT/projects/mtm/vlm.yaml
Normal file
8
examples/MMPT/projects/mtm/vlm.yaml
Normal file
@ -0,0 +1,8 @@
|
||||
includes: projects/mtm/mmfusionmtm.yaml
|
||||
project_dir: mtm/vlm
|
||||
task_group:
|
||||
pretrain:
|
||||
dataset:
|
||||
sampled_min_len: 8
|
||||
loss:
|
||||
loss_cls: MTM
|
47
examples/MMPT/projects/mtm/vlm/coin.yaml
Normal file
47
examples/MMPT/projects/mtm/vlm/coin.yaml
Normal file
@ -0,0 +1,47 @@
|
||||
dataset:
|
||||
video_processor: VideoProcessor
|
||||
bert_name: bert-base-uncased
|
||||
meta_processor: COINActionSegmentationMetaProcessor
|
||||
train_path: data/coin/COIN.json
|
||||
val_path: data/coin/COIN.json
|
||||
vfeat_dir: data/feat/feat_coin_s3d
|
||||
text_processor: COINActionSegmentationTextProcessor
|
||||
aligner: COINActionSegmentationAligner
|
||||
num_iso_layer: 12
|
||||
sliding_window: 8
|
||||
sliding_window_size: 32
|
||||
max_video_len: 32
|
||||
max_len: 96
|
||||
fairseq:
|
||||
common:
|
||||
tensorboard_logdir: run
|
||||
log_interval: 1000
|
||||
fp16: true
|
||||
dataset:
|
||||
num_workers: 4
|
||||
batch_size: 1
|
||||
optimization:
|
||||
lr:
|
||||
- 5.0e-05
|
||||
clip_norm: 2.0
|
||||
optimizer: adam
|
||||
adam_betas: (0.9, 0.98)
|
||||
lr_scheduler: polynomial_decay
|
||||
total_num_update: 1000000
|
||||
warmup_updates: 122
|
||||
weight_decay: 0.0
|
||||
ddp_backend: no_c10d
|
||||
max_epoch: 8
|
||||
checkpoint:
|
||||
restore_file: runs/mtm/vlm/checkpoint_best.pt
|
||||
reset_optimizer: true
|
||||
reset_dataloader: true
|
||||
reset_meters: true
|
||||
save_dir: runs/mtm/vlm/coin
|
||||
task_type: sweep_big
|
||||
model:
|
||||
model_cls: MMFusionActionSegmentation
|
||||
mm_encoder_cls: MMBertForTokenClassification
|
||||
use_seg_emb: true
|
||||
loss:
|
||||
loss_cls: CrossEntropy
|
53
examples/MMPT/projects/mtm/vlm/crosstask.yaml
Normal file
53
examples/MMPT/projects/mtm/vlm/crosstask.yaml
Normal file
@ -0,0 +1,53 @@
|
||||
dataset:
|
||||
video_processor: CrossTaskVideoProcessor
|
||||
bert_name: bert-base-uncased
|
||||
meta_processor: CrossTaskMetaProcessor
|
||||
train_path: data/crosstask/crosstask_release/videos.csv
|
||||
train_csv_path: data/crosstask/crosstask_release/videos.csv
|
||||
val_path: data/crosstask/crosstask_release/videos_val.csv
|
||||
val_csv_path: data/crosstask/crosstask_release/videos_val.csv
|
||||
primary_path: data/crosstask/crosstask_release/tasks_primary.txt
|
||||
related_path: data/crosstask/crosstask_release/tasks_related.txt
|
||||
vfeat_dir: data/feat/feat_crosstask_s3d
|
||||
annotation_path: data/crosstask/crosstask_release/annotations
|
||||
n_train: 30
|
||||
text_processor: CrossTaskTextProcessor
|
||||
aligner: CrossTaskAligner
|
||||
num_iso_layer: 12
|
||||
sliding_window: 16
|
||||
sliding_window_size: 32
|
||||
max_video_len: 32
|
||||
max_len: 96
|
||||
fairseq:
|
||||
common:
|
||||
tensorboard_logdir: run
|
||||
log_interval: 1000
|
||||
fp16: true
|
||||
dataset:
|
||||
num_workers: 4
|
||||
batch_size: 1
|
||||
optimization:
|
||||
lr:
|
||||
- 5.0e-05
|
||||
clip_norm: 2.0
|
||||
optimizer: adam
|
||||
adam_betas: (0.9, 0.98)
|
||||
lr_scheduler: polynomial_decay
|
||||
total_num_update: 1000000
|
||||
warmup_updates: 122
|
||||
weight_decay: 0.0
|
||||
ddp_backend: no_c10d
|
||||
max_epoch: 5
|
||||
checkpoint:
|
||||
restore_file: runs/mtm/vlm/checkpoint11.pt
|
||||
reset_optimizer: true
|
||||
reset_dataloader: true
|
||||
reset_meters: true
|
||||
save_dir: runs/mtm/vlm/crosstask
|
||||
task_type: sweep_small
|
||||
model:
|
||||
model_cls: MMFusionActionLocalization
|
||||
mm_encoder_cls: MMBertForJoint
|
||||
use_seg_emb: true
|
||||
loss:
|
||||
loss_cls: BCE
|
55
examples/MMPT/projects/mtm/vlm/how2.yaml
Normal file
55
examples/MMPT/projects/mtm/vlm/how2.yaml
Normal file
@ -0,0 +1,55 @@
|
||||
dataset:
|
||||
video_processor: ShardedVideoProcessor
|
||||
bert_name: bert-base-uncased
|
||||
meta_processor: ShardedHow2MetaProcessor
|
||||
train_path: data/how2/how2_s3d_train.lst
|
||||
val_path: data/how2/how2_s3d_val.lst
|
||||
vfeat_dir: data/feat/feat_how2_s3d_shard_small
|
||||
text_processor: ShardedTextProcessor
|
||||
tfeat_dir: data/feat/feat_how2_s3d_shard_small/raw_caption_dedup.bert-base-uncased.
|
||||
aligner: MFMMLMAligner
|
||||
subsampling: 32
|
||||
sampled_min_len: 8
|
||||
sampled_max_len: 64
|
||||
max_video_len: 32
|
||||
max_len: 96
|
||||
lazy_vfeat_mask: true
|
||||
mfm_probability: 0.15
|
||||
mlm_probability: 0.15
|
||||
mm_prob: 0.5
|
||||
fairseq:
|
||||
common:
|
||||
tensorboard_logdir: run
|
||||
log_interval: 1000
|
||||
fp16: true
|
||||
dataset:
|
||||
num_workers: 4
|
||||
batch_size: 256
|
||||
optimization:
|
||||
lr:
|
||||
- 5.0e-05
|
||||
clip_norm: 2.0
|
||||
optimizer: adam
|
||||
adam_betas: (0.9, 0.98)
|
||||
lr_scheduler: polynomial_decay
|
||||
total_num_update: 1000000
|
||||
warmup_updates: 1000
|
||||
weight_decay: 0.0
|
||||
ddp_backend: no_c10d
|
||||
max_epoch: 15
|
||||
checkpoint:
|
||||
save_dir: runs/mtm/vlm
|
||||
save_interval_updates: 1024
|
||||
keep_interval_updates: 2
|
||||
keep_last_epochs: 30
|
||||
task_type: sweep_big
|
||||
slurm_config: big
|
||||
eval:
|
||||
save_path: runs/mtm/vlm
|
||||
model:
|
||||
model_cls: MMFusionMTM
|
||||
mm_encoder_cls: MMBertForMFMMLM
|
||||
use_seg_emb: true
|
||||
loss:
|
||||
loss_cls: MTM
|
||||
task: VLMTask
|
31
examples/MMPT/projects/mtm/vlm/test_coin.yaml
Normal file
31
examples/MMPT/projects/mtm/vlm/test_coin.yaml
Normal file
@ -0,0 +1,31 @@
|
||||
slurm_config: big
|
||||
task_type: local_predict
|
||||
dataset:
|
||||
split: test
|
||||
video_processor: VideoProcessor
|
||||
aligner: COINActionSegmentationAligner
|
||||
bert_name: bert-base-uncased
|
||||
test_path: data/coin/COIN.json
|
||||
meta_processor: COINActionSegmentationMetaProcessor
|
||||
vfeat_dir: data/feat/feat_coin_s3d
|
||||
text_processor: COINActionSegmentationTextProcessor
|
||||
num_iso_layer: 12
|
||||
sliding_window: 16
|
||||
sliding_window_size: 32
|
||||
max_video_len: 32
|
||||
max_len: 96
|
||||
fairseq:
|
||||
dataset:
|
||||
batch_size: 1
|
||||
valid_subset: test
|
||||
num_workers: 2
|
||||
common_eval:
|
||||
path: runs/mtm/vlm/coin/checkpoint_best.pt
|
||||
model:
|
||||
model_cls: MMFusionActionSegmentation
|
||||
mm_encoder_cls: MMBertForTokenClassification
|
||||
use_seg_emb: true
|
||||
eval:
|
||||
save_path: runs/mtm/vlm/coin/eval
|
||||
metric: COINActionSegmentationMetric
|
||||
predictor: COINPredictor
|
38
examples/MMPT/projects/mtm/vlm/test_crosstask.yaml
Normal file
38
examples/MMPT/projects/mtm/vlm/test_crosstask.yaml
Normal file
@ -0,0 +1,38 @@
|
||||
slurm_config: big
|
||||
task_type: local_predict
|
||||
dataset:
|
||||
split: test
|
||||
video_processor: CrossTaskVideoProcessor
|
||||
aligner: CrossTaskAligner
|
||||
bert_name: bert-base-uncased
|
||||
meta_processor: CrossTaskMetaProcessor
|
||||
test_path: data/crosstask/crosstask_release/videos_val.csv
|
||||
train_csv_path: data/crosstask/crosstask_release/videos.csv
|
||||
val_path: data/crosstask/crosstask_release/videos_val.csv
|
||||
val_csv_path: data/crosstask/crosstask_release/videos_val.csv
|
||||
primary_path: data/crosstask/crosstask_release/tasks_primary.txt
|
||||
related_path: data/crosstask/crosstask_release/tasks_related.txt
|
||||
vfeat_dir: data/feat/feat_crosstask_s3d
|
||||
annotation_path: data/crosstask/crosstask_release/annotations
|
||||
n_train: 30
|
||||
text_processor: CrossTaskTextProcessor
|
||||
num_iso_layer: 12
|
||||
sliding_window: 16
|
||||
sliding_window_size: 32
|
||||
max_video_len: 32
|
||||
max_len: 96
|
||||
fairseq:
|
||||
dataset:
|
||||
batch_size: 1
|
||||
valid_subset: test
|
||||
num_workers: 2
|
||||
common_eval:
|
||||
path: runs/mtm/vlm/crosstask/checkpoint_best.pt
|
||||
model:
|
||||
model_cls: MMFusionActionLocalization
|
||||
mm_encoder_cls: MMBertForJoint
|
||||
use_seg_emb: true
|
||||
eval:
|
||||
save_path: runs/mtm/vlm/crosstask/eval
|
||||
metric: CrossTaskMetric
|
||||
predictor: CrossTaskPredictor
|
38
examples/MMPT/projects/mtm/vlm/test_crosstask_zs.yaml
Normal file
38
examples/MMPT/projects/mtm/vlm/test_crosstask_zs.yaml
Normal file
@ -0,0 +1,38 @@
|
||||
slurm_config: big
|
||||
task_type: local_predict
|
||||
dataset:
|
||||
split: test
|
||||
video_processor: CrossTaskVideoProcessor
|
||||
aligner: CrossTaskAligner
|
||||
bert_name: bert-base-uncased
|
||||
meta_processor: CrossTaskMetaProcessor
|
||||
test_path: data/crosstask/crosstask_release/videos_val.csv
|
||||
train_csv_path: data/crosstask/crosstask_release/videos.csv
|
||||
val_path: data/crosstask/crosstask_release/videos_val.csv
|
||||
val_csv_path: data/crosstask/crosstask_release/videos_val.csv
|
||||
primary_path: data/crosstask/crosstask_release/tasks_primary.txt
|
||||
related_path: data/crosstask/crosstask_release/tasks_related.txt
|
||||
vfeat_dir: data/feat/feat_crosstask_s3d
|
||||
annotation_path: data/crosstask/crosstask_release/annotations
|
||||
n_train: 30
|
||||
text_processor: CrossTaskTextProcessor
|
||||
num_iso_layer: 12
|
||||
sliding_window: 16
|
||||
sliding_window_size: 32
|
||||
max_video_len: 32
|
||||
max_len: 96
|
||||
fairseq:
|
||||
dataset:
|
||||
batch_size: 1
|
||||
valid_subset: test
|
||||
num_workers: 2
|
||||
common_eval:
|
||||
path: runs/mtm/vlm/checkpoint_best.pt
|
||||
model:
|
||||
model_cls: MMFusionActionLocalization
|
||||
mm_encoder_cls: MMBertForJoint
|
||||
use_seg_emb: true
|
||||
eval:
|
||||
save_path: runs/mtm/vlm/crosstask_zs/eval
|
||||
metric: CrossTaskMetric
|
||||
predictor: CrossTaskPredictor
|
29
examples/MMPT/projects/mtm/vlm/test_vtt.yaml
Normal file
29
examples/MMPT/projects/mtm/vlm/test_vtt.yaml
Normal file
@ -0,0 +1,29 @@
|
||||
slurm_config: big
|
||||
task_type: local_predict
|
||||
dataset:
|
||||
split: test
|
||||
video_processor: VideoProcessor
|
||||
aligner: DSAligner
|
||||
bert_name: bert-base-uncased
|
||||
meta_processor: MSRVTTMetaProcessor
|
||||
test_path: data/msrvtt/MSRVTT_JSFUSION_test.csv
|
||||
vfeat_dir: data/feat/feat_vtt_s3d
|
||||
text_processor: MSRVTTTextProcessor
|
||||
num_iso_layer: 12
|
||||
max_video_len: 32
|
||||
max_len: 96
|
||||
fairseq:
|
||||
dataset:
|
||||
batch_size: 256
|
||||
valid_subset: test
|
||||
num_workers: 2
|
||||
common_eval:
|
||||
path: runs/mtm/vlm/vtt/checkpoint_last.pt
|
||||
model:
|
||||
model_cls: MMFusionJoint
|
||||
mm_encoder_cls: MMBertForJoint
|
||||
use_seg_emb: true
|
||||
eval:
|
||||
save_path: runs/mtm/vlm/vtt/eval
|
||||
metric: RetrievalMetric
|
||||
predictor: RetrievalPredictor
|
29
examples/MMPT/projects/mtm/vlm/test_vttqa.yaml
Normal file
29
examples/MMPT/projects/mtm/vlm/test_vttqa.yaml
Normal file
@ -0,0 +1,29 @@
|
||||
slurm_config: big
|
||||
task_type: local_predict
|
||||
dataset:
|
||||
split: test
|
||||
video_processor: VideoProcessor
|
||||
aligner: MSRVTTQAAligner
|
||||
bert_name: bert-base-uncased
|
||||
meta_processor: MSRVTTQAMetaProcessor
|
||||
test_path: data/msrvtt-qa/MSR_MC_test.csv
|
||||
vfeat_dir: data/feat/feat_vtt_s3d
|
||||
text_processor: MSRVTTQATextProcessor
|
||||
num_iso_layer: 12
|
||||
max_video_len: 32
|
||||
max_len: 96
|
||||
fairseq:
|
||||
dataset:
|
||||
batch_size: 256
|
||||
valid_subset: test
|
||||
num_workers: 2
|
||||
common_eval:
|
||||
path: runs/mtm/vlm/vttqa/checkpoint_last.pt
|
||||
model:
|
||||
model_cls: MMFusionJoint
|
||||
mm_encoder_cls: MMBertForJoint
|
||||
use_seg_emb: true
|
||||
eval:
|
||||
save_path: runs/mtm/vlm/vttqa/eval
|
||||
metric: QAMetric
|
||||
predictor: QAPredictor
|
31
examples/MMPT/projects/mtm/vlm/test_youcook.yaml
Normal file
31
examples/MMPT/projects/mtm/vlm/test_youcook.yaml
Normal file
@ -0,0 +1,31 @@
|
||||
slurm_config: big
|
||||
task_type: local_predict
|
||||
dataset:
|
||||
split: test
|
||||
video_processor: YoucookVideoProcessor
|
||||
aligner: DSAligner
|
||||
bert_name: bert-base-uncased
|
||||
meta_processor: YoucookMetaProcessor
|
||||
test_path: data/youcook/youcook_val.pkl
|
||||
trainval_annotation: data/youcook/youcookii_annotations_trainval.json
|
||||
use_annotation_text: true
|
||||
vfeat_dir: data/feat/feat_youcook_s3d
|
||||
text_processor: TextProcessor
|
||||
num_iso_layer: 12
|
||||
max_video_len: 32
|
||||
max_len: 96
|
||||
fairseq:
|
||||
dataset:
|
||||
batch_size: 256
|
||||
valid_subset: test
|
||||
num_workers: 2
|
||||
common_eval:
|
||||
path: runs/mtm/vlm/youcook/checkpoint_last.pt
|
||||
model:
|
||||
model_cls: MMFusionJoint
|
||||
mm_encoder_cls: MMBertForJoint
|
||||
use_seg_emb: true
|
||||
eval:
|
||||
save_path: runs/mtm/vlm/youcook/eval
|
||||
metric: RetrievalMetric
|
||||
predictor: RetrievalPredictor
|
32
examples/MMPT/projects/mtm/vlm/test_youcookcap.yaml
Normal file
32
examples/MMPT/projects/mtm/vlm/test_youcookcap.yaml
Normal file
@ -0,0 +1,32 @@
|
||||
slurm_config: big
|
||||
task_type: local_predict
|
||||
dataset:
|
||||
split: test
|
||||
video_processor: YoucookVideoProcessor
|
||||
aligner: DSNLGAligner
|
||||
bert_name: bert-base-uncased
|
||||
meta_processor: YoucookNLGMetaProcessor
|
||||
test_path: data/youcook/val_list.txt
|
||||
trainval_annotation: data/youcook/youcookii_annotations_trainval.json
|
||||
vfeat_dir: data/feat/feat_youcook_s3d
|
||||
text_processor: NLGTextProcessor
|
||||
max_video_len: 32
|
||||
max_len: 96
|
||||
fairseq:
|
||||
dataset:
|
||||
batch_size: 256
|
||||
valid_subset: test
|
||||
num_workers: 2
|
||||
common_eval:
|
||||
path: runs/mtm/vlm/youcookcap/checkpoint_best.pt
|
||||
model:
|
||||
model_cls: MMFusionNLG
|
||||
mm_encoder_cls: MMBertForNLG
|
||||
max_decode_length: 24
|
||||
use_seg_emb: true
|
||||
eval:
|
||||
save_path: runs/mtm/vlm/youcookcap/eval
|
||||
metric: NLGMetric
|
||||
predictor: NLGPredictor
|
||||
gen_param:
|
||||
num_beams: 5
|
49
examples/MMPT/projects/mtm/vlm/vtt.yaml
Normal file
49
examples/MMPT/projects/mtm/vlm/vtt.yaml
Normal file
@ -0,0 +1,49 @@
|
||||
dataset:
|
||||
video_processor: VideoProcessor
|
||||
bert_name: bert-base-uncased
|
||||
meta_processor: MSRVTTMetaProcessor
|
||||
train_path: data/msrvtt/MSRVTT_train.csv
|
||||
jsfusion_path: data/msrvtt/MSRVTT_JSFUSION_test.csv
|
||||
full_test_path: data/msrvtt/MSRVTT_FULL_test.csv
|
||||
dup: 20
|
||||
val_path: data/msrvtt/MSRVTT_JSFUSION_test.csv
|
||||
vfeat_dir: data/feat/feat_vtt_s3d
|
||||
text_processor: MSRVTTTextProcessor
|
||||
json_path: data/msrvtt/MSRVTT_data.json
|
||||
aligner: DSAligner
|
||||
num_iso_layer: 12
|
||||
max_video_len: 32
|
||||
max_len: 96
|
||||
fairseq:
|
||||
common:
|
||||
tensorboard_logdir: run
|
||||
log_interval: 1000
|
||||
fp16: true
|
||||
dataset:
|
||||
num_workers: 4
|
||||
batch_size: 256
|
||||
optimization:
|
||||
lr:
|
||||
- 5.0e-05
|
||||
clip_norm: 2.0
|
||||
optimizer: adam
|
||||
adam_betas: (0.9, 0.98)
|
||||
lr_scheduler: polynomial_decay
|
||||
total_num_update: 1000000
|
||||
warmup_updates: 122
|
||||
weight_decay: 0.0
|
||||
ddp_backend: no_c10d
|
||||
max_epoch: 10
|
||||
checkpoint:
|
||||
restore_file: runs/mtm/vlm/checkpoint_best.pt
|
||||
reset_optimizer: true
|
||||
reset_dataloader: true
|
||||
reset_meters: true
|
||||
save_dir: runs/mtm/vlm/vtt
|
||||
task_type: sweep_small
|
||||
model:
|
||||
model_cls: MMFusionJoint
|
||||
mm_encoder_cls: MMBertForJoint
|
||||
use_seg_emb: true
|
||||
loss:
|
||||
loss_cls: T2VContraLoss
|
47
examples/MMPT/projects/mtm/vlm/vttqa.yaml
Normal file
47
examples/MMPT/projects/mtm/vlm/vttqa.yaml
Normal file
@ -0,0 +1,47 @@
|
||||
dataset:
|
||||
video_processor: VideoProcessor
|
||||
bert_name: bert-base-uncased
|
||||
meta_processor: MSRVTTMetaProcessor
|
||||
train_path: data/msrvtt/MSRVTT_train.csv
|
||||
dup: 20
|
||||
val_path: data/msrvtt/MSRVTT_JSFUSION_test.csv
|
||||
vfeat_dir: data/feat/feat_vtt_s3d
|
||||
text_processor: MSRVTTTextProcessor
|
||||
json_path: data/msrvtt/MSRVTT_data.json
|
||||
aligner: DSAligner
|
||||
num_iso_layer: 12
|
||||
max_video_len: 32
|
||||
max_len: 96
|
||||
fairseq:
|
||||
common:
|
||||
tensorboard_logdir: run
|
||||
log_interval: 1000
|
||||
fp16: true
|
||||
dataset:
|
||||
num_workers: 4
|
||||
batch_size: 128
|
||||
optimization:
|
||||
lr:
|
||||
- 5.0e-05
|
||||
clip_norm: 2.0
|
||||
optimizer: adam
|
||||
adam_betas: (0.9, 0.98)
|
||||
lr_scheduler: polynomial_decay
|
||||
total_num_update: 1000000
|
||||
warmup_updates: 122
|
||||
weight_decay: 0.0
|
||||
ddp_backend: no_c10d
|
||||
max_epoch: 5
|
||||
checkpoint:
|
||||
restore_file: runs/mtm/vlm/checkpoint_best.pt
|
||||
reset_optimizer: true
|
||||
reset_dataloader: true
|
||||
reset_meters: true
|
||||
save_dir: runs/mtm/vlm/vttqa
|
||||
task_type: sweep_small
|
||||
model:
|
||||
model_cls: MMFusionJoint
|
||||
mm_encoder_cls: MMBertForJoint
|
||||
use_seg_emb: true
|
||||
loss:
|
||||
loss_cls: V2TContraLoss
|
47
examples/MMPT/projects/mtm/vlm/youcook.yaml
Normal file
47
examples/MMPT/projects/mtm/vlm/youcook.yaml
Normal file
@ -0,0 +1,47 @@
|
||||
dataset:
|
||||
video_processor: YoucookVideoProcessor
|
||||
bert_name: bert-base-uncased
|
||||
meta_processor: YoucookMetaProcessor
|
||||
train_path: data/youcook/youcook_train.pkl
|
||||
val_path: data/youcook/youcook_val.pkl
|
||||
trainval_annotation: data/youcook/youcookii_annotations_trainval.json
|
||||
use_annotation_text: true
|
||||
vfeat_dir: data/feat/feat_youcook_s3d
|
||||
text_processor: TextProcessor
|
||||
aligner: DSAligner
|
||||
num_iso_layer: 12
|
||||
max_video_len: 32
|
||||
max_len: 96
|
||||
fairseq:
|
||||
common:
|
||||
tensorboard_logdir: run
|
||||
log_interval: 1000
|
||||
fp16: true
|
||||
dataset:
|
||||
num_workers: 4
|
||||
batch_size: 128
|
||||
optimization:
|
||||
lr:
|
||||
- 5.0e-05
|
||||
clip_norm: 2.0
|
||||
optimizer: adam
|
||||
adam_betas: (0.9, 0.98)
|
||||
lr_scheduler: polynomial_decay
|
||||
total_num_update: 1000000
|
||||
warmup_updates: 122
|
||||
weight_decay: 0.0
|
||||
ddp_backend: no_c10d
|
||||
max_epoch: 10
|
||||
checkpoint:
|
||||
restore_file: runs/mtm/vlm/checkpoint_best.pt
|
||||
reset_optimizer: true
|
||||
reset_dataloader: true
|
||||
reset_meters: true
|
||||
save_dir: runs/mtm/vlm/youcook
|
||||
task_type: sweep_small
|
||||
model:
|
||||
model_cls: MMFusionJoint
|
||||
mm_encoder_cls: MMBertForJoint
|
||||
use_seg_emb: true
|
||||
loss:
|
||||
loss_cls: T2VContraLoss
|
45
examples/MMPT/projects/mtm/vlm/youcookcap.yaml
Normal file
45
examples/MMPT/projects/mtm/vlm/youcookcap.yaml
Normal file
@ -0,0 +1,45 @@
|
||||
dataset:
|
||||
video_processor: YoucookVideoProcessor
|
||||
bert_name: bert-base-uncased
|
||||
meta_processor: YoucookNLGMetaProcessor
|
||||
train_path: data/youcook/train_list.txt
|
||||
val_path: data/youcook/val_list.txt
|
||||
trainval_annotation: data/youcook/youcookii_annotations_trainval.json
|
||||
vfeat_dir: data/feat/feat_youcook_s3d
|
||||
text_processor: NLGTextProcessor
|
||||
aligner: DSNLGAligner
|
||||
max_video_len: 32
|
||||
max_len: 96
|
||||
fairseq:
|
||||
common:
|
||||
tensorboard_logdir: run
|
||||
log_interval: 1000
|
||||
fp16: true
|
||||
dataset:
|
||||
num_workers: 4
|
||||
batch_size: 128
|
||||
optimization:
|
||||
lr:
|
||||
- 5.0e-05
|
||||
clip_norm: 2.0
|
||||
optimizer: adam
|
||||
adam_betas: (0.9, 0.98)
|
||||
lr_scheduler: polynomial_decay
|
||||
total_num_update: 1000000
|
||||
warmup_updates: 122
|
||||
weight_decay: 0.0
|
||||
ddp_backend: no_c10d
|
||||
max_epoch: 10
|
||||
checkpoint:
|
||||
restore_file: runs/mtm/vlm/checkpoint_best.pt
|
||||
reset_optimizer: true
|
||||
reset_dataloader: true
|
||||
reset_meters: true
|
||||
save_dir: runs/mtm/vlm/youcookcap
|
||||
task_type: sweep_small
|
||||
model:
|
||||
model_cls: MMFusionNLG
|
||||
mm_encoder_cls: MMBertForNLG
|
||||
use_seg_emb: true
|
||||
loss:
|
||||
loss_cls: NLGLoss
|
10
examples/MMPT/projects/retri/videoclip.yaml
Normal file
10
examples/MMPT/projects/retri/videoclip.yaml
Normal file
@ -0,0 +1,10 @@
|
||||
includes: projects/retri/videoretri.yaml
|
||||
project_dir: retri/videoclip
|
||||
task_group:
|
||||
pretrain:
|
||||
model:
|
||||
model_cls: MMFusionSeparate
|
||||
mm_encoder_cls:
|
||||
video_encoder_cls: MMBertForEncoder
|
||||
text_encoder_cls: BertModel
|
||||
num_hidden_video_layers: 6
|
49
examples/MMPT/projects/retri/videoclip/coin_videoclip.yaml
Normal file
49
examples/MMPT/projects/retri/videoclip/coin_videoclip.yaml
Normal file
@ -0,0 +1,49 @@
|
||||
dataset:
|
||||
video_processor: VideoProcessor
|
||||
bert_name: bert-base-uncased
|
||||
meta_processor: COINActionSegmentationMetaProcessor
|
||||
train_path: data/coin/COIN.json
|
||||
val_path: data/coin/COIN.json
|
||||
vfeat_dir: data/feat/feat_coin_s3d
|
||||
text_processor: COINActionSegmentationTextProcessor
|
||||
aligner: COINActionSegmentationAligner
|
||||
num_iso_layer: 12
|
||||
sliding_window: 8
|
||||
sliding_window_size: 32
|
||||
max_video_len: 32
|
||||
max_len: 96
|
||||
fairseq:
|
||||
common:
|
||||
tensorboard_logdir: run
|
||||
log_interval: 1000
|
||||
fp16: true
|
||||
dataset:
|
||||
num_workers: 4
|
||||
batch_size: 1
|
||||
optimization:
|
||||
lr:
|
||||
- 5.0e-05
|
||||
clip_norm: 2.0
|
||||
optimizer: adam
|
||||
adam_betas: (0.9, 0.98)
|
||||
lr_scheduler: polynomial_decay
|
||||
total_num_update: 1000000
|
||||
warmup_updates: 122
|
||||
weight_decay: 0.0
|
||||
ddp_backend: no_c10d
|
||||
max_epoch: 8
|
||||
checkpoint:
|
||||
restore_file: runs/retri/videoclip/checkpoint_best.pt
|
||||
reset_optimizer: true
|
||||
reset_dataloader: true
|
||||
reset_meters: true
|
||||
save_dir: runs/retri/videoclip/coin
|
||||
task_type: sweep_big
|
||||
model:
|
||||
model_cls: MMFusionSeparateActionSegmentation
|
||||
mm_encoder_cls: null
|
||||
video_encoder_cls: MMBertForTokenClassification
|
||||
text_encoder_cls: BertModel
|
||||
num_hidden_video_layers: 6
|
||||
loss:
|
||||
loss_cls: CrossEntropy
|
@ -0,0 +1,55 @@
|
||||
dataset:
|
||||
video_processor: CrossTaskVideoProcessor
|
||||
bert_name: bert-base-uncased
|
||||
meta_processor: CrossTaskMetaProcessor
|
||||
train_path: data/crosstask/crosstask_release/videos.csv
|
||||
train_csv_path: data/crosstask/crosstask_release/videos.csv
|
||||
val_path: data/crosstask/crosstask_release/videos_val.csv
|
||||
val_csv_path: data/crosstask/crosstask_release/videos_val.csv
|
||||
primary_path: data/crosstask/crosstask_release/tasks_primary.txt
|
||||
related_path: data/crosstask/crosstask_release/tasks_related.txt
|
||||
vfeat_dir: data/feat/feat_crosstask_s3d
|
||||
annotation_path: data/crosstask/crosstask_release/annotations
|
||||
n_train: 30
|
||||
text_processor: CrossTaskTextProcessor
|
||||
aligner: CrossTaskAligner
|
||||
num_iso_layer: 12
|
||||
sliding_window: 16
|
||||
sliding_window_size: 32
|
||||
max_video_len: 32
|
||||
max_len: 96
|
||||
fairseq:
|
||||
common:
|
||||
tensorboard_logdir: run
|
||||
log_interval: 1000
|
||||
fp16: true
|
||||
dataset:
|
||||
num_workers: 4
|
||||
batch_size: 1
|
||||
optimization:
|
||||
lr:
|
||||
- 5.0e-05
|
||||
clip_norm: 2.0
|
||||
optimizer: adam
|
||||
adam_betas: (0.9, 0.98)
|
||||
lr_scheduler: polynomial_decay
|
||||
total_num_update: 1000000
|
||||
warmup_updates: 122
|
||||
weight_decay: 0.0
|
||||
ddp_backend: no_c10d
|
||||
max_epoch: 5
|
||||
checkpoint:
|
||||
restore_file: runs/retri/videoclip/checkpoint_best.pt
|
||||
reset_optimizer: true
|
||||
reset_dataloader: true
|
||||
reset_meters: true
|
||||
save_dir: runs/retri/videoclip/crosstask
|
||||
task_type: sweep_small
|
||||
model:
|
||||
model_cls: MMFusionSeparateActionLocalization
|
||||
mm_encoder_cls: null
|
||||
video_encoder_cls: MMBertForEncoder
|
||||
text_encoder_cls: BertModel
|
||||
num_hidden_video_layers: 6
|
||||
loss:
|
||||
loss_cls: BCE
|
65
examples/MMPT/projects/retri/videoclip/how2.yaml
Normal file
65
examples/MMPT/projects/retri/videoclip/how2.yaml
Normal file
@ -0,0 +1,65 @@
|
||||
dataset:
|
||||
video_processor: ShardedVideoRetriVideoProcessor
|
||||
bert_name: bert-base-uncased
|
||||
meta_processor: ShardedHow2VideoRetriMetaProcessor
|
||||
train_path: data/how2/how2_s3d_train.lst
|
||||
val_path: data/how2/how2_s3d_val.lst
|
||||
vfeat_dir: data/feat/feat_how2_s3d_shard_small
|
||||
text_processor: ShardedVideoRetriTextProcessor
|
||||
tfeat_dir: data/feat/feat_how2_s3d_shard_small/raw_caption_dedup.bert-base-uncased.
|
||||
aligner: VideoRetriOverlappedAligner
|
||||
subsampling: 1
|
||||
sampled_min_len: 8
|
||||
sampled_max_len: 64
|
||||
max_video_len: 32
|
||||
max_len: 96
|
||||
lazy_vfeat_mask: true
|
||||
mfm_probability: 0.15
|
||||
mlm_probability: 0.15
|
||||
mm_prob: 0.5
|
||||
sampled_video_min_len: 3
|
||||
sampled_video_max_len: 32
|
||||
num_video_per_batch: 32
|
||||
clip_per_video: 16
|
||||
fairseq:
|
||||
common:
|
||||
tensorboard_logdir: run
|
||||
log_interval: 1000
|
||||
fp16: true
|
||||
dataset:
|
||||
num_workers: 4
|
||||
batch_size: 1
|
||||
optimization:
|
||||
lr:
|
||||
- 5.0e-05
|
||||
clip_norm: 2.0
|
||||
optimizer: adam
|
||||
adam_betas: (0.9, 0.98)
|
||||
lr_scheduler: polynomial_decay
|
||||
total_num_update: 1000000
|
||||
warmup_updates: 1000
|
||||
weight_decay: 0.0
|
||||
ddp_backend: no_c10d
|
||||
max_epoch: 25
|
||||
checkpoint:
|
||||
save_dir: runs/retri/videoclip
|
||||
save_interval_updates: 1024
|
||||
keep_interval_updates: 2
|
||||
keep_last_epochs: 30
|
||||
task_type: sweep_big
|
||||
slurm_config: big
|
||||
eval:
|
||||
save_path: runs/retri/videoclip
|
||||
model:
|
||||
model_cls: MMFusionSeparate
|
||||
mm_encoder_cls: null
|
||||
video_encoder_cls: MMBertForEncoder
|
||||
text_encoder_cls: BertModel
|
||||
num_hidden_video_layers: 6
|
||||
loss:
|
||||
loss_cls: MMContraLoss
|
||||
task: VideoRetriTask
|
||||
retri_epoch: 1
|
||||
vectorpool_cls: VideoVectorPool
|
||||
retriever_cls: VectorRetriever
|
||||
num_cands: 64
|
@ -0,0 +1,33 @@
|
||||
slurm_config: big
|
||||
task_type: local_predict
|
||||
dataset:
|
||||
split: test
|
||||
video_processor: VideoProcessor
|
||||
aligner: COINActionSegmentationAligner
|
||||
bert_name: bert-base-uncased
|
||||
test_path: data/coin/COIN.json
|
||||
meta_processor: COINActionSegmentationMetaProcessor
|
||||
vfeat_dir: data/feat/feat_coin_s3d
|
||||
text_processor: COINActionSegmentationTextProcessor
|
||||
num_iso_layer: 12
|
||||
sliding_window: 16
|
||||
sliding_window_size: 32
|
||||
max_video_len: 32
|
||||
max_len: 96
|
||||
fairseq:
|
||||
dataset:
|
||||
batch_size: 1
|
||||
valid_subset: test
|
||||
num_workers: 2
|
||||
common_eval:
|
||||
path: runs/retri/videoclip/coin/checkpoint_best.pt
|
||||
model:
|
||||
model_cls: MMFusionSeparateActionSegmentation
|
||||
mm_encoder_cls: null
|
||||
video_encoder_cls: MMBertForTokenClassification
|
||||
text_encoder_cls: BertModel
|
||||
num_hidden_video_layers: 6
|
||||
eval:
|
||||
save_path: runs/retri/videoclip/coin/eval
|
||||
metric: COINActionSegmentationMetric
|
||||
predictor: COINPredictor
|
33
examples/MMPT/projects/retri/videoclip/test_coin_zs.yaml
Normal file
33
examples/MMPT/projects/retri/videoclip/test_coin_zs.yaml
Normal file
@ -0,0 +1,33 @@
|
||||
slurm_config: big
|
||||
task_type: local_predict
|
||||
dataset:
|
||||
split: test
|
||||
video_processor: VideoProcessor
|
||||
aligner: COINActionSegmentationAligner
|
||||
bert_name: bert-base-uncased
|
||||
test_path: data/coin/COIN.json
|
||||
meta_processor: COINActionSegmentationMetaProcessor
|
||||
vfeat_dir: data/feat/feat_coin_s3d
|
||||
text_processor: COINActionSegmentationTextProcessor
|
||||
num_iso_layer: 12
|
||||
sliding_window: 16
|
||||
sliding_window_size: 32
|
||||
max_video_len: 32
|
||||
max_len: 96
|
||||
fairseq:
|
||||
dataset:
|
||||
batch_size: 1
|
||||
valid_subset: test
|
||||
num_workers: 2
|
||||
common_eval:
|
||||
path: runs/retri/videoclip/checkpoint_best.pt
|
||||
model:
|
||||
model_cls: MMFusionSeparate
|
||||
mm_encoder_cls: null
|
||||
video_encoder_cls: MMBertForEncoder
|
||||
text_encoder_cls: BertModel
|
||||
num_hidden_video_layers: 6
|
||||
eval:
|
||||
save_path: runs/retri/videoclip/coin_zs/eval
|
||||
metric: COINActionSegmentationMetric
|
||||
predictor: COINZSPredictor
|
@ -0,0 +1,40 @@
|
||||
slurm_config: big
|
||||
task_type: local_predict
|
||||
dataset:
|
||||
split: test
|
||||
video_processor: CrossTaskVideoProcessor
|
||||
aligner: CrossTaskAligner
|
||||
bert_name: bert-base-uncased
|
||||
meta_processor: CrossTaskMetaProcessor
|
||||
test_path: data/crosstask/crosstask_release/videos_val.csv
|
||||
train_csv_path: data/crosstask/crosstask_release/videos.csv
|
||||
val_path: data/crosstask/crosstask_release/videos_val.csv
|
||||
val_csv_path: data/crosstask/crosstask_release/videos_val.csv
|
||||
primary_path: data/crosstask/crosstask_release/tasks_primary.txt
|
||||
related_path: data/crosstask/crosstask_release/tasks_related.txt
|
||||
vfeat_dir: data/feat/feat_crosstask_s3d
|
||||
annotation_path: data/crosstask/crosstask_release/annotations
|
||||
n_train: 30
|
||||
text_processor: CrossTaskTextProcessor
|
||||
num_iso_layer: 12
|
||||
sliding_window: 16
|
||||
sliding_window_size: 32
|
||||
max_video_len: 32
|
||||
max_len: 96
|
||||
fairseq:
|
||||
dataset:
|
||||
batch_size: 1
|
||||
valid_subset: test
|
||||
num_workers: 2
|
||||
common_eval:
|
||||
path: runs/retri/videoclip/crosstask/checkpoint_best.pt
|
||||
model:
|
||||
model_cls: MMFusionSeparateActionLocalization
|
||||
mm_encoder_cls: null
|
||||
video_encoder_cls: MMBertForEncoder
|
||||
text_encoder_cls: BertModel
|
||||
num_hidden_video_layers: 6
|
||||
eval:
|
||||
save_path: runs/retri/videoclip/crosstask/eval
|
||||
metric: CrossTaskMetric
|
||||
predictor: CrossTaskPredictor
|
@ -0,0 +1,40 @@
|
||||
slurm_config: big
|
||||
task_type: local_predict
|
||||
dataset:
|
||||
split: test
|
||||
video_processor: CrossTaskVideoProcessor
|
||||
aligner: CrossTaskAligner
|
||||
bert_name: bert-base-uncased
|
||||
meta_processor: CrossTaskMetaProcessor
|
||||
test_path: data/crosstask/crosstask_release/videos_val.csv
|
||||
train_csv_path: data/crosstask/crosstask_release/videos.csv
|
||||
val_path: data/crosstask/crosstask_release/videos_val.csv
|
||||
val_csv_path: data/crosstask/crosstask_release/videos_val.csv
|
||||
primary_path: data/crosstask/crosstask_release/tasks_primary.txt
|
||||
related_path: data/crosstask/crosstask_release/tasks_related.txt
|
||||
vfeat_dir: data/feat/feat_crosstask_s3d
|
||||
annotation_path: data/crosstask/crosstask_release/annotations
|
||||
n_train: 30
|
||||
text_processor: CrossTaskTextProcessor
|
||||
num_iso_layer: 12
|
||||
sliding_window: 16
|
||||
sliding_window_size: 32
|
||||
max_video_len: 32
|
||||
max_len: 96
|
||||
fairseq:
|
||||
dataset:
|
||||
batch_size: 1
|
||||
valid_subset: test
|
||||
num_workers: 2
|
||||
common_eval:
|
||||
path: runs/retri/videoclip/checkpoint_best.pt
|
||||
model:
|
||||
model_cls: MMFusionSeparateActionLocalization
|
||||
mm_encoder_cls: null
|
||||
video_encoder_cls: MMBertForEncoder
|
||||
text_encoder_cls: BertModel
|
||||
num_hidden_video_layers: 6
|
||||
eval:
|
||||
save_path: runs/retri/videoclip/crosstask_zs/eval
|
||||
metric: CrossTaskMetric
|
||||
predictor: CrossTaskPredictor
|
31
examples/MMPT/projects/retri/videoclip/test_didemo_zs.yaml
Normal file
31
examples/MMPT/projects/retri/videoclip/test_didemo_zs.yaml
Normal file
@ -0,0 +1,31 @@
|
||||
slurm_config: big
|
||||
task_type: local_predict
|
||||
dataset:
|
||||
split: test
|
||||
video_processor: VideoProcessor
|
||||
aligner: DiDeMoAligner
|
||||
bert_name: bert-base-uncased
|
||||
meta_processor: DiDeMoMetaProcessor
|
||||
test_path: data/didemo/test_data.json
|
||||
vfeat_dir: data/feat/feat_didemo_s3d
|
||||
text_processor: DiDeMoTextProcessor
|
||||
num_iso_layer: 12
|
||||
max_video_len: 32
|
||||
max_len: 96
|
||||
fairseq:
|
||||
dataset:
|
||||
batch_size: 256
|
||||
valid_subset: test
|
||||
num_workers: 2
|
||||
common_eval:
|
||||
path: runs/retri/videoclip/checkpoint_best.pt
|
||||
model:
|
||||
model_cls: MMFusionSeparate
|
||||
mm_encoder_cls: null
|
||||
video_encoder_cls: MMBertForEncoder
|
||||
text_encoder_cls: BertModel
|
||||
num_hidden_video_layers: 6
|
||||
eval:
|
||||
save_path: runs/retri/videoclip/didemo_zs/eval
|
||||
metric: DiDeMoMetric
|
||||
predictor: DiDeMoPredictor
|
@ -0,0 +1,31 @@
|
||||
slurm_config: big
|
||||
task_type: local_predict
|
||||
dataset:
|
||||
split: test
|
||||
video_processor: VideoProcessor
|
||||
aligner: DSAligner
|
||||
bert_name: bert-base-uncased
|
||||
meta_processor: MSRVTTMetaProcessor
|
||||
test_path: data/msrvtt/MSRVTT_JSFUSION_test.csv
|
||||
vfeat_dir: data/feat/feat_vtt_s3d
|
||||
text_processor: MSRVTTTextProcessor
|
||||
num_iso_layer: 12
|
||||
max_video_len: 32
|
||||
max_len: 96
|
||||
fairseq:
|
||||
dataset:
|
||||
batch_size: 256
|
||||
valid_subset: test
|
||||
num_workers: 2
|
||||
common_eval:
|
||||
path: runs/retri/videoclip/vtt/checkpoint_last.pt
|
||||
model:
|
||||
model_cls: MMFusionSeparate
|
||||
mm_encoder_cls: null
|
||||
video_encoder_cls: MMBertForEncoder
|
||||
text_encoder_cls: BertModel
|
||||
num_hidden_video_layers: 6
|
||||
eval:
|
||||
save_path: runs/retri/videoclip/vtt/eval
|
||||
metric: RetrievalMetric
|
||||
predictor: RetrievalPredictor
|
31
examples/MMPT/projects/retri/videoclip/test_vtt_zs.yaml
Normal file
31
examples/MMPT/projects/retri/videoclip/test_vtt_zs.yaml
Normal file
@ -0,0 +1,31 @@
|
||||
slurm_config: big
|
||||
task_type: local_predict
|
||||
dataset:
|
||||
split: test
|
||||
video_processor: VideoProcessor
|
||||
aligner: DSAligner
|
||||
bert_name: bert-base-uncased
|
||||
meta_processor: MSRVTTMetaProcessor
|
||||
test_path: data/msrvtt/MSRVTT_JSFUSION_test.csv
|
||||
vfeat_dir: data/feat/feat_vtt_s3d
|
||||
text_processor: MSRVTTTextProcessor
|
||||
num_iso_layer: 12
|
||||
max_video_len: 32
|
||||
max_len: 96
|
||||
fairseq:
|
||||
dataset:
|
||||
batch_size: 256
|
||||
valid_subset: test
|
||||
num_workers: 2
|
||||
common_eval:
|
||||
path: runs/retri/videoclip/checkpoint_best.pt
|
||||
model:
|
||||
model_cls: MMFusionSeparate
|
||||
mm_encoder_cls: null
|
||||
video_encoder_cls: MMBertForEncoder
|
||||
text_encoder_cls: BertModel
|
||||
num_hidden_video_layers: 6
|
||||
eval:
|
||||
save_path: runs/retri/videoclip/vtt_zs/eval
|
||||
metric: RetrievalMetric
|
||||
predictor: RetrievalPredictor
|
@ -0,0 +1,31 @@
|
||||
slurm_config: big
|
||||
task_type: local_predict
|
||||
dataset:
|
||||
split: test
|
||||
video_processor: VideoProcessor
|
||||
aligner: MSRVTTQAAligner
|
||||
bert_name: bert-base-uncased
|
||||
meta_processor: MSRVTTQAMetaProcessor
|
||||
test_path: data/msrvtt-qa/MSR_MC_test.csv
|
||||
vfeat_dir: data/feat/feat_vtt_s3d
|
||||
text_processor: MSRVTTQATextProcessor
|
||||
num_iso_layer: 12
|
||||
max_video_len: 32
|
||||
max_len: 96
|
||||
fairseq:
|
||||
dataset:
|
||||
batch_size: 256
|
||||
valid_subset: test
|
||||
num_workers: 2
|
||||
common_eval:
|
||||
path: runs/retri/videoclip/vttqa/checkpoint_last.pt
|
||||
model:
|
||||
model_cls: MMFusionSeparate
|
||||
mm_encoder_cls: null
|
||||
video_encoder_cls: MMBertForEncoder
|
||||
text_encoder_cls: BertModel
|
||||
num_hidden_video_layers: 6
|
||||
eval:
|
||||
save_path: runs/retri/videoclip/vttqa/eval
|
||||
metric: QAMetric
|
||||
predictor: QAPredictor
|
31
examples/MMPT/projects/retri/videoclip/test_vttqa_zs.yaml
Normal file
31
examples/MMPT/projects/retri/videoclip/test_vttqa_zs.yaml
Normal file
@ -0,0 +1,31 @@
|
||||
slurm_config: big
|
||||
task_type: local_predict
|
||||
dataset:
|
||||
split: test
|
||||
video_processor: VideoProcessor
|
||||
aligner: MSRVTTQAAligner
|
||||
bert_name: bert-base-uncased
|
||||
meta_processor: MSRVTTQAMetaProcessor
|
||||
test_path: data/msrvtt-qa/MSR_MC_test.csv
|
||||
vfeat_dir: data/feat/feat_vtt_s3d
|
||||
text_processor: MSRVTTQATextProcessor
|
||||
num_iso_layer: 12
|
||||
max_video_len: 32
|
||||
max_len: 96
|
||||
fairseq:
|
||||
dataset:
|
||||
batch_size: 256
|
||||
valid_subset: test
|
||||
num_workers: 2
|
||||
common_eval:
|
||||
path: runs/retri/videoclip/checkpoint_best.pt
|
||||
model:
|
||||
model_cls: MMFusionSeparate
|
||||
mm_encoder_cls: null
|
||||
video_encoder_cls: MMBertForEncoder
|
||||
text_encoder_cls: BertModel
|
||||
num_hidden_video_layers: 6
|
||||
eval:
|
||||
save_path: runs/retri/videoclip/vttqa_zs/eval
|
||||
metric: QAMetric
|
||||
predictor: QAPredictor
|
@ -0,0 +1,33 @@
|
||||
slurm_config: big
|
||||
task_type: local_predict
|
||||
dataset:
|
||||
split: test
|
||||
video_processor: YoucookVideoProcessor
|
||||
aligner: DSAligner
|
||||
bert_name: bert-base-uncased
|
||||
meta_processor: YoucookMetaProcessor
|
||||
test_path: data/youcook/youcook_val.pkl
|
||||
trainval_annotation: data/youcook/youcookii_annotations_trainval.json
|
||||
use_annotation_text: true
|
||||
vfeat_dir: data/feat/feat_youcook_s3d
|
||||
text_processor: TextProcessor
|
||||
num_iso_layer: 12
|
||||
max_video_len: 32
|
||||
max_len: 96
|
||||
fairseq:
|
||||
dataset:
|
||||
batch_size: 256
|
||||
valid_subset: test
|
||||
num_workers: 2
|
||||
common_eval:
|
||||
path: runs/retri/videoclip/youcook/checkpoint_last.pt
|
||||
model:
|
||||
model_cls: MMFusionSeparate
|
||||
mm_encoder_cls: null
|
||||
video_encoder_cls: MMBertForEncoder
|
||||
text_encoder_cls: BertModel
|
||||
num_hidden_video_layers: 6
|
||||
eval:
|
||||
save_path: runs/retri/videoclip/youcook/eval
|
||||
metric: RetrievalMetric
|
||||
predictor: RetrievalPredictor
|
33
examples/MMPT/projects/retri/videoclip/test_youcook_zs.yaml
Normal file
33
examples/MMPT/projects/retri/videoclip/test_youcook_zs.yaml
Normal file
@ -0,0 +1,33 @@
|
||||
slurm_config: big
|
||||
task_type: local_predict
|
||||
dataset:
|
||||
split: test
|
||||
video_processor: YoucookVideoProcessor
|
||||
aligner: DSAligner
|
||||
bert_name: bert-base-uncased
|
||||
meta_processor: YoucookMetaProcessor
|
||||
test_path: data/youcook/youcook_val.pkl
|
||||
trainval_annotation: data/youcook/youcookii_annotations_trainval.json
|
||||
use_annotation_text: true
|
||||
vfeat_dir: data/feat/feat_youcook_s3d
|
||||
text_processor: TextProcessor
|
||||
num_iso_layer: 12
|
||||
max_video_len: 32
|
||||
max_len: 96
|
||||
fairseq:
|
||||
dataset:
|
||||
batch_size: 256
|
||||
valid_subset: test
|
||||
num_workers: 2
|
||||
common_eval:
|
||||
path: runs/retri/videoclip/checkpoint_best.pt
|
||||
model:
|
||||
model_cls: MMFusionSeparate
|
||||
mm_encoder_cls: null
|
||||
video_encoder_cls: MMBertForEncoder
|
||||
text_encoder_cls: BertModel
|
||||
num_hidden_video_layers: 6
|
||||
eval:
|
||||
save_path: runs/retri/videoclip/youcook_zs/eval
|
||||
metric: RetrievalMetric
|
||||
predictor: RetrievalPredictor
|
51
examples/MMPT/projects/retri/videoclip/vtt_videoclip.yaml
Normal file
51
examples/MMPT/projects/retri/videoclip/vtt_videoclip.yaml
Normal file
@ -0,0 +1,51 @@
|
||||
dataset:
|
||||
video_processor: VideoProcessor
|
||||
bert_name: bert-base-uncased
|
||||
meta_processor: MSRVTTMetaProcessor
|
||||
train_path: data/msrvtt/MSRVTT_train.csv
|
||||
jsfusion_path: data/msrvtt/MSRVTT_JSFUSION_test.csv
|
||||
full_test_path: data/msrvtt/MSRVTT_FULL_test.csv
|
||||
dup: 20
|
||||
val_path: data/msrvtt/MSRVTT_JSFUSION_test.csv
|
||||
vfeat_dir: data/feat/feat_vtt_s3d
|
||||
text_processor: MSRVTTTextProcessor
|
||||
json_path: data/msrvtt/MSRVTT_data.json
|
||||
aligner: DSAligner
|
||||
num_iso_layer: 12
|
||||
max_video_len: 32
|
||||
max_len: 96
|
||||
fairseq:
|
||||
common:
|
||||
tensorboard_logdir: run
|
||||
log_interval: 1000
|
||||
fp16: true
|
||||
dataset:
|
||||
num_workers: 4
|
||||
batch_size: 224
|
||||
optimization:
|
||||
lr:
|
||||
- 5.0e-05
|
||||
clip_norm: 2.0
|
||||
optimizer: adam
|
||||
adam_betas: (0.9, 0.98)
|
||||
lr_scheduler: polynomial_decay
|
||||
total_num_update: 1000000
|
||||
warmup_updates: 122
|
||||
weight_decay: 0.0
|
||||
ddp_backend: no_c10d
|
||||
max_epoch: 10
|
||||
checkpoint:
|
||||
restore_file: runs/retri/videoclip/checkpoint_best.pt
|
||||
reset_optimizer: true
|
||||
reset_dataloader: true
|
||||
reset_meters: true
|
||||
save_dir: runs/retri/videoclip/vtt
|
||||
task_type: sweep_small
|
||||
model:
|
||||
model_cls: MMFusionSeparate
|
||||
mm_encoder_cls: null
|
||||
video_encoder_cls: MMBertForEncoder
|
||||
text_encoder_cls: BertModel
|
||||
num_hidden_video_layers: 6
|
||||
loss:
|
||||
loss_cls: T2VContraLoss
|
49
examples/MMPT/projects/retri/videoclip/vttqa_videoclip.yaml
Normal file
49
examples/MMPT/projects/retri/videoclip/vttqa_videoclip.yaml
Normal file
@ -0,0 +1,49 @@
|
||||
dataset:
|
||||
video_processor: VideoProcessor
|
||||
bert_name: bert-base-uncased
|
||||
meta_processor: MSRVTTMetaProcessor
|
||||
train_path: data/msrvtt/MSRVTT_train.csv
|
||||
dup: 20
|
||||
val_path: data/msrvtt/MSRVTT_JSFUSION_test.csv
|
||||
vfeat_dir: data/feat/feat_vtt_s3d
|
||||
text_processor: MSRVTTTextProcessor
|
||||
json_path: data/msrvtt/MSRVTT_data.json
|
||||
aligner: DSAligner
|
||||
num_iso_layer: 12
|
||||
max_video_len: 32
|
||||
max_len: 96
|
||||
fairseq:
|
||||
common:
|
||||
tensorboard_logdir: run
|
||||
log_interval: 1000
|
||||
fp16: true
|
||||
dataset:
|
||||
num_workers: 4
|
||||
batch_size: 128
|
||||
optimization:
|
||||
lr:
|
||||
- 5.0e-05
|
||||
clip_norm: 2.0
|
||||
optimizer: adam
|
||||
adam_betas: (0.9, 0.98)
|
||||
lr_scheduler: polynomial_decay
|
||||
total_num_update: 1000000
|
||||
warmup_updates: 122
|
||||
weight_decay: 0.0
|
||||
ddp_backend: no_c10d
|
||||
max_epoch: 5
|
||||
checkpoint:
|
||||
restore_file: runs/retri/videoclip/checkpoint_best.pt
|
||||
reset_optimizer: true
|
||||
reset_dataloader: true
|
||||
reset_meters: true
|
||||
save_dir: runs/retri/videoclip/vttqa
|
||||
task_type: sweep_small
|
||||
model:
|
||||
model_cls: MMFusionSeparate
|
||||
mm_encoder_cls: null
|
||||
video_encoder_cls: MMBertForEncoder
|
||||
text_encoder_cls: BertModel
|
||||
num_hidden_video_layers: 6
|
||||
loss:
|
||||
loss_cls: V2TContraLoss
|
@ -0,0 +1,49 @@
|
||||
dataset:
|
||||
video_processor: YoucookVideoProcessor
|
||||
bert_name: bert-base-uncased
|
||||
meta_processor: YoucookMetaProcessor
|
||||
train_path: data/youcook/youcook_train.pkl
|
||||
val_path: data/youcook/youcook_val.pkl
|
||||
trainval_annotation: data/youcook/youcookii_annotations_trainval.json
|
||||
use_annotation_text: true
|
||||
vfeat_dir: data/feat/feat_youcook_s3d
|
||||
text_processor: TextProcessor
|
||||
aligner: DSAligner
|
||||
num_iso_layer: 12
|
||||
max_video_len: 32
|
||||
max_len: 96
|
||||
fairseq:
|
||||
common:
|
||||
tensorboard_logdir: run
|
||||
log_interval: 1000
|
||||
fp16: true
|
||||
dataset:
|
||||
num_workers: 4
|
||||
batch_size: 128
|
||||
optimization:
|
||||
lr:
|
||||
- 5.0e-05
|
||||
clip_norm: 2.0
|
||||
optimizer: adam
|
||||
adam_betas: (0.9, 0.98)
|
||||
lr_scheduler: polynomial_decay
|
||||
total_num_update: 1000000
|
||||
warmup_updates: 122
|
||||
weight_decay: 0.0
|
||||
ddp_backend: no_c10d
|
||||
max_epoch: 10
|
||||
checkpoint:
|
||||
restore_file: runs/retri/videoclip/checkpoint_best.pt
|
||||
reset_optimizer: true
|
||||
reset_dataloader: true
|
||||
reset_meters: true
|
||||
save_dir: runs/retri/videoclip/youcook
|
||||
task_type: sweep_small
|
||||
model:
|
||||
model_cls: MMFusionSeparate
|
||||
mm_encoder_cls: null
|
||||
video_encoder_cls: MMBertForEncoder
|
||||
text_encoder_cls: BertModel
|
||||
num_hidden_video_layers: 6
|
||||
loss:
|
||||
loss_cls: T2VContraLoss
|
51
examples/MMPT/projects/retri/videoretri.yaml
Normal file
51
examples/MMPT/projects/retri/videoretri.yaml
Normal file
@ -0,0 +1,51 @@
|
||||
includes: projects/mfmmlm.yaml
|
||||
project_dir: retri/videoretri
|
||||
run_task:
|
||||
- how2.yaml
|
||||
task_group:
|
||||
pretrain:
|
||||
task: VideoRetriTask
|
||||
retri_epoch: 1
|
||||
vectorpool_cls: VideoVectorPool
|
||||
retriever_cls: VectorRetriever
|
||||
num_cands: 64
|
||||
dataset:
|
||||
train_path: data/how2/how2_s3d_train.lst
|
||||
meta_processor: ShardedHow2VideoRetriMetaProcessor
|
||||
video_processor: ShardedVideoRetriVideoProcessor
|
||||
text_processor: ShardedVideoRetriTextProcessor
|
||||
aligner: VideoRetriOverlappedAligner
|
||||
sampled_video_min_len: 3
|
||||
sampled_video_max_len: 32
|
||||
sampled_min_len: 8
|
||||
sampled_max_len: 64
|
||||
num_video_per_batch: 32
|
||||
# do not use subsampling as it changes fairseq batch_size.
|
||||
subsampling: 1 # disable subsampling
|
||||
clip_per_video: 16
|
||||
fairseq:
|
||||
dataset:
|
||||
batch_size: 1
|
||||
optimization:
|
||||
max_epoch: 25
|
||||
model:
|
||||
model_cls: MMFusionShare
|
||||
mm_encoder_cls: MMBertForEncoder
|
||||
loss:
|
||||
loss_cls: MMContraLoss
|
||||
finetune:
|
||||
task_list: [vtt_videoclip.yaml, youcook_videoclip.yaml, vttqa_videoclip.yaml, crosstask_videoclip.yaml, coin_videoclip.yaml]
|
||||
test:
|
||||
task_list:
|
||||
- test_youcook_zs.yaml
|
||||
- test_vtt_zs.yaml
|
||||
- test_vttqa_zs.yaml
|
||||
- test_crosstask_zs_videoclip.yaml
|
||||
- test_coin_zs.yaml
|
||||
- test_didemo_zs.yaml
|
||||
- test_youcook_videoclip.yaml
|
||||
- test_vtt_videoclip.yaml
|
||||
- test_vttqa_videoclip.yaml
|
||||
- test_crosstask_videoclip.yaml
|
||||
- test_coin_videoclip.yaml
|
||||
|
25
examples/MMPT/projects/task/coin.yaml
Normal file
25
examples/MMPT/projects/task/coin.yaml
Normal file
@ -0,0 +1,25 @@
|
||||
includes: projects/task/ft.yaml
|
||||
task_type: sweep_big
|
||||
dataset:
|
||||
meta_processor: COINActionSegmentationMetaProcessor
|
||||
train_path: data/coin/COIN.json
|
||||
val_path: data/coin/COIN.json
|
||||
vfeat_dir: data/feat/feat_coin_s3d
|
||||
video_processor: VideoProcessor
|
||||
text_processor: COINActionSegmentationTextProcessor
|
||||
aligner: COINActionSegmentationAligner
|
||||
num_iso_layer: 12
|
||||
sliding_window: 8
|
||||
sliding_window_size: 32
|
||||
model:
|
||||
model_cls: MMFusionActionSegmentation
|
||||
mm_encoder_cls: MMBertForTokenClassification
|
||||
loss:
|
||||
loss_cls: CrossEntropy
|
||||
fairseq:
|
||||
dataset:
|
||||
batch_size: 1
|
||||
optimization:
|
||||
max_epoch: 8
|
||||
checkpoint:
|
||||
save_dir: runs/task/coin
|
7
examples/MMPT/projects/task/coin_videoclip.yaml
Normal file
7
examples/MMPT/projects/task/coin_videoclip.yaml
Normal file
@ -0,0 +1,7 @@
|
||||
includes: projects/task/coin.yaml
|
||||
model:
|
||||
model_cls: MMFusionSeparateActionSegmentation
|
||||
mm_encoder_cls:
|
||||
video_encoder_cls: MMBertForTokenClassification
|
||||
text_encoder_cls: BertModel # dummy, not used.
|
||||
num_hidden_video_layers: 6
|
31
examples/MMPT/projects/task/crosstask.yaml
Normal file
31
examples/MMPT/projects/task/crosstask.yaml
Normal file
@ -0,0 +1,31 @@
|
||||
includes: projects/task/ft.yaml
|
||||
dataset:
|
||||
meta_processor: CrossTaskMetaProcessor
|
||||
train_path: data/crosstask/crosstask_release/videos.csv # dummy
|
||||
train_csv_path: data/crosstask/crosstask_release/videos.csv
|
||||
val_path: data/crosstask/crosstask_release/videos_val.csv # dummy
|
||||
val_csv_path: data/crosstask/crosstask_release/videos_val.csv
|
||||
primary_path: data/crosstask/crosstask_release/tasks_primary.txt
|
||||
related_path: data/crosstask/crosstask_release/tasks_related.txt
|
||||
vfeat_dir: data/feat/feat_crosstask_s3d
|
||||
annotation_path: data/crosstask/crosstask_release/annotations
|
||||
n_train: 30
|
||||
video_processor: CrossTaskVideoProcessor
|
||||
text_processor: CrossTaskTextProcessor
|
||||
aligner: CrossTaskAligner
|
||||
num_iso_layer: 12
|
||||
sliding_window: 16
|
||||
sliding_window_size: 32
|
||||
model:
|
||||
model_cls: MMFusionActionLocalization
|
||||
mm_encoder_cls: MMBertForJoint
|
||||
loss:
|
||||
loss_cls: BCE
|
||||
fairseq:
|
||||
dataset:
|
||||
batch_size: 1
|
||||
optimization:
|
||||
max_epoch: 5
|
||||
checkpoint:
|
||||
save_dir: runs/task/crosstask
|
||||
restore_file: runs/task/checkpoint11.pt # for VLM
|
10
examples/MMPT/projects/task/crosstask_videoclip.yaml
Normal file
10
examples/MMPT/projects/task/crosstask_videoclip.yaml
Normal file
@ -0,0 +1,10 @@
|
||||
includes: projects/task/crosstask.yaml
|
||||
model:
|
||||
model_cls: MMFusionSeparateActionLocalization
|
||||
mm_encoder_cls:
|
||||
video_encoder_cls: MMBertForEncoder
|
||||
text_encoder_cls: BertModel # dummy, not used.
|
||||
num_hidden_video_layers: 6
|
||||
fairseq:
|
||||
checkpoint:
|
||||
restore_file: runs/task/checkpoint_best.pt # overwrite the default of VLM.
|
20
examples/MMPT/projects/task/default.yaml
Normal file
20
examples/MMPT/projects/task/default.yaml
Normal file
@ -0,0 +1,20 @@
|
||||
# this yaml cannot be run alone. you must use `how2.yaml`, `vtt.yaml` etc for training.
|
||||
dataset:
|
||||
video_processor: VideoProcessor
|
||||
bert_name: bert-base-uncased
|
||||
fairseq:
|
||||
common:
|
||||
tensorboard_logdir: run
|
||||
log_interval: 1000
|
||||
dataset:
|
||||
num_workers: 4
|
||||
optimization:
|
||||
lr: [ 0.00005 ]
|
||||
clip_norm: 2.0
|
||||
optimizer: adam
|
||||
adam_betas: (0.9, 0.98)
|
||||
lr_scheduler: polynomial_decay
|
||||
total_num_update: 1000000 # backward compatible on fairseq 1.0.0a0+af0389f for reproducibility.
|
||||
warmup_updates: 1000
|
||||
weight_decay: 0.0
|
||||
ddp_backend: no_c10d
|
13
examples/MMPT/projects/task/ft.yaml
Normal file
13
examples/MMPT/projects/task/ft.yaml
Normal file
@ -0,0 +1,13 @@
|
||||
includes: projects/task/default.yaml
|
||||
# all derived config will be run by fairseq-train.
|
||||
task_type: sweep_small
|
||||
fairseq:
|
||||
optimization:
|
||||
warmup_updates: 122 # copied from roberta glue: https://github.com/pytorch/fairseq/blob/master/examples/roberta/README.glue.md
|
||||
checkpoint:
|
||||
# save_interval_updates: 512
|
||||
# borrowed from Roberta script.
|
||||
restore_file: runs/task/checkpoint_best.pt
|
||||
reset_optimizer: True
|
||||
reset_dataloader: True
|
||||
reset_meters: True
|
22
examples/MMPT/projects/task/how2.yaml
Normal file
22
examples/MMPT/projects/task/how2.yaml
Normal file
@ -0,0 +1,22 @@
|
||||
includes: projects/task/default.yaml
|
||||
task_type: sweep_big
|
||||
slurm_config: big
|
||||
dataset:
|
||||
meta_processor: ShardedHow2MetaProcessor
|
||||
train_path: data/how2/how2_s3d_train.lst
|
||||
val_path: data/how2/how2_s3d_val.lst
|
||||
video_processor: ShardedVideoProcessor
|
||||
vfeat_dir: data/feat/feat_how2_s3d_shard_small
|
||||
text_processor: ShardedTextProcessor
|
||||
tfeat_dir: data/feat/feat_how2_s3d_shard_small/raw_caption_dedup.bert-base-uncased.
|
||||
aligner: FixedLenAligner
|
||||
# disable direct running of this yaml
|
||||
eval:
|
||||
save_path: runs/task
|
||||
fairseq:
|
||||
checkpoint:
|
||||
save_dir: runs/task
|
||||
save_interval_updates: 1024
|
||||
keep_interval_updates: 2
|
||||
keep_last_epochs: 30
|
||||
|
13
examples/MMPT/projects/task/test.yaml
Normal file
13
examples/MMPT/projects/task/test.yaml
Normal file
@ -0,0 +1,13 @@
|
||||
# this yaml cannot be run alone: implement a test_${dataset}.yaml
|
||||
slurm_config: big
|
||||
task_type: local_predict
|
||||
dataset:
|
||||
split: test
|
||||
video_processor: VideoProcessor
|
||||
aligner: DSAligner
|
||||
bert_name: bert-base-uncased
|
||||
fairseq:
|
||||
dataset:
|
||||
batch_size: 256
|
||||
valid_subset: test
|
||||
num_workers: 2
|
24
examples/MMPT/projects/task/test_coin.yaml
Normal file
24
examples/MMPT/projects/task/test_coin.yaml
Normal file
@ -0,0 +1,24 @@
|
||||
includes: projects/task/test.yaml
|
||||
dataset:
|
||||
split: test
|
||||
test_path: data/coin/COIN.json
|
||||
meta_processor: COINActionSegmentationMetaProcessor
|
||||
vfeat_dir: data/feat/feat_coin_s3d
|
||||
video_processor: VideoProcessor
|
||||
text_processor: COINActionSegmentationTextProcessor
|
||||
aligner: COINActionSegmentationAligner
|
||||
num_iso_layer: 12
|
||||
sliding_window: 16
|
||||
sliding_window_size: 32
|
||||
model:
|
||||
model_cls: MMFusionActionSegmentation
|
||||
mm_encoder_cls: MMBertForTokenClassification
|
||||
eval:
|
||||
save_path: runs/task/coin/eval
|
||||
fairseq:
|
||||
dataset:
|
||||
batch_size: 1
|
||||
common_eval:
|
||||
path: runs/task/coin/checkpoint_best.pt
|
||||
metric: COINActionSegmentationMetric
|
||||
predictor: COINPredictor
|
7
examples/MMPT/projects/task/test_coin_videoclip.yaml
Normal file
7
examples/MMPT/projects/task/test_coin_videoclip.yaml
Normal file
@ -0,0 +1,7 @@
|
||||
includes: projects/task/test_coin.yaml
|
||||
model:
|
||||
model_cls: MMFusionSeparateActionSegmentation
|
||||
mm_encoder_cls:
|
||||
video_encoder_cls: MMBertForTokenClassification
|
||||
text_encoder_cls: BertModel # dummy, not used.
|
||||
num_hidden_video_layers: 6
|
13
examples/MMPT/projects/task/test_coin_zs.yaml
Normal file
13
examples/MMPT/projects/task/test_coin_zs.yaml
Normal file
@ -0,0 +1,13 @@
|
||||
includes: projects/task/test_coin.yaml
|
||||
model:
|
||||
model_cls: MMFusionSeparate
|
||||
mm_encoder_cls:
|
||||
video_encoder_cls: MMBertForEncoder
|
||||
text_encoder_cls: BertModel
|
||||
num_hidden_video_layers: 6
|
||||
eval:
|
||||
save_path: runs/task/coin_zs/eval
|
||||
fairseq:
|
||||
common_eval:
|
||||
path: runs/task/checkpoint_best.pt
|
||||
predictor: COINZSPredictor
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user