tools

tools/count_videos_time.py

@@ -0,0 +1,45 @@
+# Copyright (c) 2024 Bytedance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import matplotlib.pyplot as plt
+from latentsync.utils.util import count_video_time, gather_video_paths_recursively
+from tqdm import tqdm
+
+
+def plot_histogram(data, fig_path):
+    # Create histogram
+    plt.hist(data, bins=30, edgecolor="black")
+
+    # Add titles and labels
+    plt.title("Histogram of Data Distribution")
+    plt.xlabel("Video time")
+    plt.ylabel("Frequency")
+
+    # Save plot as an image file
+    plt.savefig(fig_path)  # Save as PNG file. You can also use 'histogram.jpg', 'histogram.pdf', etc.
+
+
+def main(input_dir, fig_path):
+    video_paths = gather_video_paths_recursively(input_dir)
+    video_times = []
+    for video_path in tqdm(video_paths):
+        video_times.append(count_video_time(video_path))
+    plot_histogram(video_times, fig_path)
+
+
+if __name__ == "__main__":
+    input_dir = "validation"
+    fig_path = "histogram.png"
+
+    main(input_dir, fig_path)

tools/download_youtube_videos.py

@@ -0,0 +1,113 @@
+# Copyright (c) 2024 Bytedance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import subprocess
+from concurrent.futures import ThreadPoolExecutor
+import pandas as pd
+from tqdm import tqdm
+
+"""
+To use this python file, first install yt-dlp by:
+
+pip install yt-dlp==2024.5.27
+"""
+
+
+def download_video(video_url, video_path):
+    get_video_channel_command = f"yt-dlp --print channel {video_url}"
+    result = subprocess.run(get_video_channel_command, shell=True, capture_output=True, text=True)
+    channel = result.stdout.strip()
+    if channel in unwanted_channels:
+        return
+    download_video_command = f"yt-dlp -f bestvideo+bestaudio --skip-unavailable-fragments --merge-output-format mp4 '{video_url}' --output '{video_path}' --external-downloader aria2c --external-downloader-args '-x 16 -k 1M'"
+    try:
+        subprocess.run(download_video_command, shell=True)  # ignore_security_alert_wait_for_fix RCE
+    except KeyboardInterrupt:
+        print("Stopped")
+        exit()
+    except:
+        print(f"Error downloading video {video_url}")
+
+
+def download_videos(num_workers, video_urls, video_paths):
+    with ThreadPoolExecutor(max_workers=num_workers) as executor:
+        executor.map(download_video, video_urls, video_paths)
+
+
+def read_video_urls(csv_file_path: str, language_column, video_url_column):
+    video_urls = []
+    print("Reading video urls...")
+    df = pd.read_csv(csv_file_path, sep=",")
+    for row in tqdm(df.itertuples(), total=len(df)):
+        language = getattr(row, language_column)
+        video_url = getattr(row, video_url_column)
+        if "clip" in video_url:
+            continue
+        video_urls.append((language, video_url))
+    return video_urls
+
+
+def extract_vid(video_url):
+    if "watch?v=" in video_url:  # ignore_security_alert_wait_for_fix RCE
+        return video_url.split("watch?v=")[1][:11]
+    elif "shorts/" in video_url:
+        return video_url.split("shorts/")[1][:11]
+    elif "youtu.be/" in video_url:
+        return video_url.split("youtu.be/")[1][:11]
+    elif "&v=" in video_url:
+        return video_url.split("&v=")[1][:11]
+    else:
+        print(f"Invalid video url: {video_url}")
+        return None
+
+
+def main(csv_file_path, language_column, video_url_column, output_dir, num_workers):
+    os.makedirs(output_dir, exist_ok=True)
+    all_video_urls = read_video_urls(csv_file_path, language_column, video_url_column)
+
+    video_paths = []
+    video_urls = []
+
+    print("Extracting vid...")
+    for language, video_url in tqdm(all_video_urls):
+        vid = extract_vid(video_url)
+        if vid is None:
+            continue
+        video_path = os.path.join(output_dir, language.lower(), f"vid_{vid}.mp4")
+        if os.path.isfile(video_path):
+            continue
+        os.makedirs(os.path.dirname(video_path), exist_ok=True)
+        video_paths.append(video_path)
+        video_urls.append(video_url)
+
+    if len(video_paths) == 0:
+        print("All videos have been downloaded")
+        exit()
+    else:
+        print(f"Downloading {len(video_paths)} videos")
+
+    download_videos(num_workers, video_urls, video_paths)
+
+
+if __name__ == "__main__":
+    csv_file_path = "dcc.csv"
+    language_column = "video_language"
+    video_url_column = "video_link"
+    output_dir = "/mnt/bn/maliva-gen-ai-v2/chunyu.li/multilingual/raw"
+    num_workers = 50
+
+    unwanted_channels = ["TEDx Talks", "DaePyeong Mukbang", "Joeman"]
+
+    main(csv_file_path, language_column, video_url_column, output_dir, num_workers)

tools/move_files_recur.py

@@ -0,0 +1,48 @@
+# Copyright (c) 2024 Bytedance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import shutil
+from tqdm import tqdm
+
+paths = []
+
+
+def gather_paths(input_dir, output_dir):
+    os.makedirs(output_dir, exist_ok=True)
+
+    for video in sorted(os.listdir(input_dir)):
+        if video.endswith(".mp4"):
+            video_input = os.path.join(input_dir, video)
+            video_output = os.path.join(output_dir, video)
+            if os.path.isfile(video_output):
+                continue
+            paths.append([video_input, output_dir])
+        elif os.path.isdir(os.path.join(input_dir, video)):
+            gather_paths(os.path.join(input_dir, video), os.path.join(output_dir, video))
+
+
+def main(input_dir, output_dir):
+    print(f"Recursively gathering video paths of {input_dir} ...")
+    gather_paths(input_dir, output_dir)
+
+    for video_input, output_dir in tqdm(paths):
+        shutil.move(video_input, output_dir)
+
+
+if __name__ == "__main__":
+    input_dir = "/mnt/bn/maliva-gen-ai-v2/chunyu.li/multilingual_dcc"
+    output_dir = "/mnt/bn/maliva-gen-ai-v2/chunyu.li/multilingual"
+
+    main(input_dir, output_dir)

tools/occupy_gpu.py

@@ -0,0 +1,60 @@
+# Copyright (c) 2024 Bytedance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import torch
+import os
+import torch.multiprocessing as mp
+import time
+
+
+def check_mem(cuda_device):
+    devices_info = (
+        os.popen('"/usr/bin/nvidia-smi" --query-gpu=memory.total,memory.used --format=csv,nounits,noheader')
+        .read()
+        .strip()
+        .split("\n")
+    )
+    total, used = devices_info[int(cuda_device)].split(",")
+    return total, used
+
+
+def loop(cuda_device):
+    cuda_i = torch.device(f"cuda:{cuda_device}")
+    total, used = check_mem(cuda_device)
+    total = int(total)
+    used = int(used)
+    max_mem = int(total * 0.9)
+    block_mem = max_mem - used
+    while True:
+        x = torch.rand(20, 512, 512, dtype=torch.float, device=cuda_i)
+        y = torch.rand(20, 512, 512, dtype=torch.float, device=cuda_i)
+        time.sleep(0.001)
+        x = torch.matmul(x, y)
+
+
+def main():
+    if torch.cuda.is_available():
+        num_processes = torch.cuda.device_count()
+        processes = list()
+        for i in range(num_processes):
+            p = mp.Process(target=loop, args=(i,))
+            p.start()
+            processes.append(p)
+        for p in processes:
+            p.join()
+
+
+if __name__ == "__main__":
+    torch.multiprocessing.set_start_method("spawn")
+    main()

tools/remove_outdated_files.py

@@ -0,0 +1,34 @@
+# Copyright (c) 2024 Bytedance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import subprocess
+
+
+def remove_outdated_files(input_dir, begin_date, end_date):
+    # Remove files from a specific time period
+    for subdir in os.listdir(input_dir):
+        if subdir >= begin_date and subdir <= end_date:
+            subdir_path = os.path.join(input_dir, subdir)
+            command = f"rm -rf {subdir_path}"
+            subprocess.run(command, shell=True)
+            print(f"Deleted: {subdir_path}")
+
+
+if __name__ == "__main__":
+    input_dir = "/mnt/bn/video-datasets/output/syncnet"
+    begin_date = "train-2024_06_19-16:25:44"
+    end_date = "train-2024_08_03-07:39:58"
+
+    remove_outdated_files(input_dir, begin_date, end_date)

tools/write_fileslist.py

@@ -0,0 +1,45 @@
+# Copyright (c) 2024 Bytedance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from tqdm import tqdm
+from latentsync.utils.util import gather_video_paths_recursively
+
+
+def write_fileslist(fileslist_path):
+    with open(fileslist_path, "w") as _:
+        pass
+
+
+def append_fileslist(fileslist_path, video_paths):
+    with open(fileslist_path, "a") as f:
+        for video_path in tqdm(video_paths):
+            f.write(f"{video_path}\n")
+
+
+def process_input_dir(fileslist_path, input_dir):
+    print(f"Processing input dir: {input_dir}")
+    video_paths = gather_video_paths_recursively(input_dir)
+    append_fileslist(fileslist_path, video_paths)
+
+
+if __name__ == "__main__":
+    fileslist_path = "/mnt/bn/maliva-gen-ai-v2/chunyu.li/fileslist/all_data_v6.txt"
+
+    write_fileslist(fileslist_path)
+    process_input_dir(fileslist_path, "/mnt/bn/maliva-gen-ai-v2/chunyu.li/VoxCeleb2/high_visual_quality/train")
+    process_input_dir(fileslist_path, "/mnt/bn/maliva-gen-ai-v2/chunyu.li/HDTF/high_visual_quality/train")
+    process_input_dir(fileslist_path, "/mnt/bn/maliva-gen-ai-v2/chunyu.li/avatars/high_visual_quality/train")
+    process_input_dir(fileslist_path, "/mnt/bn/maliva-gen-ai-v2/chunyu.li/multilingual/high_visual_quality")
+    process_input_dir(fileslist_path, "/mnt/bn/maliva-gen-ai-v2/chunyu.li/celebv_text/high_visual_quality/train")
+    process_input_dir(fileslist_path, "/mnt/bn/maliva-gen-ai-v2/chunyu.li/youtube/high_visual_quality")