diff --git a/tools/count_videos_time.py b/tools/count_videos_time.py new file mode 100644 index 0000000..cb37842 --- /dev/null +++ b/tools/count_videos_time.py @@ -0,0 +1,45 @@ +# Copyright (c) 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import matplotlib.pyplot as plt +from latentsync.utils.util import count_video_time, gather_video_paths_recursively +from tqdm import tqdm + + +def plot_histogram(data, fig_path): + # Create histogram + plt.hist(data, bins=30, edgecolor="black") + + # Add titles and labels + plt.title("Histogram of Data Distribution") + plt.xlabel("Video time") + plt.ylabel("Frequency") + + # Save plot as an image file + plt.savefig(fig_path) # Save as PNG file. You can also use 'histogram.jpg', 'histogram.pdf', etc. + + +def main(input_dir, fig_path): + video_paths = gather_video_paths_recursively(input_dir) + video_times = [] + for video_path in tqdm(video_paths): + video_times.append(count_video_time(video_path)) + plot_histogram(video_times, fig_path) + + +if __name__ == "__main__": + input_dir = "validation" + fig_path = "histogram.png" + + main(input_dir, fig_path) diff --git a/tools/download_youtube_videos.py b/tools/download_youtube_videos.py new file mode 100644 index 0000000..7cf4529 --- /dev/null +++ b/tools/download_youtube_videos.py @@ -0,0 +1,113 @@ +# Copyright (c) 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import subprocess +from concurrent.futures import ThreadPoolExecutor +import pandas as pd +from tqdm import tqdm + +""" +To use this python file, first install yt-dlp by: + +pip install yt-dlp==2024.5.27 +""" + + +def download_video(video_url, video_path): + get_video_channel_command = f"yt-dlp --print channel {video_url}" + result = subprocess.run(get_video_channel_command, shell=True, capture_output=True, text=True) + channel = result.stdout.strip() + if channel in unwanted_channels: + return + download_video_command = f"yt-dlp -f bestvideo+bestaudio --skip-unavailable-fragments --merge-output-format mp4 '{video_url}' --output '{video_path}' --external-downloader aria2c --external-downloader-args '-x 16 -k 1M'" + try: + subprocess.run(download_video_command, shell=True) # ignore_security_alert_wait_for_fix RCE + except KeyboardInterrupt: + print("Stopped") + exit() + except: + print(f"Error downloading video {video_url}") + + +def download_videos(num_workers, video_urls, video_paths): + with ThreadPoolExecutor(max_workers=num_workers) as executor: + executor.map(download_video, video_urls, video_paths) + + +def read_video_urls(csv_file_path: str, language_column, video_url_column): + video_urls = [] + print("Reading video urls...") + df = pd.read_csv(csv_file_path, sep=",") + for row in tqdm(df.itertuples(), total=len(df)): + language = getattr(row, language_column) + video_url = getattr(row, video_url_column) + if "clip" in video_url: + continue + video_urls.append((language, video_url)) + return video_urls + + +def extract_vid(video_url): + if "watch?v=" in video_url: # ignore_security_alert_wait_for_fix RCE + return video_url.split("watch?v=")[1][:11] + elif "shorts/" in video_url: + return video_url.split("shorts/")[1][:11] + elif "youtu.be/" in video_url: + return video_url.split("youtu.be/")[1][:11] + elif "&v=" in video_url: + return video_url.split("&v=")[1][:11] + else: + print(f"Invalid video url: {video_url}") + return None + + +def main(csv_file_path, language_column, video_url_column, output_dir, num_workers): + os.makedirs(output_dir, exist_ok=True) + all_video_urls = read_video_urls(csv_file_path, language_column, video_url_column) + + video_paths = [] + video_urls = [] + + print("Extracting vid...") + for language, video_url in tqdm(all_video_urls): + vid = extract_vid(video_url) + if vid is None: + continue + video_path = os.path.join(output_dir, language.lower(), f"vid_{vid}.mp4") + if os.path.isfile(video_path): + continue + os.makedirs(os.path.dirname(video_path), exist_ok=True) + video_paths.append(video_path) + video_urls.append(video_url) + + if len(video_paths) == 0: + print("All videos have been downloaded") + exit() + else: + print(f"Downloading {len(video_paths)} videos") + + download_videos(num_workers, video_urls, video_paths) + + +if __name__ == "__main__": + csv_file_path = "dcc.csv" + language_column = "video_language" + video_url_column = "video_link" + output_dir = "/mnt/bn/maliva-gen-ai-v2/chunyu.li/multilingual/raw" + num_workers = 50 + + unwanted_channels = ["TEDx Talks", "DaePyeong Mukbang", "Joeman"] + + main(csv_file_path, language_column, video_url_column, output_dir, num_workers) diff --git a/tools/move_files_recur.py b/tools/move_files_recur.py new file mode 100644 index 0000000..a66ba36 --- /dev/null +++ b/tools/move_files_recur.py @@ -0,0 +1,48 @@ +# Copyright (c) 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import shutil +from tqdm import tqdm + +paths = [] + + +def gather_paths(input_dir, output_dir): + os.makedirs(output_dir, exist_ok=True) + + for video in sorted(os.listdir(input_dir)): + if video.endswith(".mp4"): + video_input = os.path.join(input_dir, video) + video_output = os.path.join(output_dir, video) + if os.path.isfile(video_output): + continue + paths.append([video_input, output_dir]) + elif os.path.isdir(os.path.join(input_dir, video)): + gather_paths(os.path.join(input_dir, video), os.path.join(output_dir, video)) + + +def main(input_dir, output_dir): + print(f"Recursively gathering video paths of {input_dir} ...") + gather_paths(input_dir, output_dir) + + for video_input, output_dir in tqdm(paths): + shutil.move(video_input, output_dir) + + +if __name__ == "__main__": + input_dir = "/mnt/bn/maliva-gen-ai-v2/chunyu.li/multilingual_dcc" + output_dir = "/mnt/bn/maliva-gen-ai-v2/chunyu.li/multilingual" + + main(input_dir, output_dir) diff --git a/tools/occupy_gpu.py b/tools/occupy_gpu.py new file mode 100644 index 0000000..f101012 --- /dev/null +++ b/tools/occupy_gpu.py @@ -0,0 +1,60 @@ +# Copyright (c) 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +import os +import torch.multiprocessing as mp +import time + + +def check_mem(cuda_device): + devices_info = ( + os.popen('"/usr/bin/nvidia-smi" --query-gpu=memory.total,memory.used --format=csv,nounits,noheader') + .read() + .strip() + .split("\n") + ) + total, used = devices_info[int(cuda_device)].split(",") + return total, used + + +def loop(cuda_device): + cuda_i = torch.device(f"cuda:{cuda_device}") + total, used = check_mem(cuda_device) + total = int(total) + used = int(used) + max_mem = int(total * 0.9) + block_mem = max_mem - used + while True: + x = torch.rand(20, 512, 512, dtype=torch.float, device=cuda_i) + y = torch.rand(20, 512, 512, dtype=torch.float, device=cuda_i) + time.sleep(0.001) + x = torch.matmul(x, y) + + +def main(): + if torch.cuda.is_available(): + num_processes = torch.cuda.device_count() + processes = list() + for i in range(num_processes): + p = mp.Process(target=loop, args=(i,)) + p.start() + processes.append(p) + for p in processes: + p.join() + + +if __name__ == "__main__": + torch.multiprocessing.set_start_method("spawn") + main() diff --git a/tools/remove_outdated_files.py b/tools/remove_outdated_files.py new file mode 100644 index 0000000..00b4a60 --- /dev/null +++ b/tools/remove_outdated_files.py @@ -0,0 +1,34 @@ +# Copyright (c) 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import subprocess + + +def remove_outdated_files(input_dir, begin_date, end_date): + # Remove files from a specific time period + for subdir in os.listdir(input_dir): + if subdir >= begin_date and subdir <= end_date: + subdir_path = os.path.join(input_dir, subdir) + command = f"rm -rf {subdir_path}" + subprocess.run(command, shell=True) + print(f"Deleted: {subdir_path}") + + +if __name__ == "__main__": + input_dir = "/mnt/bn/video-datasets/output/syncnet" + begin_date = "train-2024_06_19-16:25:44" + end_date = "train-2024_08_03-07:39:58" + + remove_outdated_files(input_dir, begin_date, end_date) diff --git a/tools/write_fileslist.py b/tools/write_fileslist.py new file mode 100644 index 0000000..2132a24 --- /dev/null +++ b/tools/write_fileslist.py @@ -0,0 +1,45 @@ +# Copyright (c) 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from tqdm import tqdm +from latentsync.utils.util import gather_video_paths_recursively + + +def write_fileslist(fileslist_path): + with open(fileslist_path, "w") as _: + pass + + +def append_fileslist(fileslist_path, video_paths): + with open(fileslist_path, "a") as f: + for video_path in tqdm(video_paths): + f.write(f"{video_path}\n") + + +def process_input_dir(fileslist_path, input_dir): + print(f"Processing input dir: {input_dir}") + video_paths = gather_video_paths_recursively(input_dir) + append_fileslist(fileslist_path, video_paths) + + +if __name__ == "__main__": + fileslist_path = "/mnt/bn/maliva-gen-ai-v2/chunyu.li/fileslist/all_data_v6.txt" + + write_fileslist(fileslist_path) + process_input_dir(fileslist_path, "/mnt/bn/maliva-gen-ai-v2/chunyu.li/VoxCeleb2/high_visual_quality/train") + process_input_dir(fileslist_path, "/mnt/bn/maliva-gen-ai-v2/chunyu.li/HDTF/high_visual_quality/train") + process_input_dir(fileslist_path, "/mnt/bn/maliva-gen-ai-v2/chunyu.li/avatars/high_visual_quality/train") + process_input_dir(fileslist_path, "/mnt/bn/maliva-gen-ai-v2/chunyu.li/multilingual/high_visual_quality") + process_input_dir(fileslist_path, "/mnt/bn/maliva-gen-ai-v2/chunyu.li/celebv_text/high_visual_quality/train") + process_input_dir(fileslist_path, "/mnt/bn/maliva-gen-ai-v2/chunyu.li/youtube/high_visual_quality")