initial
This commit is contained in:
rainlizard
@@ -0,0 +1,188 @@
|
||||
import requests
|
||||
import json
|
||||
import time
|
||||
import os
|
||||
|
||||
# Constants
|
||||
OUTPUT_DIR = "tag_lists"
|
||||
CATEGORY_FILES = {
|
||||
0: "general.txt", # General tags
|
||||
1: "artist.txt", # Artist tags
|
||||
3: "copyright.txt", # Copyright tags
|
||||
4: "character.txt", # Character tags
|
||||
5: "meta.txt", # Meta tags
|
||||
'all': "all_tags.txt" # All tags combined
|
||||
}
|
||||
BASE_URL = "https://danbooru.donmai.us/tags.json"
|
||||
LIMIT = 200 # Maximum limit per request
|
||||
MIN_COUNT = 100 # Minimum post count
|
||||
|
||||
def ensure_output_dir():
|
||||
"""Create output directory if it doesn't exist"""
|
||||
if not os.path.exists(OUTPUT_DIR):
|
||||
os.makedirs(OUTPUT_DIR)
|
||||
|
||||
def get_output_path(filename):
|
||||
"""Get full path for output file"""
|
||||
return os.path.join(OUTPUT_DIR, filename)
|
||||
|
||||
def get_tags_with_count_over(min_count):
|
||||
"""
|
||||
Retrieve all tags that have a post count higher than min_count
|
||||
"""
|
||||
all_tags = []
|
||||
page = 1
|
||||
more_results = True
|
||||
|
||||
print(f"Fetching tags with count over {min_count}...")
|
||||
|
||||
while more_results:
|
||||
# Parameters for the API request
|
||||
params = {
|
||||
"search[post_count]": f">{min_count}",
|
||||
"search[order]": "count", # Order by post count (highest first)
|
||||
"limit": LIMIT,
|
||||
"page": page
|
||||
}
|
||||
|
||||
try:
|
||||
print(f"Fetching page {page}...")
|
||||
response = requests.get(BASE_URL, params=params)
|
||||
response.raise_for_status() # Raise an exception for HTTP errors
|
||||
|
||||
tags = response.json()
|
||||
|
||||
if not tags:
|
||||
more_results = False
|
||||
else:
|
||||
all_tags.extend(tags)
|
||||
page += 1
|
||||
|
||||
# Be nice to the API and avoid rate limiting
|
||||
time.sleep(1)
|
||||
|
||||
except requests.exceptions.RequestException as e:
|
||||
print(f"Error fetching data: {e}")
|
||||
break
|
||||
|
||||
return all_tags
|
||||
|
||||
def save_tags_to_files(tags, append=False):
|
||||
"""
|
||||
Save the tags to separate files based on their category and to all_tags.txt
|
||||
append: If True, append to files; if False, overwrite files
|
||||
"""
|
||||
try:
|
||||
# Group tags by category
|
||||
categorized_tags = {}
|
||||
for tag in tags:
|
||||
category = tag['category']
|
||||
if category in CATEGORY_FILES:
|
||||
if category not in categorized_tags:
|
||||
categorized_tags[category] = []
|
||||
categorized_tags[category].append(tag)
|
||||
|
||||
# Save each category to its respective file
|
||||
for category, category_tags in categorized_tags.items():
|
||||
filename = get_output_path(CATEGORY_FILES[category])
|
||||
mode = 'a' if append else 'w'
|
||||
|
||||
with open(filename, mode, encoding='utf-8') as f:
|
||||
# Sort by post_count in descending order
|
||||
sorted_tags = sorted(category_tags, key=lambda x: x['post_count'], reverse=True)
|
||||
for tag in sorted_tags:
|
||||
f.write(f"{tag['name']}, {tag['post_count']}\n")
|
||||
|
||||
print(f"Saved {len(category_tags)} {CATEGORY_FILES[category]} tags")
|
||||
|
||||
# Save to all_tags.txt
|
||||
filename = get_output_path(CATEGORY_FILES['all'])
|
||||
mode = 'a' if append else 'w'
|
||||
|
||||
with open(filename, mode, encoding='utf-8') as f:
|
||||
# Sort all tags by post_count
|
||||
sorted_tags = sorted(tags, key=lambda x: x['post_count'], reverse=True)
|
||||
for tag in sorted_tags:
|
||||
f.write(f"{tag['name']}, {tag['post_count']}\n")
|
||||
|
||||
print(f"Saved {len(tags)} tags to {CATEGORY_FILES['all']}")
|
||||
|
||||
except IOError as e:
|
||||
print(f"Error saving to files: {e}")
|
||||
|
||||
def get_last_saved_page():
|
||||
"""
|
||||
Determine the last page that was processed by counting lines in all_tags.txt
|
||||
"""
|
||||
try:
|
||||
filepath = get_output_path(CATEGORY_FILES['all'])
|
||||
try:
|
||||
with open(filepath, 'r', encoding='utf-8') as f:
|
||||
total_lines = sum(1 for _ in f)
|
||||
# Calculate the last completed page based on LIMIT tags per page
|
||||
last_page = (total_lines + LIMIT - 1) // LIMIT
|
||||
return last_page
|
||||
except FileNotFoundError:
|
||||
return 0
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error reading files: {e}")
|
||||
return 0
|
||||
|
||||
def clear_output_files():
|
||||
"""Clear all category files"""
|
||||
for filename in CATEGORY_FILES.values():
|
||||
filepath = get_output_path(filename)
|
||||
open(filepath, 'w').close()
|
||||
|
||||
def main():
|
||||
ensure_output_dir()
|
||||
|
||||
# Get the last processed page
|
||||
start_page = get_last_saved_page()
|
||||
page = start_page + 1 if start_page > 0 else 1
|
||||
more_results = True
|
||||
total_tags = 0
|
||||
|
||||
print(f"Fetching tags with count over {MIN_COUNT}...")
|
||||
print(f"Resuming from page {page}...")
|
||||
|
||||
# Clear files if starting from beginning
|
||||
if page == 1:
|
||||
clear_output_files()
|
||||
|
||||
while more_results:
|
||||
# Parameters for the API request
|
||||
params = {
|
||||
"search[post_count]": f">{MIN_COUNT}",
|
||||
"search[order]": "count", # Order by post count (highest first)
|
||||
"limit": LIMIT,
|
||||
"page": page
|
||||
}
|
||||
|
||||
try:
|
||||
print(f"Fetching page {page}...")
|
||||
response = requests.get(BASE_URL, params=params)
|
||||
response.raise_for_status()
|
||||
|
||||
tags = response.json()
|
||||
|
||||
if not tags:
|
||||
more_results = False
|
||||
else:
|
||||
# Save this batch immediately
|
||||
save_tags_to_files(tags, append=(page != 1))
|
||||
total_tags += len(tags)
|
||||
page += 1
|
||||
|
||||
# Be nice to the API and avoid rate limiting
|
||||
time.sleep(1)
|
||||
|
||||
except requests.exceptions.RequestException as e:
|
||||
print(f"Error fetching data: {e}")
|
||||
break
|
||||
|
||||
print(f"Finished! Found {total_tags} tags with count over {MIN_COUNT}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,334 @@
|
||||
highres, 5595469
|
||||
commentary_request, 3438490
|
||||
absurdres, 1920074
|
||||
commentary, 1498312
|
||||
bad_id, 1225107
|
||||
bad_pixiv_id, 959155
|
||||
photoshop_(medium), 706995
|
||||
translation_request, 571704
|
||||
translated, 567569
|
||||
english_commentary, 507613
|
||||
official_art, 329138
|
||||
bad_twitter_id, 208941
|
||||
commission, 198270
|
||||
chinese_commentary, 180133
|
||||
symbol-only_commentary, 121635
|
||||
artist_request, 113913
|
||||
md5_mismatch, 106427
|
||||
non-web_source, 103039
|
||||
lowres, 96381
|
||||
korean_commentary, 91989
|
||||
traditional_media, 90491
|
||||
animated, 89165
|
||||
scan, 86559
|
||||
copyright_request, 82563
|
||||
partial_commentary, 79950
|
||||
variant_set, 74247
|
||||
skeb_commission, 72060
|
||||
paid_reward_available, 71106
|
||||
game_cg, 70223
|
||||
revision, 61538
|
||||
character_request, 54547
|
||||
bad_link, 50063
|
||||
second-party_source, 48742
|
||||
video, 48506
|
||||
mixed-language_commentary, 46770
|
||||
duplicate, 37505
|
||||
check_translation, 37485
|
||||
spoilers, 36633
|
||||
third-party_source, 34705
|
||||
source_request, 29981
|
||||
resolution_mismatch, 28768
|
||||
pixiv_commission, 26732
|
||||
jpeg_artifacts, 26170
|
||||
hashtag-only_commentary, 25344
|
||||
protected_link, 25109
|
||||
third-party_edit, 24887
|
||||
pixel-perfect_duplicate, 24104
|
||||
large_variant_set, 23492
|
||||
animated_gif, 21828
|
||||
self-upload, 21364
|
||||
painting_(medium), 21170
|
||||
inactive_account, 20477
|
||||
partially_translated, 19987
|
||||
sound, 19621
|
||||
ugoira, 18070
|
||||
photo_(medium), 17425
|
||||
check_commentary, 16229
|
||||
paid_reward, 16021
|
||||
derivative_work, 16001
|
||||
tall_image, 15858
|
||||
watercolor_(medium), 15768
|
||||
anime_screenshot, 14943
|
||||
source_smaller, 14822
|
||||
textless_version, 14483
|
||||
unfinished, 14239
|
||||
looping_animation, 14023
|
||||
bad_tumblr_id, 13473
|
||||
marker_(medium), 13453
|
||||
painttool_sai_(medium), 13444
|
||||
imageboard_desourced, 13394
|
||||
dated_commentary, 12666
|
||||
source_larger, 12559
|
||||
bad_source, 12442
|
||||
graphite_(medium), 11017
|
||||
image_sample, 10115
|
||||
doujinshi, 9437
|
||||
promotional_art, 9283
|
||||
colorized, 9141
|
||||
cropped, 9043
|
||||
patreon_reward, 8365
|
||||
commissioner_upload, 8358
|
||||
clip_studio_paint_(medium), 8187
|
||||
ai-generated, 8060
|
||||
making-of_available, 8044
|
||||
artist_collaboration, 7985
|
||||
audible_music, 7588
|
||||
photo-referenced, 7515
|
||||
novel_illustration, 7470
|
||||
official_wallpaper, 6957
|
||||
audible_speech, 6848
|
||||
hard-translated, 6791
|
||||
bad_deviantart_id, 6669
|
||||
has_bad_revision, 6502
|
||||
concept_art, 6156
|
||||
off-topic, 6086
|
||||
colored_pencil_(medium), 6082
|
||||
resized, 5827
|
||||
corrupted_twitter_file, 5657
|
||||
pen_(medium), 5264
|
||||
bad_nicoseiga_id, 5030
|
||||
alt_text, 4990
|
||||
magazine_scan, 4856
|
||||
pixiv_sample, 4626
|
||||
procreate_(medium), 4575
|
||||
incredibly_absurdres, 4245
|
||||
clothing_request, 4048
|
||||
screenshot_redraw, 3717
|
||||
fanbox_reward, 3584
|
||||
scan_artifacts, 3581
|
||||
ai-assisted, 3518
|
||||
paint.net_(medium), 3325
|
||||
check_copyright, 3277
|
||||
artbook, 3265
|
||||
archived_source, 3119
|
||||
annotated, 3064
|
||||
stitched, 3055
|
||||
mixed_media, 3028
|
||||
flower_request, 2858
|
||||
blender_(medium), 2789
|
||||
has_censored_revision, 2715
|
||||
spanish_commentary, 2691
|
||||
bad_bcy_id, 2683
|
||||
tagme, 2641
|
||||
upscaled, 2573
|
||||
gender_request, 2535
|
||||
adversarial_noise, 2516
|
||||
downscaled, 2494
|
||||
millipen_(medium), 2470
|
||||
making-of, 2429
|
||||
key_visual, 2257
|
||||
wide_image, 2237
|
||||
english_audio, 2205
|
||||
production_art, 2202
|
||||
gimp_(medium), 2181
|
||||
romaji_commentary, 2169
|
||||
live2d, 2166
|
||||
nude_filter, 2129
|
||||
check_artist, 2123
|
||||
check_character, 2116
|
||||
alternate_language, 2109
|
||||
bad_drawr_id, 2108
|
||||
redrawn, 2096
|
||||
vector_trace, 2041
|
||||
cosplay_photo, 1996
|
||||
acrylic_paint_(medium), 1976
|
||||
japanese_audio, 1907
|
||||
has_downscaled_revision, 1783
|
||||
crease, 1763
|
||||
alpha_transparency, 1755
|
||||
russian_commentary, 1620
|
||||
food_request, 1512
|
||||
headwear_request, 1460
|
||||
gift_art, 1433
|
||||
weapon_request, 1422
|
||||
nib_pen_(medium), 1421
|
||||
unconventional_media, 1419
|
||||
lossy-lossless, 1376
|
||||
decensored, 1356
|
||||
third-party_watermark, 1307
|
||||
guest_art, 1239
|
||||
thai_commentary, 1235
|
||||
bad_nijie_id, 1224
|
||||
flash, 1202
|
||||
detexted, 1140
|
||||
ink_(medium), 1140
|
||||
engrish_commentary, 1136
|
||||
ibispaint_(medium), 1104
|
||||
end_card, 1096
|
||||
oil_painting_(medium), 1084
|
||||
commentary_typo, 1066
|
||||
merchandise_available, 1060
|
||||
fantia_reward, 1033
|
||||
album_cover_redraw, 999
|
||||
ballpoint_pen_(medium), 940
|
||||
aliasing, 895
|
||||
annotation_request, 882
|
||||
enty_reward, 835
|
||||
60+fps, 826
|
||||
bilingual_commentary, 817
|
||||
koikatsu_(medium), 817
|
||||
game_screenshot, 802
|
||||
tumblr_sample, 775
|
||||
watercolor_pencil_(medium), 773
|
||||
ruffle_compatible, 768
|
||||
art_study, 758
|
||||
calligraphy_brush_(medium), 758
|
||||
waifu2x, 756
|
||||
mikumikudance_(medium), 753
|
||||
twitter_sample, 748
|
||||
comic_panel_redraw, 716
|
||||
pastel_(medium), 696
|
||||
self-portrait, 694
|
||||
bad_lofter_id, 684
|
||||
source_mismatch, 679
|
||||
eyecatch, 673
|
||||
corrupted_metadata, 669
|
||||
hard-translated_(non-english), 654
|
||||
source_filmmaker_(medium), 650
|
||||
pdf_available, 650
|
||||
bad_yandere_id, 644
|
||||
music_video, 632
|
||||
object_request, 620
|
||||
extraction, 616
|
||||
bad_piapro_id, 614
|
||||
screenshot, 608
|
||||
reverse_translation, 604
|
||||
e-hentai_sample, 598
|
||||
indonesian_commentary, 590
|
||||
bad_weibo_id, 589
|
||||
artstation_sample, 586
|
||||
bmp-to-png_conversion, 583
|
||||
completion_time, 570
|
||||
spine_(medium), 558
|
||||
mousepad_(medium), 550
|
||||
deviantart_sample, 536
|
||||
costume_request, 530
|
||||
bad_artstation_id, 522
|
||||
parody_request, 514
|
||||
vehicle_request, 510
|
||||
papercraft_(medium), 501
|
||||
headshop, 497
|
||||
yandere_sample, 494
|
||||
bad_hentai-foundry_id, 477
|
||||
partially_annotated, 464
|
||||
adoptable, 462
|
||||
tegaki_draw_and_tweet, 443
|
||||
rotated, 440
|
||||
bad_instagram_id, 430
|
||||
french_commentary, 419
|
||||
cosplay_request, 417
|
||||
vgen_commission, 416
|
||||
paint_chat_(medium), 409
|
||||
exif_rotation, 403
|
||||
number-only_commentary, 400
|
||||
emofuri, 392
|
||||
game_model, 390
|
||||
figure_available, 366
|
||||
thumbnail_collage, 366
|
||||
psd_available, 360
|
||||
epilepsy_warning, 359
|
||||
medium_request, 355
|
||||
krita_(medium), 353
|
||||
ugoira_conversion, 348
|
||||
corrupted_file, 339
|
||||
microsoft_paint_(medium), 336
|
||||
portuguese_commentary, 327
|
||||
german_commentary, 303
|
||||
bad_fanbox_id, 301
|
||||
audible_internal_cumshot, 299
|
||||
bad_daum_id, 294
|
||||
hairstyle_request, 287
|
||||
contest_winner, 282
|
||||
gesture_request, 278
|
||||
color_ink_(medium), 274
|
||||
animated_png, 270
|
||||
countdown_illustration, 268
|
||||
check_clothing, 260
|
||||
zbrush_(medium), 259
|
||||
figure_(medium), 258
|
||||
gif_artifacts, 253
|
||||
reversed, 250
|
||||
bad_fantia_id, 246
|
||||
bird_request, 246
|
||||
app_filter, 244
|
||||
nicoseiga_sample, 244
|
||||
art_trade, 241
|
||||
gumroad_reward, 240
|
||||
stable_diffusion, 236
|
||||
plant_request, 220
|
||||
color_halftone, 212
|
||||
has_cropped_revision, 209
|
||||
custom_maid_3d_2, 207
|
||||
nai_diffusion, 206
|
||||
vietnamese_commentary, 206
|
||||
meme_request, 202
|
||||
flash_game, 201
|
||||
bad_aspect_ratio, 199
|
||||
volume_warning, 197
|
||||
medibang_paint_(medium), 197
|
||||
collaboration_request, 196
|
||||
flipnote_studio_(medium), 195
|
||||
roulette_animation, 194
|
||||
blueprint_(medium), 194
|
||||
check_gender, 191
|
||||
animal_request, 191
|
||||
pose_request, 187
|
||||
zip_available, 185
|
||||
fixed, 183
|
||||
bad_poipiku_id, 175
|
||||
neckwear_request, 165
|
||||
song_request, 163
|
||||
hair_ornament_request, 158
|
||||
lego_(medium), 158
|
||||
reference_request, 157
|
||||
character_counter_request, 156
|
||||
italian_commentary, 156
|
||||
gouache_(medium), 154
|
||||
bad_drawcrowd_id, 152
|
||||
style_request, 151
|
||||
naver_sample, 150
|
||||
recolored, 150
|
||||
bad_tinami_id, 149
|
||||
color_banding, 148
|
||||
bad_blogspot_id, 144
|
||||
thumbnail_surprise, 143
|
||||
bleed_through, 142
|
||||
non-repeating_animation, 142
|
||||
food_art_(medium), 140
|
||||
has_watermarked_revision, 138
|
||||
autodesk_3ds_max_(medium), 138
|
||||
metadata_pun, 135
|
||||
magazine_request, 128
|
||||
mecha_request, 125
|
||||
canvas_(medium), 124
|
||||
video_game_cover_redraw, 124
|
||||
bad_newgrounds_id, 120
|
||||
fanfiction_available, 119
|
||||
check_source, 115
|
||||
animal_ear_request, 114
|
||||
paper_cutout_(medium), 110
|
||||
model_kit_(medium), 110
|
||||
has_artifacted_revision, 109
|
||||
adobe_firefly_(medium), 109
|
||||
poorly_translated, 109
|
||||
ko-fi_commission, 108
|
||||
brush_(medium), 108
|
||||
multilingual_commentary, 106
|
||||
mojibake_commentary, 105
|
||||
instrument_request, 104
|
||||
reference_photo, 104
|
||||
maya_(medium), 102
|
||||
bad_fandom_id, 102
|
||||
crayon_(medium), 102
|
||||
cleaned, 101
|
||||
Reference in New Issue
Block a user