sd-webui-dataset-tag-editor/scripts/main.py

from typing import NamedTuple, Type, Dict, Any
from modules import shared, script_callbacks, scripts
from modules.shared import opts
import gradio as gr
import json
from pathlib import Path
from collections import namedtuple

import scripts.ui as ui
from scripts.dte_instance import dte_instance

# ================================================================
# General Callbacks
# ================================================================

CONFIG_PATH = Path(scripts.basedir(), 'config.json')


SortBy = dte_instance.SortBy
SortOrder = dte_instance.SortOrder

GeneralConfig = namedtuple('GeneralConfig', [
    'backup',
    'dataset_dir', 
    'caption_ext', 
    'load_recursive', 
    'load_caption_from_filename', 
    'replace_new_line', 
    'use_interrogator', 
    'use_interrogator_names',
    'use_custom_threshold_booru', 
    'custom_threshold_booru', 
    'use_custom_threshold_waifu', 
    'custom_threshold_waifu',
    'save_kohya_metadata',
    'meta_output_path',
    'meta_input_path',
    'meta_overwrite',
    'meta_save_as_caption',
    'meta_use_full_path'
    ])
FilterConfig = namedtuple('FilterConfig', ['sw_prefix', 'sw_suffix', 'sw_regex','sort_by', 'sort_order', 'logic'])
BatchEditConfig = namedtuple('BatchEditConfig', ['show_only_selected', 'prepend', 'use_regex', 'target', 'sw_prefix', 'sw_suffix', 'sw_regex', 'sory_by', 'sort_order', 'batch_sort_by', 'batch_sort_order', 'token_count'])
EditSelectedConfig = namedtuple('EditSelectedConfig', ['auto_copy', 'sort_on_save', 'warn_change_not_saved', 'use_interrogator_name', 'sort_by', 'sort_order'])
MoveDeleteConfig = namedtuple('MoveDeleteConfig', ['range', 'target', 'caption_ext', 'destination'])

CFG_GENERAL_DEFAULT = GeneralConfig(True, '', '.txt', False, True, False, 'No', [], False, 0.7, False, 0.35, False, '', '', True, False, False)
CFG_FILTER_P_DEFAULT = FilterConfig(False, False, False, SortBy.ALPHA.value, SortOrder.ASC.value, 'AND')
CFG_FILTER_N_DEFAULT = FilterConfig(False, False, False, SortBy.ALPHA.value, SortOrder.ASC.value, 'OR')
CFG_BATCH_EDIT_DEFAULT = BatchEditConfig(True, False, False, 'Only Selected Tags', False, False, False, SortBy.ALPHA.value, SortOrder.ASC.value, SortBy.ALPHA.value, SortOrder.ASC.value, 75)
CFG_EDIT_SELECTED_DEFAULT = EditSelectedConfig(False, False, False, '', SortBy.ALPHA.value, SortOrder.ASC.value)
CFG_MOVE_DELETE_DEFAULT = MoveDeleteConfig('Selected One', [], '.txt', '')

class Config:
    def __init__(self):
        self.config = dict()

    def load(self):
        if not CONFIG_PATH.is_file():
            self.config = dict()
            return
        try:
            self.config = json.loads(CONFIG_PATH.read_text('utf8'))
        except:
            print('[tag-editor] Error on loading config.json. Default settings will be loaded.')
            self.config = dict()
        else:
            print('[tag-editor] Settings has been read from config.json')

    def save(self):
        CONFIG_PATH.write_text(json.dumps(self.config, indent=4), 'utf8')
    
    def read(self, name: str):
        return self.config.get(name)

    def write(self, cfg: dict, name: str):
        self.config[name] = cfg

config = Config()

def write_general_config(*args):
    cfg = GeneralConfig(*args)
    config.write(cfg._asdict(), 'general')

def write_filter_config(*args):
    hlen = len(args) // 2
    cfg_p = FilterConfig(*args[:hlen])
    cfg_n = FilterConfig(*args[hlen:])
    config.write({'positive':cfg_p._asdict(), 'negative':cfg_n._asdict()}, 'filter')

def write_batch_edit_config(*args):
    cfg = BatchEditConfig(*args)
    config.write(cfg._asdict(), 'batch_edit')

def write_edit_selected_config(*args):
    cfg = EditSelectedConfig(*args)
    config.write(cfg._asdict(), 'edit_selected')

def write_move_delete_config(*args):
    cfg = MoveDeleteConfig(*args)
    config.write(cfg._asdict(), 'file_move_delete')

def read_config(name: str, config_type: Type, default: NamedTuple, compat_func = None):
    d = config.read(name)
    cfg = default
    if d:
        if compat_func: d = compat_func(d)
        d = cfg._asdict() | d
        d = {k:v for k,v in d.items() if k in cfg._asdict().keys()}
        cfg = config_type(**d)
    return cfg

def read_general_config():
    # for compatibility
    generalcfg_intterogator_names = [
        ('use_blip_to_prefill', 'BLIP'),
        ('use_git_to_prefill', 'GIT-large-COCO'),
        ('use_booru_to_prefill', 'DeepDanbooru'),
        ('use_waifu_to_prefill', 'wd-v1-4-vit-tagger')
    ]
    use_interrogator_names = []
    def compat_func(d: Dict[str, Any]):
        if 'use_interrogator_names' in d.keys():
            return d
        for cfg in generalcfg_intterogator_names:
            if d.get(cfg[0]):
                use_interrogator_names.append(cfg[1])
        d['use_interrogator_names'] = use_interrogator_names
        return d
    return read_config('general', GeneralConfig, CFG_GENERAL_DEFAULT, compat_func)

def read_filter_config():
    d = config.read('filter')
    d_p = d.get('positive') if d else None
    d_n = d.get('negative') if d else None
    cfg_p = CFG_FILTER_P_DEFAULT
    cfg_n = CFG_FILTER_N_DEFAULT
    if d_p:
        d_p = cfg_p._asdict() | d_p
        d_p = {k:v for k,v in d_p.items() if k in cfg_p._asdict().keys()}
        cfg_p = FilterConfig(**d_p)
    if d_n:
        d_n = cfg_n._asdict() | d_n
        d_n = {k:v for k,v in d_n.items() if k in cfg_n._asdict().keys()}
        cfg_n = FilterConfig(**d_n)
    return cfg_p, cfg_n

def read_batch_edit_config():
    return read_config('batch_edit', BatchEditConfig, CFG_BATCH_EDIT_DEFAULT)

def read_edit_selected_config():
    return read_config('edit_selected', EditSelectedConfig, CFG_EDIT_SELECTED_DEFAULT)

def read_move_delete_config():
    return read_config('file_move_delete', MoveDeleteConfig, CFG_MOVE_DELETE_DEFAULT)

# ================================================================
# General Callbacks for Updating UIs
# ================================================================

def get_filters():
    filters = [ui.filter_by_tags.tag_filter_ui.get_filter(), ui.filter_by_tags.tag_filter_ui_neg.get_filter()] + [ui.filter_by_selection.path_filter]
    return filters

def update_gallery():
    img_indices = ui.dte_instance.get_filtered_imgindices(filters=get_filters())
    total_image_num = len(ui.dte_instance.dataset)
    displayed_image_num = len(img_indices)
    ui.gallery_state.register_value('Displayed Images', f'{displayed_image_num} / {total_image_num} total')
    ui.gallery_state.register_value('Current Tag Filter', f"{ui.filter_by_tags.tag_filter_ui.get_filter()} {' AND ' if ui.filter_by_tags.tag_filter_ui.get_filter().tags and ui.filter_by_tags.tag_filter_ui_neg.get_filter().tags else ''} {ui.filter_by_tags.tag_filter_ui_neg.get_filter()}")
    ui.gallery_state.register_value('Current Selection Filter', f'{len(ui.filter_by_selection.path_filter.paths)} images')
    return [
        [str(i) for i in img_indices],
        1,
        -1,
        -1,
        -1,
        ui.gallery_state.get_current_gallery_txt()
        ]

def update_filter_and_gallery():
    return \
        [ui.filter_by_tags.tag_filter_ui.cbg_tags_update(), ui.filter_by_tags.tag_filter_ui_neg.cbg_tags_update()] +\
        update_gallery() +\
        ui.batch_edit_captions.get_common_tags(get_filters, ui.filter_by_tags) +\
        [', '.join(ui.filter_by_tags.tag_filter_ui.filter.tags)] +\
        [ui.batch_edit_captions.tag_select_ui_remove.cbg_tags_update()] +\
        ['', '']


# ================================================================
# Script Callbacks
# ================================================================

def on_ui_tabs():
    config.load()

    cfg_general = read_general_config()
    cfg_filter_p, cfg_filter_n = read_filter_config()
    cfg_batch_edit = read_batch_edit_config()
    cfg_edit_selected = read_edit_selected_config()
    cfg_file_move_delete = read_move_delete_config()

    with gr.Blocks(analytics_enabled=False) as dataset_tag_editor_interface:

        gr.HTML(value="""
        This extension works well with text captions in comma-separated style (such as the tags generated by DeepBooru interrogator).
        """)

        ui.toprow.create_ui(cfg_general)

        with gr.Accordion(label='Reload/Save Settings (config.json)', open=False):
            with gr.Row():
                btn_reload_config_file = gr.Button(value='Reload settings')
                btn_save_setting_as_default = gr.Button(value='Save current settings')
                btn_restore_default = gr.Button(value='Restore settings to default')

        with gr.Row().style(equal_height=False):
            with gr.Column():
                ui.load_dataset.create_ui(cfg_general)
                ui.dataset_gallery.create_ui(opts.dataset_editor_image_columns)
                ui.gallery_state.create_ui()

            with gr.Tab(label='Filter by Tags'):
                ui.filter_by_tags.create_ui(cfg_filter_p, cfg_filter_n, get_filters)
            
            with gr.Tab(label='Filter by Selection'):
                ui.filter_by_selection.create_ui(opts.dataset_editor_image_columns)

            with gr.Tab(label='Batch Edit Captions'):
                ui.batch_edit_captions.create_ui(cfg_batch_edit, get_filters)

            with gr.Tab(label='Edit Caption of Selected Image'):
                ui.edit_caption_of_selected_image.create_ui(cfg_edit_selected)

            with gr.Tab(label='Move or Delete Files'):
                ui.move_or_delete_files.create_ui(cfg_file_move_delete)

        #----------------------------------------------------------------
        # General

        components_general = [
            ui.toprow.cb_backup, ui.load_dataset.tb_img_directory, ui.load_dataset.tb_caption_file_ext, ui.load_dataset.cb_load_recursive,
            ui.load_dataset.cb_load_caption_from_filename, ui.load_dataset.cb_replace_new_line_with_comma, ui.load_dataset.rb_use_interrogator, ui.load_dataset.dd_intterogator_names,
            ui.load_dataset.cb_use_custom_threshold_booru, ui.load_dataset.sl_custom_threshold_booru, ui.load_dataset.cb_use_custom_threshold_waifu, ui.load_dataset.sl_custom_threshold_waifu,
            ui.toprow.cb_save_kohya_metadata, ui.toprow.tb_metadata_output, ui.toprow.tb_metadata_input, ui.toprow.cb_metadata_overwrite, ui.toprow.cb_metadata_as_caption, ui.toprow.cb_metadata_use_fullpath
        ]
        components_filter = \
            [ui.filter_by_tags.tag_filter_ui.cb_prefix, ui.filter_by_tags.tag_filter_ui.cb_suffix, ui.filter_by_tags.tag_filter_ui.cb_regex, ui.filter_by_tags.tag_filter_ui.rb_sort_by, ui.filter_by_tags.tag_filter_ui.rb_sort_order, ui.filter_by_tags.tag_filter_ui.rb_logic] +\
            [ui.filter_by_tags.tag_filter_ui_neg.cb_prefix, ui.filter_by_tags.tag_filter_ui_neg.cb_suffix, ui.filter_by_tags.tag_filter_ui_neg.cb_regex, ui.filter_by_tags.tag_filter_ui_neg.rb_sort_by, ui.filter_by_tags.tag_filter_ui_neg.rb_sort_order, ui.filter_by_tags.tag_filter_ui_neg.rb_logic]
        components_batch_edit = [
            ui.batch_edit_captions.cb_show_only_tags_selected, ui.batch_edit_captions.cb_prepend_tags, ui.batch_edit_captions.cb_use_regex,
            ui.batch_edit_captions.rb_sr_replace_target,
            ui.batch_edit_captions.tag_select_ui_remove.cb_prefix, ui.batch_edit_captions.tag_select_ui_remove.cb_suffix, ui.batch_edit_captions.tag_select_ui_remove.cb_regex,
            ui.batch_edit_captions.tag_select_ui_remove.rb_sort_by, ui.batch_edit_captions.tag_select_ui_remove.rb_sort_order,
            ui.batch_edit_captions.rb_sort_by, ui.batch_edit_captions.rb_sort_order,
            ui.batch_edit_captions.nb_token_count
        ]
        components_edit_selected = [
            ui.edit_caption_of_selected_image.cb_copy_caption_automatically, ui.edit_caption_of_selected_image.cb_sort_caption_on_save,
            ui.edit_caption_of_selected_image.cb_ask_save_when_caption_changed, ui.edit_caption_of_selected_image.dd_intterogator_names_si,
            ui.edit_caption_of_selected_image.rb_sort_by, ui.edit_caption_of_selected_image.rb_sort_order
        ]
        components_move_delete = [
            ui.move_or_delete_files.rb_move_or_delete_target_data, ui.move_or_delete_files.cbg_move_or_delete_target_file,
            ui.move_or_delete_files.tb_move_or_delete_caption_ext, ui.move_or_delete_files.tb_move_or_delete_destination_dir
        ]
        
        configurable_components = components_general + components_filter + components_batch_edit + components_edit_selected + components_move_delete

        def reload_config_file():
            config.load()
            p, n = read_filter_config()
            print('[tag-editor] Reload config.json')
            return read_general_config() + p + n + read_batch_edit_config() + read_edit_selected_config() + read_move_delete_config()

        btn_reload_config_file.click(
            fn=reload_config_file,
            outputs=configurable_components
        )

        def save_settings_callback(*a):
            p = 0
            def inc(v):
                nonlocal p
                p += v
                return p
            write_general_config(*a[p:inc(len(components_general))])
            write_filter_config(*a[p:inc(len(components_filter))])
            write_batch_edit_config(*a[p:inc(len(components_batch_edit))])
            write_edit_selected_config(*a[p:inc(len(components_edit_selected))])
            write_move_delete_config(*a[p:])
            config.save()
            print('[tag-editor] Current settings have been saved into config.json')

        btn_save_setting_as_default.click(
            fn=save_settings_callback,
            inputs=configurable_components
        )

        def restore_default_settings():
            write_general_config(*CFG_GENERAL_DEFAULT)
            write_filter_config(*CFG_FILTER_P_DEFAULT, *CFG_FILTER_N_DEFAULT)
            write_batch_edit_config(*CFG_BATCH_EDIT_DEFAULT)
            write_edit_selected_config(*CFG_EDIT_SELECTED_DEFAULT)
            write_move_delete_config(*CFG_MOVE_DELETE_DEFAULT)
            print('[tag-editor] Restore default settings')
            return CFG_GENERAL_DEFAULT + CFG_FILTER_P_DEFAULT + CFG_FILTER_N_DEFAULT + CFG_BATCH_EDIT_DEFAULT + CFG_EDIT_SELECTED_DEFAULT + CFG_MOVE_DELETE_DEFAULT
            

        btn_restore_default.click(
            fn=restore_default_settings,
            outputs=configurable_components
        )

        o_update_gallery = [ui.dataset_gallery.cbg_hidden_dataset_filter, ui.dataset_gallery.nb_hidden_dataset_filter_apply, ui.dataset_gallery.nb_hidden_image_index, ui.dataset_gallery.nb_hidden_image_index_prev, ui.edit_caption_of_selected_image.nb_hidden_image_index_save_or_not, ui.gallery_state.txt_gallery]

        o_update_filter_and_gallery = \
            [ui.filter_by_tags.tag_filter_ui.cbg_tags, ui.filter_by_tags.tag_filter_ui_neg.cbg_tags] + \
            o_update_gallery + \
            [ui.batch_edit_captions.tb_common_tags, ui.batch_edit_captions.tb_edit_tags] + \
            [ui.batch_edit_captions.tb_sr_selected_tags] +\
            [ui.batch_edit_captions.tag_select_ui_remove.cbg_tags] +\
            [ui.edit_caption_of_selected_image.tb_caption, ui.edit_caption_of_selected_image.tb_edit_caption]
        
        ui.toprow.set_callbacks(ui.load_dataset)
        ui.load_dataset.set_callbacks(o_update_filter_and_gallery,ui.toprow, ui.dataset_gallery, ui.filter_by_tags, ui.filter_by_selection, ui.batch_edit_captions, update_filter_and_gallery)
        ui.dataset_gallery.set_callbacks(ui.load_dataset, ui.gallery_state, get_filters)
        ui.gallery_state.set_callbacks(ui.dataset_gallery)
        ui.filter_by_tags.set_callbacks(o_update_gallery, o_update_filter_and_gallery, ui.batch_edit_captions, ui.move_or_delete_files, update_gallery, update_filter_and_gallery, get_filters)
        ui.filter_by_selection.set_callbacks(o_update_filter_and_gallery, ui.dataset_gallery, ui.filter_by_tags, get_filters, update_filter_and_gallery)
        ui.batch_edit_captions.set_callbacks(o_update_filter_and_gallery, ui.load_dataset, ui.filter_by_tags, get_filters, update_filter_and_gallery)
        ui.edit_caption_of_selected_image.set_callbacks(o_update_filter_and_gallery, ui.dataset_gallery, ui.load_dataset, get_filters, update_filter_and_gallery)
        ui.move_or_delete_files.set_callbacks(o_update_filter_and_gallery, ui.dataset_gallery, ui.filter_by_tags, ui.batch_edit_captions, ui.filter_by_selection, ui.edit_caption_of_selected_image, get_filters, update_filter_and_gallery)
    
    return [(dataset_tag_editor_interface, "Dataset Tag Editor", "dataset_tag_editor_interface")]


def on_ui_settings():
    section = ('dataset-tag-editor', "Dataset Tag Editor")
    shared.opts.add_option("dataset_editor_image_columns", shared.OptionInfo(6, "Number of columns on image gallery", section=section))
    shared.opts.add_option("dataset_editor_max_res", shared.OptionInfo(0, "Max resolution of temporary files", section=section))
    shared.opts.add_option("dataset_editor_use_temp_files", shared.OptionInfo(False, "Force image gallery to use temporary files", section=section))
    shared.opts.add_option("dataset_editor_use_raw_clip_token", shared.OptionInfo(True, "Use raw CLIP token to calculate token count (without emphasis or embeddings)", section=section))


script_callbacks.on_ui_settings(on_ui_settings)
script_callbacks.on_ui_tabs(on_ui_tabs)