text-generation-webui/extensions/api/util.py

import time
import traceback
from threading import Thread
from typing import Callable, Optional

from modules import shared
from modules.chat import load_character_memoized


def build_parameters(body, chat=False):

    generate_params = {
        'max_new_tokens': int(body.get('max_new_tokens', body.get('max_length', 200))),
        'do_sample': bool(body.get('do_sample', True)),
        'temperature': float(body.get('temperature', 0.5)),
        'top_p': float(body.get('top_p', 1)),
        'typical_p': float(body.get('typical_p', body.get('typical', 1))),
        'epsilon_cutoff': float(body.get('epsilon_cutoff', 0)),
        'eta_cutoff': float(body.get('eta_cutoff', 0)),
        'repetition_penalty': float(body.get('repetition_penalty', body.get('rep_pen', 1.1))),
        'encoder_repetition_penalty': float(body.get('encoder_repetition_penalty', 1.0)),
        'top_k': int(body.get('top_k', 0)),
        'min_length': int(body.get('min_length', 0)),
        'no_repeat_ngram_size': int(body.get('no_repeat_ngram_size', 0)),
        'num_beams': int(body.get('num_beams', 1)),
        'penalty_alpha': float(body.get('penalty_alpha', 0)),
        'length_penalty': float(body.get('length_penalty', 1)),
        'early_stopping': bool(body.get('early_stopping', False)),
        'mirostat_mode': int(body.get('mirostat_mode', 0)),
        'mirostat_tau': float(body.get('mirostat_tau', 5)),
        'mirostat_eta': float(body.get('mirostat_eta', 0.1)),
        'seed': int(body.get('seed', -1)),
        'add_bos_token': bool(body.get('add_bos_token', True)),
        'truncation_length': int(body.get('truncation_length', body.get('max_context_length', 2048))),
        'ban_eos_token': bool(body.get('ban_eos_token', False)),
        'skip_special_tokens': bool(body.get('skip_special_tokens', True)),
        'custom_stopping_strings': '',  # leave this blank
        'stopping_strings': body.get('stopping_strings', []),
    }

    if chat:
        character = body.get('character')
        instruction_template = body.get('instruction_template')
        name1, name2, _, greeting, context, _ = load_character_memoized(character, str(body.get('your_name', shared.settings['name1'])), shared.settings['name2'], instruct=False)
        name1_instruct, name2_instruct, _, _, context_instruct, turn_template = load_character_memoized(instruction_template, '', '', instruct=True)
        generate_params.update({
            'stop_at_newline': bool(body.get('stop_at_newline', shared.settings['stop_at_newline'])),
            'chat_prompt_size': int(body.get('chat_prompt_size', shared.settings['chat_prompt_size'])),
            'chat_generation_attempts': int(body.get('chat_generation_attempts', shared.settings['chat_generation_attempts'])),
            'mode': str(body.get('mode', 'chat')),
            'name1': name1,
            'name2': name2,
            'context': context,
            'greeting': greeting,
            'name1_instruct': name1_instruct,
            'name2_instruct': name2_instruct,
            'context_instruct': context_instruct,
            'turn_template': turn_template,
            'chat-instruct_command': str(body.get('chat-instruct_command', shared.settings['chat-instruct_command'])),
        })

    return generate_params

def build_parameters_train(body):
    generate_params = {
        'lora_name': str(body.get('lora_name', 'lora')),
        'always_override': bool(body.get('always_override', False)),
        'save_steps': int(body.get('save_steps', 0)),
        'micro_batch_size': int(body.get('micro_batch_size', 4)),
        'batch_size': int(body.get('batch_size', 128)),
        'epochs': int(body.get('epochs', 3)),
        'learning_rate': float(body.get('learning_rate', 3e-4)),
        'lr_scheduler_type': str(body.get('lr_scheduler_type', 'linear')),
        'lora_rank': int(body.get('lora_rank', 32)),
        'lora_alpha': int(body.get('lora_alpha', 64)),
        'lora_dropout': float(body.get('lora_dropout', 0.05)),
        'cutoff_len': int(body.get('cutoff_len', 256)),
        'dataset': str(body.get('dataset', None)),
        'eval_dataset': str(body.get('eval_dataset', None)),
        'format': str(body.get('format', None)),
        'eval_steps': int(body.get('eval_steps', 100)),
        'raw_text_file': str(body.get('raw_text_file', None)),
        'overlap_len': int(body.get('overlap_len', 128)),
        'newline_favor_len': int(body.get('newline_favor_len', 128)),
        'higher_rank_limit': bool(body.get('higher_rank_limit', False)),
        'warmup_steps': int(body.get('warmup_steps', 100)),
        'optimizer': str(body.get('optimizer', 'adamw_torch')),
        'hard_cut_string': str(body.get('hard_cut_string', '\\n\\n\\n')),
        'train_only_after': str(body.get('train_only_after', ''))
    }

    return generate_params

def try_start_cloudflared(port: int, max_attempts: int = 3, on_start: Optional[Callable[[str], None]] = None):
    Thread(target=_start_cloudflared, args=[
           port, max_attempts, on_start], daemon=True).start()


def _start_cloudflared(port: int, max_attempts: int = 3, on_start: Optional[Callable[[str], None]] = None):
    try:
        from flask_cloudflared import _run_cloudflared
    except ImportError:
        print('You should install flask_cloudflared manually')
        raise Exception(
            'flask_cloudflared not installed. Make sure you installed the requirements.txt for this extension.')

    for _ in range(max_attempts):
        try:
            public_url = _run_cloudflared(port, port + 1)

            if on_start:
                on_start(public_url)

            return
        except Exception:
            traceback.print_exc()
            time.sleep(3)

        raise Exception('Could not start cloudflared.')