text-generation-webui/extensions/openai/moderations.py

import time
import numpy as np
from numpy.linalg import norm
from extensions.openai.embeddings import get_embeddings_model


moderations_disabled = False # return 0/false
category_embeddings = None
antonym_embeddings = None
categories = [ "sexual", "hate", "harassment", "self-harm", "sexual/minors", "hate/threatening", "violence/graphic", "self-harm/intent", "self-harm/instructions", "harassment/threatening", "violence" ]
flag_threshold = 0.5


def get_category_embeddings():
    global category_embeddings, categories
    if category_embeddings is None:
        embeddings = get_embeddings_model().encode(categories).tolist()
        category_embeddings = dict(zip(categories, embeddings))

    return category_embeddings


def cosine_similarity(a, b):
    return np.dot(a, b) / (norm(a) * norm(b))


# seems most openai like with all-mpnet-base-v2
def mod_score(a, b):
    return 2.0 * np.dot(a, b)


def moderations(input):
    global category_embeddings, categories, flag_threshold, moderations_disabled
    results = {
        "id": f"modr-{int(time.time()*1e9)}",
        "model": "text-moderation-001",
        "results": [],
    }

    embeddings_model = get_embeddings_model()
    if not embeddings_model or moderations_disabled:
        results['results'] = [{
            'categories': dict([ (C, False) for C in categories]),
            'category_scores': dict([ (C, 0.0) for C in categories]),
            'flagged': False,
        }]
        return results

    category_embeddings = get_category_embeddings()


    # input, string or array
    if isinstance(input, str):
        input = [input]

    for in_str in input:
        for ine in embeddings_model.encode([in_str]).tolist():
            category_scores = dict([ (C, mod_score(category_embeddings[C], ine)) for C in categories ])
            category_flags = dict([ (C, bool(category_scores[C] > flag_threshold)) for C in categories ])
            flagged = any(category_flags.values())

            results['results'].extend([{
                'flagged': flagged,
                'categories': category_flags,
                'category_scores': category_scores,
            }])

    print(results)

    return results