from typing import List as PI_LIST, Tuple as PI_TUPLE import polars as PI_POLARS import numpy as PI_NUMPY from sklearn.feature_extraction.text import TfidfVectorizer as PI_TFIDFVECTORIZER from sklearn.preprocessing import normalize as PI_NORMALIZE def FC_BUILD_TFIDF_AND_COSINE_BEST_MATCH( PI_LI_CATEGORIES: PI_LIST[str], PI_LI_TEXTS: PI_LIST[str], PI_ST_NGRAM: str = '1,2', PI_I_MIN_DF: int = 1 ) -> PI_TUPLE[PI_NUMPY.ndarray, PI_NUMPY.ndarray]: if len(PI_LI_CATEGORIES) == 0: raise ValueError('PI_LI_CATEGORIES is empty') if len(PI_LI_TEXTS) == 0: return PI_NUMPY.array([]), PI_NUMPY.array([]) ZV_LI_NGRAM = PI_ST_NGRAM.split(',') ZV_I_NGRAM_MIN = int(ZV_LI_NGRAM[0]) ZV_I_NGRAM_MAX = int(ZV_LI_NGRAM[1]) ZV_LI_CORPUS = [] ZV_LI_CORPUS.extend(PI_LI_CATEGORIES) ZV_LI_CORPUS.extend(PI_LI_TEXTS) ZV_OB_VECT = PI_TFIDFVECTORIZER( ngram_range=( ZV_I_NGRAM_MIN, ZV_I_NGRAM_MAX ), min_df=PI_I_MIN_DF ) ZV_MX_TFIDF_ALL = ZV_OB_VECT.fit_transform(ZV_LI_CORPUS) ZV_I_CATEGORIES_N = len(PI_LI_CATEGORIES) ZV_MX_CATEGORIES = ZV_MX_TFIDF_ALL[ 0:ZV_I_CATEGORIES_N, : ] ZV_MX_TEXTS = ZV_MX_TFIDF_ALL[ ZV_I_CATEGORIES_N:, : ] ZV_MX_CATEGORIES_N = PI_NORMALIZE( ZV_MX_CATEGORIES, norm='l2', axis=1 ) ZV_MX_TEXTS_N = PI_NORMALIZE( ZV_MX_TEXTS, norm='l2', axis=1 ) ZV_MX_COS = ( ZV_MX_TEXTS_N @ ZV_MX_CATEGORIES_N.T ).toarray() ZV_AR_BEST_IDX = ZV_MX_COS.argmax(axis=1) ZV_AR_BEST_COS = ZV_MX_COS.max(axis=1) return ZV_AR_BEST_COS, ZV_AR_BEST_IDX