← Назад

TikTokenTokenizer

public string ComponentId { get; }

Language: C#

Return Value: Guid.NewGuid().ToString()

public string ComponentType

Language: C#

public NeuralComponentState State { get; set; }

Language: C#

Return Value: NeuralComponentState.Uninitialized

public INeuralComponentInfo ComponentInfo { get; }

Language: C#

public int VocabularySize { get; }

Language: C#

public ITokenizerSpecialTokens SpecialTokens

Language: C#

public TikTokenTokenizer(string modelName = "cl100k_base", ITokenizerSpecialTokens? specialTokens = null, int vocabularySize = 0)

Language: C#

public static TikTokenTokenizer ForModel(string model)

Language: C#

Создает токенизатор для конкретной модели OpenAI.

public Task InitializeAsync(CancellationToken cancellationToken = default)

Language: C#

public Task ShutdownAsync()

Language: C#

public Task<int[]> EncodeAsync(string text, CancellationToken cancellationToken = default)

Language: C#

public Task<string> DecodeAsync(int[] tokenIds, CancellationToken cancellationToken = default)

Language: C#

public Task<int> CountTokensAsync(string text)

Language: C#

public async Task<int[][]> EncodeBatchAsync(IEnumerable<string> texts, CancellationToken cancellationToken = default)

Language: C#

public Task<float[]?> GetTokenEmbeddingAsync(int tokenId)

Language: C#

public void Dispose()

Language: C#