2023-06-10 18:37:58 +08:00
|
|
|
|
using System;
|
|
|
|
|
using System.Collections.Generic;
|
|
|
|
|
|
2023-06-12 02:47:25 +08:00
|
|
|
|
namespace LLama.Common
|
2023-06-10 18:37:58 +08:00
|
|
|
|
{
|
|
|
|
|
using llama_token = Int32;
|
2023-06-21 15:21:27 +08:00
|
|
|
|
/// <summary>
|
|
|
|
|
/// The paramters used for inference.
|
|
|
|
|
/// </summary>
|
2023-06-12 02:47:25 +08:00
|
|
|
|
public class InferenceParams
|
2023-06-10 18:37:58 +08:00
|
|
|
|
{
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// number of tokens to keep from initial prompt
|
|
|
|
|
/// </summary>
|
2023-06-11 22:39:31 +08:00
|
|
|
|
public int TokensKeep { get; set; } = 0;
|
2023-06-10 18:37:58 +08:00
|
|
|
|
/// <summary>
|
|
|
|
|
/// how many new tokens to predict (n_predict), set to -1 to inifinitely generate response
|
|
|
|
|
/// until it complete.
|
|
|
|
|
/// </summary>
|
2023-06-11 22:39:31 +08:00
|
|
|
|
public int MaxTokens { get; set; } = -1;
|
2023-06-10 18:37:58 +08:00
|
|
|
|
/// <summary>
|
|
|
|
|
/// logit bias for specific tokens
|
|
|
|
|
/// </summary>
|
|
|
|
|
public Dictionary<llama_token, float>? LogitBias { get; set; } = null;
|
2023-06-11 05:44:21 +08:00
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// Sequences where the model will stop generating further tokens.
|
|
|
|
|
/// </summary>
|
2023-06-11 09:13:30 +08:00
|
|
|
|
public IEnumerable<string> AntiPrompts { get; set; } = Array.Empty<string>();
|
2023-06-10 18:37:58 +08:00
|
|
|
|
/// <summary>
|
|
|
|
|
/// path to file for saving/loading model eval state
|
|
|
|
|
/// </summary>
|
|
|
|
|
public string PathSession { get; set; } = string.Empty;
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// string to suffix user inputs with
|
|
|
|
|
/// </summary>
|
|
|
|
|
public string InputSuffix { get; set; } = string.Empty;
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// string to prefix user inputs with
|
|
|
|
|
/// </summary>
|
|
|
|
|
public string InputPrefix { get; set; } = string.Empty;
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// 0 or lower to use vocab size
|
|
|
|
|
/// </summary>
|
|
|
|
|
public int TopK { get; set; } = 40;
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// 1.0 = disabled
|
|
|
|
|
/// </summary>
|
|
|
|
|
public float TopP { get; set; } = 0.95f;
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// 1.0 = disabled
|
|
|
|
|
/// </summary>
|
|
|
|
|
public float TfsZ { get; set; } = 1.0f;
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// 1.0 = disabled
|
|
|
|
|
/// </summary>
|
|
|
|
|
public float TypicalP { get; set; } = 1.0f;
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// 1.0 = disabled
|
|
|
|
|
/// </summary>
|
|
|
|
|
public float Temperature { get; set; } = 0.8f;
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// 1.0 = disabled
|
|
|
|
|
/// </summary>
|
|
|
|
|
public float RepeatPenalty { get; set; } = 1.1f;
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// last n tokens to penalize (0 = disable penalty, -1 = context size) (repeat_last_n)
|
|
|
|
|
/// </summary>
|
|
|
|
|
public int RepeatLastTokensCount { get; set; } = 64;
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// frequency penalty coefficient
|
|
|
|
|
/// 0.0 = disabled
|
|
|
|
|
/// </summary>
|
|
|
|
|
public float FrequencyPenalty { get; set; } = .0f;
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// presence penalty coefficient
|
|
|
|
|
/// 0.0 = disabled
|
|
|
|
|
/// </summary>
|
|
|
|
|
public float PresencePenalty { get; set; } = .0f;
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// Mirostat uses tokens instead of words.
|
|
|
|
|
/// algorithm described in the paper https://arxiv.org/abs/2007.14966.
|
|
|
|
|
/// 0 = disabled, 1 = mirostat, 2 = mirostat 2.0
|
|
|
|
|
/// </summary>
|
2023-07-28 06:11:25 +08:00
|
|
|
|
public MirostatType Mirostat { get; set; } = MirostatType.Disable;
|
2023-06-10 18:37:58 +08:00
|
|
|
|
/// <summary>
|
|
|
|
|
/// target entropy
|
|
|
|
|
/// </summary>
|
|
|
|
|
public float MirostatTau { get; set; } = 5.0f;
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// learning rate
|
|
|
|
|
/// </summary>
|
|
|
|
|
public float MirostatEta { get; set; } = 0.1f;
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// consider newlines as a repeatable token (penalize_nl)
|
|
|
|
|
/// </summary>
|
|
|
|
|
public bool PenalizeNL { get; set; } = true;
|
|
|
|
|
}
|
|
|
|
|
|
2023-07-28 06:11:25 +08:00
|
|
|
|
/// <summary>
|
|
|
|
|
/// Type of "mirostat" sampling to use.
|
|
|
|
|
/// https://github.com/basusourya/mirostat
|
|
|
|
|
/// </summary>
|
|
|
|
|
public enum MirostatType
|
2023-06-10 18:37:58 +08:00
|
|
|
|
{
|
2023-07-28 06:11:25 +08:00
|
|
|
|
/// <summary>
|
|
|
|
|
/// Disable Mirostat sampling
|
|
|
|
|
/// </summary>
|
2023-06-12 02:47:25 +08:00
|
|
|
|
Disable = 0,
|
2023-07-28 06:11:25 +08:00
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// Original mirostat algorithm
|
|
|
|
|
/// </summary>
|
|
|
|
|
Mirostat = 1,
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// Mirostat 2.0 algorithm
|
|
|
|
|
/// </summary>
|
|
|
|
|
Mirostat2 = 2
|
2023-06-10 18:37:58 +08:00
|
|
|
|
}
|
|
|
|
|
}
|