/// Analyzes test for the how frequently one word follows another.
  /// When given a seed word, automatically generates random new text
  /// based on the previously analyzed frequence.
  TShannonizer = class
  private
    FProbabilities: TTokenProbability;
    FFrequencies: TTokenFrequencies;
  public
    constructor Create;
    destructor Destroy; override;
    /// Converts and removes problematic characters and whitespace
    class function CleanInputText(const Text: string): string; static;
    /// Extracts the last token from the input text for use in generating new text
    class function ExtractLastToken(const Text: string): string; static;
    /// Tokenizes the input text and creates a frequency analysis
    /// (Automatically calls CleanInputText)
    procedure AnalyzeText(const Text: string);
    /// Uses the last token from StartWord to seed new generated text.
    /// Stops after Length tokens (words) or if a new word cannot be found.
    /// Uses the RandSeed to randomize the output.
    function GenerateText(const StartWord: string; Length: Integer): string;
    /// Loads a previously saved analysis from file
    procedure LoadFromFile(const FileName: string);
    /// Saves the current analysis to file
    procedure SaveToFile(const FileName: string);
    /// Serializes the current analysis to a string
    function SaveToString: string;
    /// Serializes the current analysis to a steam (used by SaveToFile and SaveToString)
    procedure SaveToStream(Stream: TStream);
    /// Loads analysis data from a string
    procedure LoadFromString(const DataString: string);
    /// Loads analysis data from a stream (used by LoadFromStream and LoadFromFile)
    procedure LoadFromStream(Stream: TStream);
    /// Clears all token and frequency infromation.
    /// Multiple AnalyzeText or Load calls will append to the existing analysis.
    procedure Clear;
  end;