tokenizer.hpp

Go to the documentation of this file.

//==================================================================================================
/// \file
/// This header-file is part of module \alib_strings of the \aliblong.
///
/// Copyright 2013-2026 A-Worx GmbH, Germany.
/// Published under #"mainpage_license".
//==================================================================================================
ALIB_EXPORT namespace alib {  namespace strings {
 
/// This sub-namespace provides some utility classes which are related
/// to string classes found in namespace #"alib::strings".
namespace util  {
 
//==================================================================================================
/// This class operates on strings which contains data separated by a delimiter character.
/// It identifies the substrings between the delimiters as \e tokens of type #"^Substring".
/// After an instance of this class is constructed, three methods are available:
/// - #".HasNext:" Indicates if there are further tokens available.
/// - #".Next:" Sets field #".Actual" (which is of type #"%^Substring") to reference the next token
///   and returns it.<br>
///   With each call to #".Next", a different delimiter can be provided, which then serves as the
///   delimiter for this and subsequent tokens.<br>
///   The returned token by default will be trimmed according to the current trimable characters.
/// - #"Rest:"
///   Like #".Next", however returns the complete remaining region without searching for
///   further delimiters (and tokens).<br>
///   After this method was invoked, #"HasNext()" will return \c false.
///
/// After a token was retrieved, it might be modified using the interface of class #"^Substring"
/// as the tokenizer does not rely on the bounds of the current token when receiving the next.
///
/// Objects of this class can be reused by freshly initializing them by using method #"Set".
/// Furthermore, even the field #".Rest" is allowed to be changed using the interface of
/// #"%^Substring" if it seems appropriate. The effect is the same as if the method #".Set" was
/// invoked to apply a different source string.
///
/// <b>Sample code</b>:<br>
/// The following code sample shows how to tokenize a string:
///
/// \snippet "DOX_TOKENIZER.cpp"     DOX_TOKENIZER
///
/// The output will be:
///
/// \verbinclude "DOX_TOKENIZER.txt"
///
/// @tparam TChar    The character type. Implementations for \c nchar and \c wchar are provided
///                  with type definitions #"alib::TokenizerN" and
///                  #"alib::TokenizerW".
//==================================================================================================
template<typename TChar>
class TTokenizer {
  //################################################################################################
  // Public fields
  //################################################################################################
  public:
    /// A #"^Substring" that represents the part of the underlying data that has not been
    /// tokenized, yet.
    /// It is allowed to manipulate this public field, which has a similar effect as
    /// using method #".Set".<br>
    TSubstring<TChar>               Rest;
 
    /// The actual token, which is returned with every invocation of #".Next" or #".Rest".
    /// It is allowed to manipulate this field any time.<br>
    TSubstring<TChar>               Actual;
 
    /// The white spaces characters used to trim the tokens.
    /// Defaults to #"alib::DEFAULT_WHITESPACES"
    TLocalString<TChar, 8>          TrimChars;
 
 
  //################################################################################################
  // Internal fields
  //################################################################################################
  protected:
    /// The most recently set delimiter used by default for the next token extraction.
    TChar           delim;
 
    /// If \c true, empty tokens are omitted.
    bool            skipEmpty;
 
 
  //################################################################################################
  //  Constructors/Destructor
  //################################################################################################
  public:
    /// Constructs an empty tokenizer. To initialize, method #".Set" needs to be invoked.
    TTokenizer()                                                                                  {}
 
    /// Constructs a tokenizer to work on a given string.
    ///
    /// @param  src             The string to be tokenized.
    /// @param  delimiter       The delimiter that separates the tokens. Can be changed with
    ///                         every next token.
    /// @param  skipEmptyTokens If \c true, empty tokens are omitted.
    ///                         Optional and defaults to \c false.
    TTokenizer( const TString<TChar>& src, TChar delimiter, bool skipEmptyTokens= false )
    : Rest  (src)
    , Actual(nullptr)
    , TrimChars( CStringConstantsTraits<TChar>::DefaultWhitespaces() )
    , delim(delimiter)
    , skipEmpty(skipEmptyTokens)                                                                  {}
 
  //################################################################################################
  //  Interface
  //################################################################################################
  public:
    /// Resets a tokenizer to work on a given string.
    ///
    /// @param  src             The string to be tokenized
    /// @param  delimiter       The delimiter that separates the tokens. Can be changed with
    ///                         every next token.
    /// @param  skipEmptyTokens If \c true, empty tokens are omitted.
    ///                         Optional and defaults to \c false.
    void Set( const TString<TChar>& src, TChar delimiter, bool skipEmptyTokens= false ) {
        Actual          = nullptr;
        Rest            = src;
        this->delim     = delimiter;
        this->skipEmpty = skipEmptyTokens;
    }
 
    /// Returns the next token, which is afterward also available through the field #".Actual".
    /// If no further token was available, the returned #"^Substring" will be \e nulled.
    /// (see #"TString::IsNull;String::IsNull").
    /// To prevent this, the availability of the next token should be checked with the method
    /// #".HasNext".
    ///
    /// For clarification, see the explanation and sample code in this classes documentation.
    ///
    ///  @param trimming  Determines if the token is trimmed in respect to the white space
    ///                   characters defined in field #".TrimChars".
    ///                   Defaults to #"Whitespaces::Trim;2".
    ///  @param newDelim  The delimiter separates the tokens. Defaults to 0, which keeps the
    ///                   current delimiter intact.
    ///                   A new delimiter can be provided for every next token.
    /// @return The next token as #"%^Substring". A nulled string is if no next token was
    ///         available.
    ALIB_DLL
    TSubstring<TChar>&  Next( lang::Whitespaces trimming= lang::Whitespaces::Trim,
                              TChar             newDelim= '\0' );
 
    /// Returns the currently remaining string (without searching for further delimiter
    /// characters).
    /// After this call #".HasNext" will return \c false and #".Next" will return a \e nulled
    /// Substring.
    ///  @param trimming  Determines if the token is trimmed in respect to the white space
    ///                   characters defined in field #".TrimChars".
    ///                   Defaults to #"Whitespaces::Trim;2".
    /// @return The rest of the original source string, which was not returned by #".Next", yet.
    TSubstring<TChar>&  GetRest( lang::Whitespaces trimming= lang::Whitespaces::Trim ) {
        // set start, end and end of tokenizer
        Actual=  Rest;
        Rest  =  nullptr;
        if ( trimming == lang::Whitespaces::Trim )
            Actual.Trim( TrimChars );
        return Actual;
    }
 
    /// If this returns \c true, a call to #".Next" will be successful and will return a
    /// #"%^Substring" which is not \e nulled.
    /// @return \c true if a next token is available.
    bool        HasNext()        { return Rest.IsNotNull() && ( !skipEmpty || Rest.IsNotEmpty() ); }
 
}; // class Tokenizer
 
 
extern template ALIB_DLL TSubstring<nchar>& TTokenizer<nchar>::Next( lang::Whitespaces, nchar );
extern template ALIB_DLL TSubstring<wchar>& TTokenizer<wchar>::Next( lang::Whitespaces, wchar );
 
}} // namespace alib[::strings::util]
 
/// Type alias in namespace #"%alib".
using     Tokenizer=     strings::util::TTokenizer<character>;
 
/// Type alias in namespace #"%alib".
using     TokenizerN=    strings::util::TTokenizer<nchar>;
 
/// Type alias in namespace #"%alib".
using     TokenizerW=    strings::util::TTokenizer<wchar>;
 
 
 
} // namespace [alib]