active/core/btes/tokens/atokenizer.go

// ------------------------------------------------------------------------
//   Project active2
//   Active Thing (activething.com) git.activething.com/go
//
//   File name  atokenizer.go
//   Created by DEV
//   Modified   24/04/2024
//
//   Copyright 2024 activething.com 
// ------------------------------------------------------------------------
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ------------------------------------------------------------------------

package tokens






const  	(

	topicMinLen    	int = 1
	topicMaxLen    	int = 255
	topicMaxTokens 	int = 32
	topicMinTokens 	int = 1

	tokenMinLen 	int = 1
	tokenMaxLen 	int = 256

	//
	// Token chars
	//
	tokenSep      	byte = '.'
	tokenWildcard 	byte = '*'
	tokenRelative 	byte = '+'
	tokenMatchini 	byte = '{'
	tokenMatchend 	byte = '}'

)


type 	(




	// TokenizerCore
	// Struct type
	// Implements -> Tokenizer
	//
	TokenizerCore 		struct {
		TokenSep      	byte
		TokenWildcard 	byte
		TokenRelative 	byte
		TokenMathcIni 	byte
		TokenMatchEnd 	byte
		MinLenToken   	int
		MaxLenToken   	int
		MinLenTopic   	int
		MaxLenTopic   	int
		MinTokens     	int
		MaxTokens     	int

		MatchBuilder 	mchs.Matchers
	}


	// TokenizerCoreFnc
	// Function type
	//
	//
	TokenizerCoreFnc 	func (tkz *TokenizerCore)

)


// NewTokenizerCore
// Function constructor -> TokenizerCore
//
//
func NewTokenizerCore(ops ...TokenizerCoreFnc)(*TokenizerCore){
	tk := &TokenizerCore{
		TokenSep 		:  tokenSep,
		TokenWildcard 	: tokenWildcard,
		TokenRelative 	: tokenRelative,
		TokenMathcIni 	: tokenMatchini,
		TokenMatchEnd 	: tokenMatchend,
		MinLenToken   	:   tokenMinLen,
		MaxLenToken   	:   tokenMaxLen,
		MinLenTopic   	:   topicMinLen,
		MaxLenTopic   	:   topicMaxLen,
		MinTokens     	: topicMinTokens,
		MaxTokens     	: topicMaxTokens,
	}

	for _,o := range ops {
		o(tk) }

	if tk.MaxTokens > topicMaxTokens {
		tk.MaxTokens = topicMaxTokens
	}

	if tk.MatchBuilder == nil {
		tk.MatchBuilder = mchs.MatchersNoop{} }

	return tk
}


// NewTopic
// Function member -> TokenizerCore
//
//
func (t TokenizerCore) NewTopic (tpc []byte, alw Kind) *Topic { return NewTopic(t,tpc,alw) }


// Tokenize
// Function member -> TokenizerCore
// Implements -> Tokenizer
//
//
func (t TokenizerCore) Tokenize(tpc []byte, alw Kind) (Tokens, Kind){
	ln := len(tpc)
	if ln < t.MinLenTopic || ln > t.MaxLenTopic {
		return []Token{*(NewInvalidToken(G.ErrInvalidTopicLen,tpc)) }, TokenKindInvalid
	}

	tl := [topicMaxTokens]Token{}
	tt := TokenKindNone
	m,c,x := 0,0,0

	for i:=0; i<ln; i++ {
		switch tpc[i] {
		case t.TokenMathcIni : m++
		case t.TokenMatchEnd : m--
		case t.TokenSep:
			if m != 0 {
				continue }
			if c >= t.MaxTokens {
				tl[c] = *(NewInvalidToken(G.ErrInvalidTopicLen,tpc[x:]))
				return tl[:c+1], tt | TokenKindInvalid
			}
			tl[c] = t.NewToken(tpc[x:i])
			tk := tl[c].Kind()
			if tk == TokenKindRelative {
				tl[c]= NewInvalidToken(G.ErrInvalidToken,tpc[x:i])
			} else if tk & alw == 0 {
				tt |= TokenKindInvalid
			}
			tt |= tl[c].Kind()
			x = i + 1
			c++
		}
	}

	if m != 0 {
		tl[c] = *(NewInvalidToken(G.ErrInvalidTopic,tpc[x:]))
	} else {
		tl[c] = t.NewToken(tpc[x:])
	}

	return tl[:c+1], tt | tl[c].Kind()

}


// Split
// Function member -> TokenizerCore
//
//
func (t TokenizerCore) Split (tpc []byte, alw Kind)([][]byte, Kind){
	var ms, ps,x int
	var tk, pk Kind
	ln := len(tpc)
	ar := [topicMaxTokens][]byte{}

	for i:=0; i< ln; i++ {
		switch tpc[i] {
		case t.TokenMathcIni  : ms++
		case t.TokenMatchEnd  : ms--
		case t.TokenSep:
			if ms == 0 {
				ar[ps] = tpc[x:i]
				if pk = t.Kind(ar[ps]); pk & alw == 0 {
					return nil, TokenKindInvalid
				}
				tk|= pk
				x = i + 1
				ps++
			}
		}
	}
	ar[ps] = tpc[x:]
	if pk = t.Kind(ar[ps]); pk& alw == 0 {
		return nil, TokenKindInvalid
	}
	tk |= pk

	return ar[:ps+1],tk
}




// NewToken
// Function member -> TokenizerCore
//
//
func (t TokenizerCore) NewToken(Token []byte)(Token){
	ln := len(Token)
	if ln < t.MinLenToken || ln > t.MaxLenToken {
		return InvalidToken{ error: nil, source: Token } }

	if ln == 1 {
		switch Token[0] {
		case t.TokenWildcard : return WildcardToken{}
		case t.TokenRelative : return RelativeToken{}
		case t.TokenMathcIni,
			t.TokenMatchEnd : return *(NewInvalidToken(G.ErrInvalidTopicLen,Token)) }
	}

	if Token[0] != t.TokenMathcIni {
		return LiteralToken(Token) }

	if Token[ln-1] == t.TokenMatchEnd && ln > t.MinLenToken+ 2 {
		m,e := t.MatchBuilder.Build(Token)
		if e == nil {
			return *(NewMatcherToken(m.Match,Token)) }
		return *(NewInvalidToken(e,Token))
	}

	return *(NewInvalidToken(mchs.Errs.ErrInvalidMatcher,Token))
}




// Kind
// Function member -> TokenizerCore
//
func (t TokenizerCore) Kind (Token []byte)(Kind){
	ln := len(Token)
	if ln < t.MinLenToken || ln > t.MaxLenToken {
		return TokenKindInvalid
	}

	if ln == 1 {
		switch Token[0] {
		case t.TokenWildcard : return TokenWildcard
		case t.TokenRelative : return TokenRelative
		case t.TokenMathcIni,
			t.TokenMatchEnd : return TokenKindInvalid
		}
	}

	if Token[0] == t.TokenMathcIni {
		if Token[ln-1] == t.TokenMatchEnd {
			return TokenMatcher }
		return TokenKindInvalid
	}
	return TokenLiteral
}