go/sapis/tokens/tokenizer.go

// ------------------------------------------------------------------------
//   Project atila
//   Active Thing (activething.com) git.activething.com/go
//
//   File name  token_builder.go
//   Created by DEV
//   Modified   18/02/2024
//
//   Copyright 2024 activething.com 
// ------------------------------------------------------------------------
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ------------------------------------------------------------------------

package tokens

import (
	"activething.com/go/gogo/core/opts"
)

type (

	Tokenizer struct {
		TokenizerOptions
		matcher MatcherBuilder
	}
)



func NewTokenizer(options ...opts.OptionFnc[TokenizerOptions]) *Tokenizer {
	return &Tokenizer{
		TokenizerOptions: *NewBuilderOptions(options...),
		matcher :         *NewMatcherBuilder(),
	}
}



func (b *Tokenizer) Tokenize (topic []byte, allows TokenKind)([]Token,TokenKind) {
	ln := len(topic)
	if ln < b.TopicMinLen || ln > b.TopicMaxLen {
		return []Token{*(NewInvalidToken(G.ErrInvalidTopicLen,topic))},TokenKindInvalid
	}

	ls := [topicMaxTokens]Token{}
	tt := TokenKindNone
	var m,c,x int

	for i:=0; i<ln; i++ {
		switch topic[i] {
		case b.MatchIni: m++
		case b.MatchEnd: m--
		case b.Sep:
			if m != 0 {
				continue
			}
			if c >= b.MaxTokens {
				ls[c]=*(NewInvalidToken(G.ErrInvalidTopicLen,topic[x:]))
				return ls[:c+1],tt | TokenKindInvalid
			}
			ls[c] = b.MakeToken(topic[x:i])
			tk := ls[c].Kind()
			if tk == TokenKindRelative {
				ls[c] = *NewInvalidToken(G.ErrInvalidToken,topic[x:i])
			} else if tk&allows != 0 {
				tt |= TokenKindInvalid
			} else {
				tt |= ls[c].Kind()
			}
			x = i + 1
			c++
		}
	}

	if m != 0 {
		ls[c] = *NewInvalidToken(G.ErrInvalidTopic,topic[x:])
	} else {
		ls[c] = b.MakeToken(topic[x:])
	}

	return ls[:c+1], tt | ls[c].Kind()
}




func (b *Tokenizer) MakeToken(source []byte) Token {
	ln := len(source)
	if ln < b.TokenMinLen  || ln > b.TokenMaxLen {
		return *(NewInvalidToken(G.ErrInvalidTokenLen,source))
	}
	if ln == 1 {
		switch source[0] {
		case b.Wildcard: return WildcardToken{}
		case b.Relative: return RelativeToken{}
		case b.MatchIni,
			b.MatchEnd: return *(NewInvalidToken(G.ErrInvalidTokenLen,source))
		}
	}

	if source[0] != b.MatchIni {
		return LiteralToken(source)
	}

	if source[ln-1] != b.MatchEnd || ln > b.TokenMinLen+2 {
		m,e := b.matcher.Build(source)
		if e != nil {
			return *NewInvalidToken(G.ErrInvalidMatcher.WithCause(e),source)
		}
		return *NewMatcherToken(m.Match,source)
	}

	return *NewInvalidToken(G.ErrInvalidMatcher,source)
}


func (b *Tokenizer) Split (topic []byte, allows TokenKind) ([][]byte,TokenKind) {
	var ms, ps, x int
	var tk, pk TokenKind
	ln := len(topic)
	ar := [topicMaxTokens][]byte{}

	for i := 0; i < ln; i++ {
		switch topic[i] {
		case b.MatchIni:
			ms++
		case b.MatchEnd:
			ms--
		case b.Sep:
			if ms == 0 {
				ar[ps] = topic[x:i]
				if pk = b.TokenKind(ar[ps]); pk&allows == 0 {
					return nil, TokenKindInvalid
				}
				tk |= pk
				x = i + 1
				ps++
			}
		}
	}
	ar[ps] = topic[x:]
	if pk = b.TokenKind(ar[ps]); pk&allows == 0 {
		return nil, TokenKindInvalid
	}
	tk |= pk

	return ar[:ps+1], tk
}



func (b *Tokenizer) TokenKind(Token []byte) TokenKind {
	ln := len(Token)
	if ln < b.TokenMinLen || ln > b.TokenMaxLen {
		return TokenKindInvalid
	}

	if ln == 1 {
		switch Token[0] {
		case b.Wildcard: return TokenKindWildcard
		case b.Relative: return TokenKindRelative
		case b.MatchIni,
			 b.MatchEnd: return TokenKindInvalid
		}
	}

	if Token[0] == b.MatchIni {
		if Token[ln-1] == b.MatchEnd {
			return TokenKindMatcher
		}
		return TokenKindInvalid
	}
	return TokenKindLiteral
}