You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
68 lines
1.8 KiB
68 lines
1.8 KiB
// ------------------------------------------------------------------------
|
|
// Project atila
|
|
// Active Thing (activething.com) git.activething.com/go
|
|
//
|
|
// File name token_builder_options.go
|
|
// Created by DEV
|
|
// Modified 18/02/2024
|
|
//
|
|
// Copyright 2024 activething.com
|
|
// ------------------------------------------------------------------------
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
// ------------------------------------------------------------------------
|
|
|
|
package tokens
|
|
|
|
import "activething.com/go/gogo/core/opts"
|
|
|
|
type (
|
|
|
|
TokenizerOptions struct {
|
|
Sep byte
|
|
Wildcard byte
|
|
Relative byte
|
|
MatchIni byte
|
|
MatchEnd byte
|
|
|
|
TokenMinLen int
|
|
TokenMaxLen int
|
|
TopicMinLen int
|
|
TopicMaxLen int
|
|
MinTokens int
|
|
MaxTokens int
|
|
}
|
|
|
|
|
|
)
|
|
|
|
|
|
func NewBuilderOptions (options ... opts.OptionFnc[TokenizerOptions]) *TokenizerOptions {
|
|
op := opts.Apply(&TokenizerOptions{
|
|
Sep : tokenSep,
|
|
Wildcard : tokenWildcard,
|
|
Relative : tokenRelative,
|
|
MatchIni : tokenMatchIni,
|
|
MatchEnd : tokenMatchEnd,
|
|
|
|
TokenMinLen: tokenMinLen,
|
|
TokenMaxLen: tokenMaxLen,
|
|
TopicMinLen: topicMinLen,
|
|
TopicMaxLen: topicMaxLen,
|
|
MinTokens: topicMinTokens,
|
|
MaxTokens: topicMaxTokens,
|
|
}, options ...)
|
|
if op.MinTokens > topicMaxTokens {
|
|
op.MinTokens = topicMinTokens
|
|
}
|
|
return op
|
|
} |