停止
stop 篩選器會從分詞後的文字中移除指定的停用詞，有助於排除常見且意義較少的詞彙。您可以透過stop_words 參數來設定停用詞清單。
設定
stop 篩選器可透過stop_words 參數以內嵌方式接受停用詞清單，或透過stop_words_file 參數從已註冊的檔案資源中讀取停用詞清單。
內嵌停用詞清單
若要使用內嵌清單來運作stop 篩選器，請在篩選器配置中指定"type": "stop" ，並搭配stop_words 參數來提供停用詞清單。
analyzer_params = {
"tokenizer": "standard",
"filter":[{
"type": "stop", # Specifies the filter type as stop
"stop_words": ["of", "to", "_english_"], # Defines custom stop words and includes the English stop word list
}],
}
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
Map<String, Object> analyzerParams = new HashMap<>();
analyzerParams.put("tokenizer", "standard");
analyzerParams.put("filter",
Collections.singletonList(
new HashMap<String, Object>() {{
put("type", "stop");
put("stop_words", Arrays.asList("of", "to", "_english_"));
}}
)
);
analyzerParams := map[string]any{"tokenizer": "standard",
"filter": []any{map[string]any{
"type": "stop",
"stop_words": []string{"of", "to", "_english_"},
}}}
use milvus::v2::prelude::*;
use serde_json::json;
let analyzer_params = json!({
"tokenizer": "standard",
"filter": [{
"type": "stop",
"stop_words": ["of", "to", "_english_"]
}]
});
nlohmann::json analyzer_params = {
{"tokenizer", "standard"},
{"filter", {
{{"type", "stop"}, {"stop_words", {"of", "to", "_english_"}}}
}}
};
const analyzer_params = {
tokenizer: "standard",
filter: [{
type: "stop", // Specifies the filter type as stop
stop_words: ["of", "to", "_english_"], // Custom stop words + built-in English list
}],
};
using System.Collections.Generic;
Dictionary<string, object> analyzerParams = new()
{
["tokenizer"] = "standard",
["filter"] = new List<object>
{
new Dictionary<string, object>
{
["type"] = "stop", // Specifies the filter type as stop
["stop_words"] = new List<object> { "of", "to", "_english_" } // Defines custom stop words and includes the English stop word list
}
}
};
# restful
analyzerParams='{
"tokenizer": "standard",
"filter": [
{
"type": "stop",
"stop_words": [
"of",
"to",
"_english_"
]
}
]
}'
stop 過濾器接受以下可配置參數。
參數 |
說明 |
|---|---|
|
要從分詞過程中移除的詞彙清單。預設情況下，此篩選器會使用內建的
|
stop 篩選器是針對分詞器所產生的術語進行處理，因此必須與分詞器搭配使用。有關 Milvus 中可用的分詞器清單，請參閱「標準分詞器」及其相關頁面。
定義完analyzer_params 後，您可在定義集合架構時將其套用至VARCHAR 欄位。這使Milvus能使用指定的分析器處理該欄位中的文字，以實現高效的詞元化與過濾。詳細資訊請參閱「使用範例」。
從檔案資源載入停用詞Compatible with Milvus 3.0.x
對於大型自訂停用詞清單（例如特定語言的清單、領域詞彙表，或您希望在多個集合間共享的清單），請將詞彙儲存於檔案中，並將該檔案註冊為遠端檔案資源，接著透過 `stop_words_file ` 參數在篩選器中引用該檔案。 您可以單獨使用 `stop_words_file `，或與內嵌的 `stop_words` 搭配使用；當兩者皆設定時，篩選器會將這兩個來源合併為單一的停用詞清單。
該檔案為純 UTF‑8 文字，每行包含一個停用詞。例如：
the
of
for
將檔案上傳至您的 Milvus 叢集所配置的物件儲存庫，然後進行註冊：
from pymilvus import MilvusClient
client = MilvusClient(uri="http://localhost:19530")
# Register the uploaded file under a name you'll reference from analyzer configs.
client.add_file_resource(
name="en_stop_words",
path="file/stop_words.txt", # full S3 object key, including the rootPath prefix
)
透過 `stop_words_file` 在過濾器中引用已註冊的資源：
analyzer_params = {
"tokenizer": "standard",
"filter": [{
"type": "stop",
"stop_words_file": {
"type": "remote",
"resource_name": "en_stop_words",
"file_name": "stop_words.txt",
},
}],
}
stop_words_file 參數接受具有以下欄位的物件：
欄位 |
說明 |
|---|---|
|
資源類型。若要註冊透過 |
|
該檔案在 |
|
已註冊資源的物件儲存路徑中的檔案名稱部分（例如，若資源是透過 |
範例
在將分析器設定套用至您的集合架構之前，請使用run_analyzer 方法驗證其行為。
分析器設定
analyzer_params = {
"tokenizer": "standard",
"filter":[{
"type": "stop", # Specifies the filter type as stop
"stop_words": ["of", "to", "_english_"], # Defines custom stop words and includes the English stop word list
}],
}
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
Map<String, Object> analyzerParams = new HashMap<>();
analyzerParams.put("tokenizer", "standard");
analyzerParams.put("filter",
Collections.singletonList(
new HashMap<String, Object>() {{
put("type", "stop");
put("stop_words", Arrays.asList("of", "to", "_english_"));
}}
)
);
analyzerParams := map[string]any{"tokenizer": "standard",
"filter": []any{map[string]any{
"type": "stop",
"stop_words": []string{"of", "to", "_english_"},
}}}
use milvus::v2::prelude::*;
use serde_json::json;
let analyzer_params = json!({
"tokenizer": "standard",
"filter": [{
"type": "stop",
"stop_words": ["of", "to", "_english_"]
}]
});
nlohmann::json analyzer_params = {
{"tokenizer", "standard"},
{"filter", {
{{"type", "stop"}, {"stop_words", {"of", "to", "_english_"}}}
}}
};
const analyzer_params = {
tokenizer: "standard",
filter: [{
type: "stop", // Specifies the filter type as stop
stop_words: ["of", "to", "_english_"], // Custom stop words + built-in English list
}],
};
using System.Collections.Generic;
Dictionary<string, object> analyzerParams = new()
{
["tokenizer"] = "standard",
["filter"] = new List<object>
{
new Dictionary<string, object>
{
["type"] = "stop", // Specifies the filter type as stop
["stop_words"] = new List<object> { "of", "to", "_english_" } // Defines custom stop words and includes the English stop word list
}
}
};
# restful
analyzerParams='{
"tokenizer": "standard",
"filter": [
{
"type": "stop",
"stop_words": [
"of",
"to",
"_english_"
]
}
]
}'
使用以下方式進行驗證run_analyzer
from pymilvus import (
MilvusClient,
)
client = MilvusClient(uri="http://localhost:19530", token="root:Milvus")
# Sample text to analyze
sample_text = "The stop filter allows control over common stop words for text processing."
# Run the analyzer with the defined configuration
result = client.run_analyzer(sample_text, analyzer_params)
print("Analyzer output:", result)
import io.milvus.v2.client.ConnectConfig;
import io.milvus.v2.client.MilvusClientV2;
import io.milvus.v2.service.vector.request.RunAnalyzerReq;
import io.milvus.v2.service.vector.response.RunAnalyzerResp;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
ConnectConfig config = ConnectConfig.builder()
.uri("http://localhost:19530")
.token("root:Milvus")
.build();
MilvusClientV2 client = new MilvusClientV2(config);
List<String> texts = new ArrayList<>();
texts.add("The stop filter allows control over common stop words for text processing.");
RunAnalyzerResp resp = client.runAnalyzer(RunAnalyzerReq.builder()
.texts(texts)
.analyzerParams(analyzerParams)
.build());
List<RunAnalyzerResp.AnalyzerResult> results = resp.getResults();
import (
"context"
"fmt"
"github.com/milvus-io/milvus/client/v3/milvusclient"
)
ctx := context.Background()
client, err := milvusclient.New(ctx, &milvusclient.ClientConfig{
Address: "http://localhost:19530",
APIKey: "root:Milvus",
})
if err != nil {
fmt.Println(err.Error())
// handle error
}
texts := []string{"The stop filter allows control over common stop words for text processing."}
option := milvusclient.NewRunAnalyzerOption(texts...).
WithAnalyzerParams(analyzerParams)
result, err := client.RunAnalyzer(ctx, option)
if err != nil {
fmt.Println(err.Error())
// handle error
}
fmt.Println(result)
use milvus::v2::prelude::*;
use serde_json::json;
let config = ConnectConfig::new().uri("http://localhost:19530").token("root:Milvus");
let client = ClientV2::new(&config).await?;
let analyzer_params = json!({
"tokenizer": "standard",
"filter": [{
"type": "stop",
"stop_words": ["of", "to", "_english_"]
}]
});
let result = client
.run_analyzer(
RunAnalyzerRequest::builder()
.texts(vec!["The stop filter allows control over common stop words for text processing."])
.analyzer_params(analyzer_params)
.build()?,
)
.await?;
#include "milvus/MilvusClientV2.h"
#include <iostream>
#include <string>
auto client = milvus::MilvusClientV2::Create();
milvus::ConnectParam connect_param{"http://localhost:19530", "root:Milvus"};
auto status = client->Connect(connect_param);
if (!status.IsOk()) {
std::cout << status.Message() << std::endl;
}
std::string text = "The stop filter allows control over common stop words for text processing.";
auto request = milvus::RunAnalyzerRequest()
.AddText(text)
.WithAnalyzerParams(analyzer_params);
milvus::RunAnalyzerResponse response;
status = client->RunAnalyzer(request, response);
if (!status.IsOk()) {
std::cout << status.Message() << std::endl;
}
import { MilvusClient } from "@zilliz/milvus2-sdk-node";
const client = new MilvusClient({
address: "http://localhost:19530",
token: "root:Milvus",
});
const sample_text = "The stop filter allows control over common stop words for text processing.";
// Run the analyzer with the defined configuration
const result = await client.runAnalyzer({
text: sample_text,
analyzer_params,
});
console.log("Analyzer output:", result);
using Milvus.Client.V2;
using Milvus.Client.V2.Requests.Utility;
using Milvus.Client.V2.Responses.Utility;
using Milvus.Client.V2.Types;
MilvusClientV2 client = new(new ConnectConfig
{
Uri = "http://localhost:19530",
ApiKey = "root:Milvus"
});
// Sample text to analyze
string sampleText = "The stop filter allows control over common stop words for text processing.";
// Run the analyzer with the defined configuration
RunAnalyzerResp result = await client.RunAnalyzerAsync(new RunAnalyzerReq
{
AnalyzerParams = analyzerParams,
Texts = [sampleText]
});
foreach (AnalyzerToken t in result.Results.SelectMany(r => r.Tokens))
{
Console.WriteLine(t.Token);
}
# restful
curl --request POST \
--url "http://localhost:19530/v2/vectordb/common/run_analyzer" \
--header "Authorization: Bearer root:Milvus" \
--header "Content-Type: application/json" \
-d '{
"text": ["The stop filter allows control over common stop words for text processing."],
"analyzerParams": "{\"tokenizer\": \"standard\", \"filter\": [{\"type\": \"stop\", \"stop_words\": [\"of\", \"to\", \"_english_\"]}]}"
}'
預期輸出
['The', 'stop', 'filter', 'allows', 'control', 'over', 'common', 'stop', 'words', 'text', 'processing']
