Huruf kecil

Filter ` lowercase ` mengubah istilah yang dihasilkan oleh tokenizer menjadi huruf kecil, sehingga pencarian tidak membedakan huruf besar dan kecil. Misalnya, filter ini dapat mengubah ` ["High", "Performance", "Vector", "Database"] ` menjadi ` ["high", "performance", "vector", "database"]`.

Konfigurasi

Filter ` lowercase ` sudah terintegrasi dalam Milvus. Untuk menggunakannya, cukup tentukan namanya di bagian ` filter ` dalam berkas ` analyzer_params`.

analyzer_params = {
    "tokenizer": "standard",
    "filter": ["lowercase"],
}

Map<String, Object> analyzerParams = new HashMap<>();
analyzerParams.put("tokenizer", "standard");
analyzerParams.put("filter", Collections.singletonList("lowercase"));

analyzerParams := map[string]any{"tokenizer": "standard", "filter": []any{"lowercase"}}

use serde_json::json;

let analyzer_params = json!({
    "tokenizer": "standard",
    "filter": ["lowercase"]
});

#include <milvus/thirdparty/nlohmann/json.hpp>

nlohmann::json analyzer_params = {
    {"tokenizer", "standard"},
    {"filter", {"lowercase"}}
};

const analyzer_params = {
    "tokenizer": "standard",
    "filter": ["lowercase"],
};

using System.Collections.Generic;

Dictionary<string, object> analyzerParams = new()
{
    ["tokenizer"] = "standard",
    ["filter"] = new List<object> { "lowercase" }
};

# restful
analyzerParams='{
  "tokenizer": "standard",
  "filter": [
    "lowercase"
  ]
}'

Filter ` lowercase ` beroperasi pada istilah yang dihasilkan oleh tokenizer, sehingga harus digunakan bersama dengan tokenizer.

Setelah mendefinisikan analyzer_params, Anda dapat menerapkannya pada bidang VARCHAR saat mendefinisikan skema koleksi. Hal ini memungkinkan Milvus memproses teks dalam bidang tersebut menggunakan penganalisis yang ditentukan untuk tokenisasi dan penyaringan yang efisien. Untuk detailnya, lihat Contoh penggunaan.

Contoh

Sebelum menerapkan konfigurasi penganalisis ke skema koleksi Anda, verifikasi perilakunya menggunakan metode ` run_analyzer `.

Konfigurasi penganalisis

analyzer_params = {
    "tokenizer": "standard",
    "filter": ["lowercase"],
}

Map<String, Object> analyzerParams = new HashMap<>();
analyzerParams.put("tokenizer", "standard");
analyzerParams.put("filter", Collections.singletonList("lowercase"));

analyzerParams := map[string]any{"tokenizer": "standard", "filter": []any{"lowercase"}}

use serde_json::json;

let analyzer_params = json!({
    "tokenizer": "standard",
    "filter": ["lowercase"]
});

#include <milvus/thirdparty/nlohmann/json.hpp>

nlohmann::json analyzer_params = {
    {"tokenizer", "standard"},
    {"filter", {"lowercase"}}
};

const analyzer_params = {
    "tokenizer": "standard",
    "filter": ["lowercase"],
}

using System.Collections.Generic;

Dictionary<string, object> analyzerParams = new()
{
    ["tokenizer"] = "standard",
    ["filter"] = new List<object> { "lowercase" }
};

# restful
analyzerParams='{
  "tokenizer": "standard",
  "filter": [
    "lowercase"
  ]
}'

Verifikasi menggunakan run_analyzer

from pymilvus import (
    MilvusClient,
)

client = MilvusClient(
    uri="http://localhost:19530",
    token="root:Milvus"
)

# Sample text to analyze
sample_text = "The Lowercase Filter Ensures Uniformity In Text Processing."

# Run the standard analyzer with the defined configuration
result = client.run_analyzer(sample_text, analyzer_params)
print("Standard analyzer output:", result)

import java.util.*;
import io.milvus.v2.client.ConnectConfig;
import io.milvus.v2.client.MilvusClientV2;
import io.milvus.v2.service.vector.request.RunAnalyzerReq;
import io.milvus.v2.service.vector.response.RunAnalyzerResp;

ConnectConfig config = ConnectConfig.builder()
        .uri("http://localhost:19530")
        .token("root:Milvus")
        .build();
MilvusClientV2 client = new MilvusClientV2(config);

List<String> texts = new ArrayList<>();
texts.add("The Lowercase Filter Ensures Uniformity In Text Processing.");

RunAnalyzerResp resp = client.runAnalyzer(RunAnalyzerReq.builder()
        .texts(texts)
        .analyzerParams(analyzerParams)
        .build());
List<RunAnalyzerResp.AnalyzerResult> results = resp.getResults();

import (
    "context"
    "fmt"

    "github.com/milvus-io/milvus/client/v3/milvusclient"
)

ctx := context.Background()

client, err := milvusclient.New(ctx, &milvusclient.ClientConfig{
    Address: "http://localhost:19530",
    APIKey:  "root:Milvus",
})
if err != nil {
    fmt.Println(err.Error())
    // handle error
}

texts := []string{"The Lowercase Filter Ensures Uniformity In Text Processing."}
option := milvusclient.NewRunAnalyzerOption(texts...).
    WithAnalyzerParams(analyzerParams)

result, err := client.RunAnalyzer(ctx, option)
if err != nil {
    fmt.Println(err.Error())
    // handle error
}

for _, r := range result {
    for _, token := range r.Tokens {
        fmt.Println(token.Text)
    }
}

use milvus::v2::prelude::*;

let config = ConnectConfig::new().uri("http://localhost:19530").token("root:Milvus");
let client = ClientV2::new(&config).await?;

let response = client
    .run_analyzer(
        RunAnalyzerRequest::builder()
            .texts(["The Lowercase Filter Ensures Uniformity In Text Processing."])
            .analyzer_params(analyzer_params)
            .build()?,
    )
    .await?;

for result in response.results() {
    for token in result.get_tokens() {
        println!("Standard analyzer output: {}", token.get_text());
    }
}

#include "milvus/MilvusClientV2.h"
#include <string>
#include <iostream>

auto client = milvus::MilvusClientV2::Create();

milvus::ConnectParam connect_param{"http://localhost:19530", "root:Milvus"};
auto status = client->Connect(connect_param);
if (!status.IsOk()) {
    std::cout << status.Message() << std::endl;
}

std::string text = "The Lowercase Filter Ensures Uniformity In Text Processing.";
auto request = milvus::RunAnalyzerRequest()
                       .AddText(text)
                       .WithAnalyzerParams(analyzer_params);

milvus::RunAnalyzerResponse response;
status = client->RunAnalyzer(request, response);
if (!status.IsOk()) {
    std::cout << status.Message() << std::endl;
}

import { MilvusClient } from '@zilliz/milvus2-sdk-node';

const client = new MilvusClient({
  address: 'http://localhost:19530',
  token: "root:Milvus",
});

const sample_text = 'The Lowercase Filter Ensures Uniformity In Text Processing.';
const result = await client.runAnalyzer({
  text: sample_text,
  analyzer_params,
});
const tokens = result.results.flatMap(r => r.tokens.map(t => t.token));
console.log('Standard analyzer output:', tokens);

using System;
using Milvus.Client.V2;
using Milvus.Client.V2.Requests.Utility;
using Milvus.Client.V2.Responses.Utility;
using Milvus.Client.V2.Types;

MilvusClientV2 client = new(new ConnectConfig
{
    Uri = "http://localhost:19530",
    ApiKey = "root:Milvus"
});

// Sample text to analyze
string sampleText = "The Lowercase Filter Ensures Uniformity In Text Processing.";

// Run the standard analyzer with the defined configuration
RunAnalyzerResp result = await client.RunAnalyzerAsync(new RunAnalyzerReq
{
    AnalyzerParams = analyzerParams,
    Texts = [sampleText]
});

foreach (AnalyzerResult r in result.Results)
{
    foreach (AnalyzerToken t in r.Tokens)
    {
        Console.WriteLine($"Standard analyzer output: {t.Token}");
    }
}

curl --request POST \
  --url "http://localhost:19530/v2/vectordb/common/run_analyzer" \
  --header "Authorization: Bearer root:Milvus" \
  --header "Content-Type: application/json" \
  --data-raw '{
    "analyzerParams": "{\"tokenizer\": \"standard\", \"filter\": [\"lowercase\"]}",
    "text": ["The Lowercase Filter Ensures Uniformity In Text Processing."]
  }'

Hasil yang diharapkan

['the', 'lowercase', 'filter', 'ensures', 'uniformity', 'in', 'text', 'processing']