Ziele Untersuchung
mit Columbo Integrität von
Datenbanken Interaktion und
Portierbarkeit Ergonomie der
Schnittstellen

Angebot Produkte Projekt Beratung

Mittel Analytik Modellierung Sprachen Algebra Logik Hardware Denken Kreativität

Zusammenhänge Gesellschaft Wirtschaft Branche Firma


products/sources/formale Sprachen/C/Firefox/intl/lwbrk/gtest/ (Browser von der Mozilla Stiftung Version 136.0.1^©) Datei vom 10.2.2025 mit Größe 8 kB

Quelle TestSegmenter.cpp Sprache: C

/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim: set ts=8 sts=2 et sw=2 tw=80: */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at http://mozilla.org/MPL/2.0/. */

#include "gtest/gtest.h"

#include "mozilla/intl/Segmenter.h"
#include "mozilla/Preferences.h"

namespace mozilla::intl {

TEST(IntlSegmenter, TestLineBreakIteratorUtf16SeekOld)
{
  nsresult rv = Preferences::SetBool("intl.icu4x.segmenter.enabled", false);
  EXPECT_TRUE(rv == NS_OK);

  const SegmenterOptions options{SegmenterGranularity::Line};
  auto result = Segmenter::TryCreate("en", options);
  ASSERT_TRUE(result.isOk());
  auto lineSegmenter = result.unwrap();

  const char16_t text[] = u"hello world";
  UniquePtr<SegmentIteratorUtf16> segIter =
      lineSegmenter->Segment(MakeStringSpan(text));

  // Seek to space between "hello" and "world".
  ASSERT_EQ(segIter->Seek(5u), Some(11u));

  ASSERT_EQ(segIter->Next(), Nothing());

  // Same as calling Next().
  ASSERT_EQ(segIter->Seek(0u), Nothing());
}

TEST(IntlSegmenter, TestLineBreakIteratorUtf16Seek)
{
  nsresult rv = Preferences::SetBool("intl.icu4x.segmenter.enabled", true);
  EXPECT_TRUE(rv == NS_OK);

  const SegmenterOptions options{SegmenterGranularity::Line};
  auto result = Segmenter::TryCreate("en", options);
  ASSERT_TRUE(result.isOk());
  auto lineSegmenter = result.unwrap();

  const char16_t text[] = u"hello world";
  UniquePtr<SegmentIteratorUtf16> segIter =
      lineSegmenter->Segment(MakeStringSpan(text));

  // Seek to space between "hello" and "world".
  // UAX#14 rule returns before "w".
  ASSERT_EQ(segIter->Seek(5u), Some(6u));

  ASSERT_EQ(segIter->Next(), Some(11u));

  ASSERT_EQ(segIter->Next(), Nothing());

  // Same as calling Next().
  ASSERT_EQ(segIter->Seek(0u), Nothing());
}

TEST(IntlSegmenter, TestWordBreakIteratorUtf16Simple)
{
  const SegmenterOptions options{SegmenterGranularity::Word};
  auto result = Segmenter::TryCreate("en", options);
  ASSERT_TRUE(result.isOk());
  auto wordSegmenter = result.unwrap();

  const char16_t text[] = u"hello world";
  UniquePtr<SegmentIteratorUtf16> segIter =
      wordSegmenter->Segment(MakeStringSpan(text));

  ASSERT_EQ(segIter->Next(), Some(5u));
  ASSERT_EQ(segIter->Next(), Some(6u));
  ASSERT_EQ(segIter->Next(), Some(11u));
  ASSERT_EQ(segIter->Next(), Nothing());
}

TEST(IntlSegmenter, TestWordBreakIteratorUtf16Seek)
{
  const SegmenterOptions options{SegmenterGranularity::Word};
  auto result = Segmenter::TryCreate("en", options);
  ASSERT_TRUE(result.isOk());
  auto wordSegmenter = result.unwrap();

  const char16_t text[] = u"hello world";
  UniquePtr<SegmentIteratorUtf16> segIter =
      wordSegmenter->Segment(MakeStringSpan(text));

  // Seek to the space between "hello" and "world"
  ASSERT_EQ(segIter->Seek(5u), Some(6u));

  ASSERT_EQ(segIter->Next(), Some(11u));
  ASSERT_EQ(segIter->Next(), Nothing());

  // Same as calling Next().
  ASSERT_EQ(segIter->Seek(0u), Nothing());
}

TEST(IntlSegmenter, TestWordBreakIteratorUtf16ResetAndSeekOld)
{
  nsresult rv = Preferences::SetBool("intl.icu4x.segmenter.enabled", false);
  EXPECT_TRUE(rv == NS_OK);

  const SegmenterOptions options{SegmenterGranularity::Word};
  auto result = Segmenter::TryCreate("en", options);
  ASSERT_TRUE(result.isOk());
  auto wordSegmenter = result.unwrap();

  const char16_t text[] = u"hello world";
  UniquePtr<SegmentIteratorUtf16> segIter =
      wordSegmenter->Segment(MakeStringSpan(text));

  ASSERT_EQ(segIter->Next(), Some(5u));
  static_cast<WordBreakIteratorUtf16*>(segIter.get())
      ->Reset(MakeStringSpan(text));
  ASSERT_EQ(segIter->Next(), Some(5u));
  ASSERT_EQ(segIter->Next(), Some(6u));
  ASSERT_EQ(segIter->Next(), Some(11u));

  static_cast<WordBreakIteratorUtf16*>(segIter.get())
      ->Reset(MakeStringSpan(text));
  // Seek to space between "hello" and "world".
  ASSERT_EQ(segIter->Seek(5u), Some(6u));
}

TEST(IntlSegmenter, TestWordBreakIteratorUtf16ResetAndSeek)
{
  nsresult rv = Preferences::SetBool("intl.icu4x.segmenter.enabled", true);
  EXPECT_TRUE(rv == NS_OK);

  const SegmenterOptions options{SegmenterGranularity::Word};
  auto result = Segmenter::TryCreate("en", options);
  ASSERT_TRUE(result.isOk());
  auto wordSegmenter = result.unwrap();

  const char16_t text[] = u"hello world";
  UniquePtr<SegmentIteratorUtf16> segIter =
      wordSegmenter->Segment(MakeStringSpan(text));

  ASSERT_EQ(segIter->Next(), Some(5u));
  static_cast<WordBreakIteratorUtf16*>(segIter.get())
      ->Reset(MakeStringSpan(text));
  ASSERT_EQ(segIter->Next(), Some(5u));
  ASSERT_EQ(segIter->Next(), Some(6u));
  ASSERT_EQ(segIter->Next(), Some(11u));

  static_cast<WordBreakIteratorUtf16*>(segIter.get())
      ->Reset(MakeStringSpan(text));
  // Seek to space between "hello" and "world".
  ASSERT_EQ(segIter->Seek(5u), Some(6u));
}

TEST(IntlSegmenter, TestGraphemeClusterBreakIteratorUtf16Simple)
{
  SegmenterOptions options{SegmenterGranularity::Grapheme};
  auto result = Segmenter::TryCreate("en", options);
  ASSERT_TRUE(result.isOk());
  auto graphemeClusterSegmenter = result.unwrap();

  const char16_t text[] = u"hello world";
  UniquePtr<SegmentIteratorUtf16> segIter =
      graphemeClusterSegmenter->Segment(MakeStringSpan(text));

  ASSERT_EQ(segIter->Next(), Some(1u));
  ASSERT_EQ(segIter->Next(), Some(2u));
  ASSERT_EQ(segIter->Next(), Some(3u));
  ASSERT_EQ(segIter->Next(), Some(4u));
  ASSERT_EQ(segIter->Next(), Some(5u));
  ASSERT_EQ(segIter->Next(), Some(6u));
  ASSERT_EQ(segIter->Next(), Some(7u));
  ASSERT_EQ(segIter->Next(), Some(8u));
  ASSERT_EQ(segIter->Next(), Some(9u));
  ASSERT_EQ(segIter->Next(), Some(10u));
  ASSERT_EQ(segIter->Next(), Some(11u));
  ASSERT_EQ(segIter->Next(), Nothing());
}

TEST(IntlSegmenter, TestGraphemeClusterBreakIteratorUtf16Seek)
{
  SegmenterOptions options{SegmenterGranularity::Grapheme};
  auto result = Segmenter::TryCreate("en", options);
  ASSERT_TRUE(result.isOk());
  auto graphemeClusterSegmenter = result.unwrap();

  const char16_t text[] = u"hello world";
  UniquePtr<SegmentIteratorUtf16> segIter =
      graphemeClusterSegmenter->Segment(MakeStringSpan(text));

  // Seek to the space between "hello" and "world"
  ASSERT_EQ(segIter->Seek(5u), Some(6u));

  ASSERT_EQ(segIter->Next(), Some(7u));
  ASSERT_EQ(segIter->Next(), Some(8u));
  ASSERT_EQ(segIter->Next(), Some(9u));
  ASSERT_EQ(segIter->Next(), Some(10u));
  ASSERT_EQ(segIter->Next(), Some(11u));
  ASSERT_EQ(segIter->Next(), Nothing());

  // Same as calling Next().
  ASSERT_EQ(segIter->Seek(0u), Nothing());
}

TEST(IntlSegmenter, TestGraphemeClusterBreakReverseIteratorUtf16)
{
  const char16_t text[] = u"hello world";
  GraphemeClusterBreakReverseIteratorUtf16 segIter(MakeStringSpan(text));

  // Seek to the space between "hello" and "world"
  ASSERT_EQ(segIter.Seek(6u), Some(5u));

  ASSERT_EQ(segIter.Next(), Some(4u));
  ASSERT_EQ(segIter.Next(), Some(3u));
  ASSERT_EQ(segIter.Next(), Some(2u));
  ASSERT_EQ(segIter.Next(), Some(1u));
  ASSERT_EQ(segIter.Next(), Some(0u));
  ASSERT_EQ(segIter.Next(), Nothing());

  // Same as calling Next().
  ASSERT_EQ(segIter.Seek(0u), Nothing());
}

TEST(IntlSegmenter, TestSentenceBreakIteratorUtf16)
{
  nsresult rv = Preferences::SetBool("intl.icu4x.segmenter.enabled", true);
  EXPECT_TRUE(rv == NS_OK);

  SegmenterOptions options{SegmenterGranularity::Sentence};
  auto result = Segmenter::TryCreate("en", options);
  ASSERT_TRUE(result.isOk());
  auto sentenceSegmenter = result.unwrap();

  const char16_t text[] = u"Hello world. Hello world.";
  UniquePtr<SegmentIteratorUtf16> segIter =
      sentenceSegmenter->Segment(MakeStringSpan(text));

  ASSERT_EQ(segIter->Next(), Some(13u));
  ASSERT_EQ(segIter->Next(), Some(25u));
  ASSERT_EQ(segIter->Next(), Nothing());

  // Same as calling Next().
  ASSERT_EQ(segIter->Seek(0u), Nothing());
}

TEST(IntlSegmenter, TestSentenceBreakIteratorUtf16Seek)
{
  nsresult rv = Preferences::SetBool("intl.icu4x.segmenter.enabled", true);
  EXPECT_TRUE(rv == NS_OK);

  SegmenterOptions options{SegmenterGranularity::Sentence};
  auto result = Segmenter::TryCreate("en", options);
  ASSERT_TRUE(result.isOk());
  auto sentenceSegmenter = result.unwrap();

  const char16_t text[] = u"Hello world. Hello world.";
  UniquePtr<SegmentIteratorUtf16> segIter =
      sentenceSegmenter->Segment(MakeStringSpan(text));

  ASSERT_EQ(segIter->Seek(5u), Some(13u));
}

}  // namespace mozilla::intl

Messung V0.5

¤ Dauer der Verarbeitung: 0.26 Sekunden (vorverarbeitet) ¤

Wurzel

Suchen

Beweissystem der NASA

Beweissystem Isabelle

NIST Cobol Testsuite

Cephes Mathematical Library

Wiener Entwicklungsmethode

Haftungshinweis

Die Informationen auf dieser Webseite wurden nach bestem Wissen sorgfältig zusammengestellt. Es wird jedoch weder Vollständigkeit, noch Richtigkeit, noch Qualität der bereit gestellten Informationen zugesichert.