Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code | Sign in
(1587)

Unified Diff: icu4c/source/common/rbbi.cpp

Issue 330940043: RBBI, add caching, remove reverse rules. Base URL: svn+ssh://source.icu-project.org/repos/icu/trunk/
Patch Set: Created 6 years, 5 months ago
Use n/p to move between diff chunks; N/P to move between comments. Please Sign in to add in-line comments.
Jump to:
View side-by-side diff with in-line comments
Download patch
Index: icu4c/source/common/rbbi.cpp
===================================================================
--- icu4c/source/common/rbbi.cpp (revision 40432)
+++ icu4c/source/common/rbbi.cpp (working copy)
@@ -7,7 +7,7 @@
***************************************************************************
*/
//
-// file: rbbi.c Contains the implementation of the rule based break iterator
+// file: rbbi.cpp Contains the implementation of the rule based break iterator
// runtime engine and the API implementation for
// class RuleBasedBreakIterator
//
@@ -23,15 +23,17 @@
#include "unicode/uchriter.h"
#include "unicode/uclean.h"
#include "unicode/udata.h"
+
#include "brkeng.h"
+#include "ucln_cmn.h"
#include "cmemory.h"
#include "cstring.h"
#include "rbbidata.h"
+#include "rbbi_cache.h"
#include "rbbirb.h"
#include "uassert.h"
-#include "ucln_cmn.h"
#include "umutex.h"
-#include "uvector.h"
+#include "uvectr32.h"
// if U_LOCAL_SERVICE_HOOK is defined, then localsvc.cpp is expected to be included.
#if U_LOCAL_SERVICE_HOOK
@@ -39,16 +41,16 @@
#endif
#ifdef RBBI_DEBUG
-static UBool fTrace = FALSE;
+static UBool gTrace = FALSE;
#endif
U_NAMESPACE_BEGIN
// The state number of the starting state
-#define START_STATE 1
+constexpr int32_t START_STATE = 1;
// The state-transition value indicating "stop"
-#define STOP_STATE 0
+constexpr int32_t STOP_STATE = 0;
UOBJECT_DEFINE_RTTI_IMPLEMENTATION(RuleBasedBreakIterator)
@@ -62,9 +64,8 @@
* Constructs a RuleBasedBreakIterator that uses the already-created
* tables object that is passed in as a parameter.
*/
-RuleBasedBreakIterator::RuleBasedBreakIterator(RBBIDataHeader* data, UErrorCode &status)
-{
- init();
+RuleBasedBreakIterator::RuleBasedBreakIterator(RBBIDataHeader* data, UErrorCode &status) {
+ init(status);
fData = new RBBIDataWrapper(data, status); // status checked in constructor
if (U_FAILURE(status)) {return;}
if(fData == 0) {
@@ -80,7 +81,7 @@
RuleBasedBreakIterator::RuleBasedBreakIterator(const uint8_t *compiledRules,
uint32_t ruleLength,
UErrorCode &status) {
- init();
+ init(status);
if (U_FAILURE(status)) {
return;
}
@@ -110,7 +111,7 @@
//-------------------------------------------------------------------------------
RuleBasedBreakIterator::RuleBasedBreakIterator(UDataMemory* udm, UErrorCode &status)
{
- init();
+ init(status);
fData = new RBBIDataWrapper(udm, status); // status checked in constructor
if (U_FAILURE(status)) {return;}
if(fData == 0) {
@@ -130,7 +131,7 @@
UParseError &parseError,
UErrorCode &status)
{
- init();
+ init(status);
if (U_FAILURE(status)) {return;}
RuleBasedBreakIterator *bi = (RuleBasedBreakIterator *)
RBBIRuleBuilder::createRuleBasedBreakIterator(rules, &parseError, status);
@@ -152,7 +153,8 @@
// of rules.
//-------------------------------------------------------------------------------
RuleBasedBreakIterator::RuleBasedBreakIterator() {
- init();
+ UErrorCode status = U_ZERO_ERROR;
+ init(status);
}
@@ -165,7 +167,8 @@
RuleBasedBreakIterator::RuleBasedBreakIterator(const RuleBasedBreakIterator& other)
: BreakIterator(other)
{
- this->init();
+ UErrorCode status = U_ZERO_ERROR;
+ this->init(status);
*this = other;
}
@@ -180,7 +183,7 @@
}
fCharIter = NULL;
delete fSCharIter;
- fCharIter = NULL;
+ fSCharIter = NULL;
delete fDCharIter;
fDCharIter = NULL;
@@ -190,18 +193,17 @@
fData->removeReference();
fData = NULL;
}
- if (fCachedBreakPositions) {
- uprv_free(fCachedBreakPositions);
- fCachedBreakPositions = NULL;
- }
- if (fLanguageBreakEngines) {
- delete fLanguageBreakEngines;
- fLanguageBreakEngines = NULL;
- }
- if (fUnhandledBreakEngine) {
- delete fUnhandledBreakEngine;
- fUnhandledBreakEngine = NULL;
- }
+ delete fBreakCache;
+ fBreakCache = NULL;
+
+ delete fDictionaryCache;
+ fDictionaryCache = NULL;
+
+ delete fLanguageBreakEngines;
+ fLanguageBreakEngines = NULL;
+
+ delete fUnhandledBreakEngine;
+ fUnhandledBreakEngine = NULL;
}
/**
@@ -215,7 +217,6 @@
}
BreakIterator::operator=(that);
- reset(); // Delete break cache information
fBreakType = that.fBreakType;
if (fLanguageBreakEngines != NULL) {
delete fLanguageBreakEngines;
@@ -245,6 +246,17 @@
fData = that.fData->addReference();
}
+ fPosition = that.fPosition;
+ fRuleStatusIndex = that.fRuleStatusIndex;
+ fDone = that.fDone;
+
+ // TODO: both the dictionary and the main cache need to be copied.
+ // Current position could be within a dictionary range. Trying to continue
+ // the iteration without the caches present would go to the rules, with
+ // the assumption that the current position is on a rule boundary.
+ fBreakCache->reset(fPosition, fRuleStatusIndex);
+ fDictionaryCache->reset();
+
return *this;
}
@@ -256,15 +268,15 @@
// Initializes all fields, leaving the object in a consistent state.
//
//-----------------------------------------------------------------------------
-void RuleBasedBreakIterator::init() {
- UErrorCode status = U_ZERO_ERROR;
- fText = utext_openUChars(NULL, NULL, 0, &status);
+void RuleBasedBreakIterator::init(UErrorCode &status) {
+ fText = NULL;
fCharIter = NULL;
fSCharIter = NULL;
fDCharIter = NULL;
fData = NULL;
- fLastRuleStatusIndex = 0;
- fLastStatusIndexValid = TRUE;
+ fPosition = 0;
+ fRuleStatusIndex = 0;
+ fDone = false;
fDictionaryCharCount = 0;
fBreakType = UBRK_WORD; // Defaulting BreakType to word gives reasonable
// dictionary behavior for Break Iterators that are
@@ -271,18 +283,28 @@
// built from rules. Even better would be the ability to
// declare the type in the rules.
- fCachedBreakPositions = NULL;
- fLanguageBreakEngines = NULL;
- fUnhandledBreakEngine = NULL;
- fNumCachedBreakPositions = 0;
- fPositionInCache = 0;
+ fLanguageBreakEngines = NULL;
+ fUnhandledBreakEngine = NULL;
+ fBreakCache = NULL;
+ fDictionaryCache = NULL;
+ if (U_FAILURE(status)) {
+ return;
+ }
+
+ fText = utext_openUChars(NULL, NULL, 0, &status);
+ fDictionaryCache = new DictionaryCache(this, status);
+ fBreakCache = new BreakCache(this, status);
+ if (U_SUCCESS(status) && (fText == NULL || fDictionaryCache == NULL || fBreakCache == NULL)) {
+ status = U_MEMORY_ALLOCATION_ERROR;
+ }
+
#ifdef RBBI_DEBUG
static UBool debugInitDone = FALSE;
if (debugInitDone == FALSE) {
char *debugEnv = getenv("U_RBBIDEBUG");
if (debugEnv && uprv_strstr(debugEnv, "trace")) {
- fTrace = TRUE;
+ gTrace = TRUE;
}
debugInitDone = TRUE;
}
@@ -312,6 +334,9 @@
if (typeid(*this) != typeid(that)) {
return FALSE;
}
+ if (this == &that) {
+ return TRUE;
+ }
// The base class BreakIterator carries no state that participates in equality,
// and does not implement an equality function that would otherwise be
@@ -326,6 +351,12 @@
return FALSE;
};
+ if (!(fPosition == that2.fPosition &&
+ fRuleStatusIndex == that2.fRuleStatusIndex &&
+ fDone == that2.fDone)) {
+ return FALSE;
+ }
+
if (that2.fData == fData ||
(fData != NULL && that2.fData != NULL && *that2.fData == *fData)) {
// The two break iterators are using the same rules.
@@ -352,7 +383,8 @@
if (U_FAILURE(status)) {
return;
}
- reset();
+ fBreakCache->reset();
+ fDictionaryCache->reset();
fText = utext_clone(fText, ut, FALSE, TRUE, &status);
// Set up a dummy CharacterIterator to be returned if anyone
@@ -413,7 +445,8 @@
fCharIter = newText;
UErrorCode status = U_ZERO_ERROR;
- reset();
+ fBreakCache->reset();
+ fDictionaryCache->reset();
if (newText==NULL || newText->startIndex() != 0) {
// startIndex !=0 wants to be an error, but there's no way to report it.
// Make the iterator text be an empty string.
@@ -432,7 +465,8 @@
void
RuleBasedBreakIterator::setText(const UnicodeString& newText) {
UErrorCode status = U_ZERO_ERROR;
- reset();
+ fBreakCache->reset();
+ fDictionaryCache->reset();
fText = utext_openConstUnicodeString(fText, &newText, &status);
// Set up a character iterator on the string.
@@ -492,13 +526,12 @@
* @return The new iterator position, which is zero.
*/
int32_t RuleBasedBreakIterator::first(void) {
- reset();
- fLastRuleStatusIndex = 0;
- fLastStatusIndexValid = TRUE;
- //if (fText == NULL)
- // return BreakIterator::DONE;
-
- utext_setNativeIndex(fText, 0);
+ UErrorCode status = U_ZERO_ERROR;
+ if (!fBreakCache->seek(0)) {
+ fBreakCache->populateNear(0, status);
+ }
+ fBreakCache->current();
+ U_ASSERT(fPosition == 0);
return 0;
}
@@ -507,17 +540,12 @@
* @return The text's past-the-end offset.
*/
int32_t RuleBasedBreakIterator::last(void) {
- reset();
- if (fText == NULL) {
- fLastRuleStatusIndex = 0;
- fLastStatusIndexValid = TRUE;
- return BreakIterator::DONE;
- }
-
- fLastStatusIndexValid = FALSE;
- int32_t pos = (int32_t)utext_nativeLength(fText);
- utext_setNativeIndex(fText, pos);
- return pos;
+ int32_t endPos = (int32_t)utext_nativeLength(fText);
+ UBool endShouldBeBoundary = isBoundary(endPos); // Has side effect of setting iterator position.
+ (void)endShouldBeBoundary;
+ U_ASSERT(endShouldBeBoundary);
+ U_ASSERT(fPosition == endPos);
+ return endPos;
}
/**
@@ -530,15 +558,18 @@
* the current one.
*/
int32_t RuleBasedBreakIterator::next(int32_t n) {
- int32_t result = current();
- while (n > 0) {
- result = next();
- --n;
+ int32_t result = 0;
+ if (n > 0) {
+ for (; n > 0 && result != UBRK_DONE; --n) {
+ result = next();
+ }
+ } else if (n < 0) {
+ for (; n < 0 && result != UBRK_DONE; ++n) {
+ result = previous();
+ }
+ } else {
+ result = current();
}
- while (n < 0) {
- result = previous();
- ++n;
- }
return result;
}
@@ -547,346 +578,67 @@
* @return The position of the first boundary after this one.
*/
int32_t RuleBasedBreakIterator::next(void) {
- // if we have cached break positions and we're still in the range
- // covered by them, just move one step forward in the cache
- if (fCachedBreakPositions != NULL) {
- if (fPositionInCache < fNumCachedBreakPositions - 1) {
- ++fPositionInCache;
- int32_t pos = fCachedBreakPositions[fPositionInCache];
- utext_setNativeIndex(fText, pos);
- return pos;
- }
- else {
- reset();
- }
- }
-
- int32_t startPos = current();
- fDictionaryCharCount = 0;
- int32_t result = handleNext(fData->fForwardTable);
- if (fDictionaryCharCount > 0) {
- result = checkDictionary(startPos, result, FALSE);
- }
- return result;
+ fBreakCache->next();
+ return fDone ? UBRK_DONE : fPosition;
}
/**
- * Advances the iterator backwards, to the last boundary preceding this one.
- * @return The position of the last boundary position preceding this one.
+ * Move the iterator backwards, to the boundary preceding the current one.
+ *
+ * Starts from the current position within fText.
+ * Starting position need not be on a boundary.
+ *
+ * @return The position of the boundary position immediately preceding the starting position.
*/
int32_t RuleBasedBreakIterator::previous(void) {
- int32_t result;
- int32_t startPos;
-
- // if we have cached break positions and we're still in the range
- // covered by them, just move one step backward in the cache
- if (fCachedBreakPositions != NULL) {
- if (fPositionInCache > 0) {
- --fPositionInCache;
- // If we're at the beginning of the cache, need to reevaluate the
- // rule status
- if (fPositionInCache <= 0) {
- fLastStatusIndexValid = FALSE;
- }
- int32_t pos = fCachedBreakPositions[fPositionInCache];
- utext_setNativeIndex(fText, pos);
- return pos;
- }
- else {
- reset();
- }
- }
-
- // if we're already sitting at the beginning of the text, return DONE
- if (fText == NULL || (startPos = current()) == 0) {
- fLastRuleStatusIndex = 0;
- fLastStatusIndexValid = TRUE;
- return BreakIterator::DONE;
- }
-
- if (fData->fSafeRevTable != NULL || fData->fSafeFwdTable != NULL) {
- result = handlePrevious(fData->fReverseTable);
- if (fDictionaryCharCount > 0) {
- result = checkDictionary(result, startPos, TRUE);
- }
- return result;
- }
-
- // old rule syntax
- // set things up. handlePrevious() will back us up to some valid
- // break position before the current position (we back our internal
- // iterator up one step to prevent handlePrevious() from returning
- // the current position), but not necessarily the last one before
- // where we started
-
- int32_t start = current();
-
- (void)UTEXT_PREVIOUS32(fText);
- int32_t lastResult = handlePrevious(fData->fReverseTable);
- if (lastResult == UBRK_DONE) {
- lastResult = 0;
- utext_setNativeIndex(fText, 0);
- }
- result = lastResult;
- int32_t lastTag = 0;
- UBool breakTagValid = FALSE;
-
- // iterate forward from the known break position until we pass our
- // starting point. The last break position before the starting
- // point is our return value
-
- for (;;) {
- result = next();
- if (result == BreakIterator::DONE || result >= start) {
- break;
- }
- lastResult = result;
- lastTag = fLastRuleStatusIndex;
- breakTagValid = TRUE;
- }
-
- // fLastBreakTag wants to have the value for section of text preceding
- // the result position that we are to return (in lastResult.) If
- // the backwards rules overshot and the above loop had to do two or more
- // next()s to move up to the desired return position, we will have a valid
- // tag value. But, if handlePrevious() took us to exactly the correct result position,
- // we wont have a tag value for that position, which is only set by handleNext().
-
- // Set the current iteration position to be the last break position
- // before where we started, and then return that value.
- utext_setNativeIndex(fText, lastResult);
- fLastRuleStatusIndex = lastTag; // for use by getRuleStatus()
- fLastStatusIndexValid = breakTagValid;
-
- // No need to check the dictionary; it will have been handled by
- // next()
-
- return lastResult;
+ UErrorCode status = U_ZERO_ERROR;
+ fBreakCache->previous(status);
+ return fDone ? UBRK_DONE : fPosition;
}
/**
* Sets the iterator to refer to the first boundary position following
* the specified position.
- * @offset The position from which to begin searching for a break position.
+ * @param startPos The position from which to begin searching for a break position.
* @return The position of the first break after the current position.
*/
-int32_t RuleBasedBreakIterator::following(int32_t offset) {
- // if the offset passed in is already past the end of the text,
- // just return DONE; if it's before the beginning, return the
+int32_t RuleBasedBreakIterator::following(int32_t startPos) {
+ // if the supplied position is before the beginning, return the
// text's starting offset
- if (fText == NULL || offset >= utext_nativeLength(fText)) {
- last();
- return next();
- }
- else if (offset < 0) {
+ if (startPos < 0) {
return first();
}
// Move requested offset to a code point start. It might be on a trail surrogate,
- // or on a trail byte if the input is UTF-8.
- utext_setNativeIndex(fText, offset);
- offset = (int32_t)utext_getNativeIndex(fText);
+ // or on a trail byte if the input is UTF-8. Or it may be beyond the end of the text.
+ utext_setNativeIndex(fText, startPos);
+ startPos = (int32_t)utext_getNativeIndex(fText);
- // if we have cached break positions and offset is in the range
- // covered by them, use them
- // TODO: could use binary search
- // TODO: what if offset is outside range, but break is not?
- if (fCachedBreakPositions != NULL) {
- if (offset >= fCachedBreakPositions[0]
- && offset < fCachedBreakPositions[fNumCachedBreakPositions - 1]) {
- fPositionInCache = 0;
- // We are guaranteed not to leave the array due to range test above
- while (offset >= fCachedBreakPositions[fPositionInCache]) {
- ++fPositionInCache;
- }
- int32_t pos = fCachedBreakPositions[fPositionInCache];
- utext_setNativeIndex(fText, pos);
- return pos;
- }
- else {
- reset();
- }
- }
-
- // Set our internal iteration position (temporarily)
- // to the position passed in. If this is the _beginning_ position,
- // then we can just use next() to get our return value
-
- int32_t result = 0;
-
- if (fData->fSafeRevTable != NULL) {
- // new rule syntax
- utext_setNativeIndex(fText, offset);
- // move forward one codepoint to prepare for moving back to a
- // safe point.
- // this handles offset being between a supplementary character
- // TODO: is this still needed, with move to code point boundary handled above?
- (void)UTEXT_NEXT32(fText);
- // handlePrevious will move most of the time to < 1 boundary away
- handlePrevious(fData->fSafeRevTable);
- int32_t result = next();
- while (result <= offset) {
- result = next();
- }
- return result;
- }
- if (fData->fSafeFwdTable != NULL) {
- // backup plan if forward safe table is not available
- utext_setNativeIndex(fText, offset);
- (void)UTEXT_PREVIOUS32(fText);
- // handle next will give result >= offset
- handleNext(fData->fSafeFwdTable);
- // previous will give result 0 or 1 boundary away from offset,
- // most of the time
- // we have to
- int32_t oldresult = previous();
- while (oldresult > offset) {
- int32_t result = previous();
- if (result <= offset) {
- return oldresult;
- }
- oldresult = result;
- }
- int32_t result = next();
- if (result <= offset) {
- return next();
- }
- return result;
- }
- // otherwise, we have to sync up first. Use handlePrevious() to back
- // up to a known break position before the specified position (if
- // we can determine that the specified position is a break position,
- // we don't back up at all). This may or may not be the last break
- // position at or before our starting position. Advance forward
- // from here until we've passed the starting position. The position
- // we stop on will be the first break position after the specified one.
- // old rule syntax
-
- utext_setNativeIndex(fText, offset);
- if (offset==0 ||
- (offset==1 && utext_getNativeIndex(fText)==0)) {
- return next();
- }
- result = previous();
-
- while (result != BreakIterator::DONE && result <= offset) {
- result = next();
- }
-
- return result;
+ UErrorCode status = U_ZERO_ERROR;
+ fBreakCache->following(startPos, status);
+ return fDone ? UBRK_DONE : fPosition;
}
/**
* Sets the iterator to refer to the last boundary position before the
* specified position.
- * @offset The position to begin searching for a break from.
+ * @param offset The position to begin searching for a break from.
* @return The position of the last boundary before the starting position.
*/
int32_t RuleBasedBreakIterator::preceding(int32_t offset) {
- // if the offset passed in is already past the end of the text,
- // just return DONE; if it's before the beginning, return the
- // text's starting offset
if (fText == NULL || offset > utext_nativeLength(fText)) {
return last();
}
- else if (offset < 0) {
- return first();
- }
// Move requested offset to a code point start. It might be on a trail surrogate,
// or on a trail byte if the input is UTF-8.
+
utext_setNativeIndex(fText, offset);
- offset = (int32_t)utext_getNativeIndex(fText);
+ int32_t adjustedOffset = utext_getNativeIndex(fText);
- // if we have cached break positions and offset is in the range
- // covered by them, use them
- if (fCachedBreakPositions != NULL) {
- // TODO: binary search?
- // TODO: What if offset is outside range, but break is not?
- if (offset > fCachedBreakPositions[0]
- && offset <= fCachedBreakPositions[fNumCachedBreakPositions - 1]) {
- fPositionInCache = 0;
- while (fPositionInCache < fNumCachedBreakPositions
- && offset > fCachedBreakPositions[fPositionInCache])
- ++fPositionInCache;
- --fPositionInCache;
- // If we're at the beginning of the cache, need to reevaluate the
- // rule status
- if (fPositionInCache <= 0) {
- fLastStatusIndexValid = FALSE;
- }
- utext_setNativeIndex(fText, fCachedBreakPositions[fPositionInCache]);
- return fCachedBreakPositions[fPositionInCache];
- }
- else {
- reset();
- }
- }
-
- // if we start by updating the current iteration position to the
- // position specified by the caller, we can just use previous()
- // to carry out this operation
-
- if (fData->fSafeFwdTable != NULL) {
- // new rule syntax
- utext_setNativeIndex(fText, offset);
- int32_t newOffset = (int32_t)UTEXT_GETNATIVEINDEX(fText);
- if (newOffset != offset) {
- // Will come here if specified offset was not a code point boundary AND
- // the underlying implmentation is using UText, which snaps any non-code-point-boundary
- // indices to the containing code point.
- // For breakitereator::preceding only, these non-code-point indices need to be moved
- // up to refer to the following codepoint.
- (void)UTEXT_NEXT32(fText);
- offset = (int32_t)UTEXT_GETNATIVEINDEX(fText);
- }
-
- // TODO: (synwee) would it be better to just check for being in the middle of a surrogate pair,
- // rather than adjusting the position unconditionally?
- // (Change would interact with safe rules.)
- // TODO: change RBBI behavior for off-boundary indices to match that of UText?
- // affects only preceding(), seems cleaner, but is slightly different.
- (void)UTEXT_PREVIOUS32(fText);
- handleNext(fData->fSafeFwdTable);
- int32_t result = (int32_t)UTEXT_GETNATIVEINDEX(fText);
- while (result >= offset) {
- result = previous();
- }
- return result;
- }
- if (fData->fSafeRevTable != NULL) {
- // backup plan if forward safe table is not available
- // TODO: check whether this path can be discarded
- // It's probably OK to say that rules must supply both safe tables
- // if they use safe tables at all. We have certainly never described
- // to anyone how to work with just one safe table.
- utext_setNativeIndex(fText, offset);
- (void)UTEXT_NEXT32(fText);
-
- // handle previous will give result <= offset
- handlePrevious(fData->fSafeRevTable);
-
- // next will give result 0 or 1 boundary away from offset,
- // most of the time
- // we have to
- int32_t oldresult = next();
- while (oldresult < offset) {
- int32_t result = next();
- if (result >= offset) {
- return oldresult;
- }
- oldresult = result;
- }
- int32_t result = previous();
- if (result >= offset) {
- return previous();
- }
- return result;
- }
-
- // old rule syntax
- utext_setNativeIndex(fText, offset);
- return previous();
+ UErrorCode status = U_ZERO_ERROR;
+ fBreakCache->preceding(adjustedOffset, status);
+ return fDone ? UBRK_DONE : fPosition;
}
/**
@@ -893,21 +645,11 @@
* Returns true if the specfied position is a boundary position. As a side
* effect, leaves the iterator pointing to the first boundary position at
* or after "offset".
+ *
* @param offset the offset to check.
* @return True if "offset" is a boundary position.
*/
UBool RuleBasedBreakIterator::isBoundary(int32_t offset) {
- // the beginning index of the iterator is always a boundary position by definition
- if (offset == 0) {
- first(); // For side effects on current position, tag values.
- return TRUE;
- }
-
- if (offset == (int32_t)utext_nativeLength(fText)) {
- last(); // For side effects on current position, tag values.
- return TRUE;
- }
-
// out-of-range indexes are never boundary positions
if (offset < 0) {
first(); // For side effects on current position, tag values.
@@ -914,29 +656,42 @@
return FALSE;
}
- if (offset > utext_nativeLength(fText)) {
- last(); // For side effects on current position, tag values.
+ // Adjust offset to be on a code point boundary and not beyond the end of the text.
+ // Note that isBoundary() is always be false for offsets that are not on code point boundaries.
+ // But we still need the side effect of leaving iteration at the following boundary.
+
+ utext_setNativeIndex(fText, offset);
+ int32_t adjustedOffset = utext_getNativeIndex(fText);
+
+ bool result = false;
+ UErrorCode status = U_ZERO_ERROR;
+ if (fBreakCache->seek(adjustedOffset) || fBreakCache->populateNear(adjustedOffset, status)) {
+ result = (fBreakCache->current() == offset);
+ }
+
+ if (result && adjustedOffset < offset && utext_char32At(fText, offset) == U_SENTINEL) {
+ // Original offset is beyond the end of the text. Return FALSE, it's not a boundary,
+ // but the iteration position remains set to the end of the text, which is a boundary.
return FALSE;
}
-
- // otherwise, we can use following() on the position before the specified
- // one and return true if the position we get back is the one the user
- // specified
- utext_previous32From(fText, offset);
- int32_t backOne = (int32_t)UTEXT_GETNATIVEINDEX(fText);
- UBool result = following(backOne) == offset;
+ if (!result) {
+ // Not on a boundary. isBoundary() must leave iterator on the following boundary.
+ // Cache->seek(), above, left us on the preceding boundary, so advance one.
+ next();
+ }
return result;
}
+
/**
* Returns the current iteration position.
* @return The current iteration position.
*/
int32_t RuleBasedBreakIterator::current(void) const {
- int32_t pos = (int32_t)UTEXT_GETNATIVEINDEX(fText);
- return pos;
+ return fPosition;
}
+
//=======================================================================
// implementation
//=======================================================================
@@ -1003,15 +758,11 @@
//-----------------------------------------------------------------------------------
//
-// handleNext(stateTable)
-// This method is the actual implementation of the rbbi next() method.
-// This method initializes the state machine to state 1
-// and advances through the text character by character until we reach the end
-// of the text or the state machine transitions to state 0. We update our return
-// value every time the state machine passes through an accepting state.
+// handleNext()
+// Run the state machine to find a boundary
//
//-----------------------------------------------------------------------------------
-int32_t RuleBasedBreakIterator::handleNext(const RBBIStateTable *statetable) {
+int32_t RuleBasedBreakIterator::handleNext() {
int32_t state;
uint16_t category = 0;
RBBIRunMode mode;
@@ -1021,25 +772,29 @@
LookAheadResults lookAheadMatches;
int32_t result = 0;
int32_t initialPosition = 0;
+ const RBBIStateTable *statetable = fData->fForwardTable;
const char *tableData = statetable->fTableData;
uint32_t tableRowLen = statetable->fRowLen;
-
#ifdef RBBI_DEBUG
- if (fTrace) {
+ if (gTrace) {
RBBIDebugPuts("Handle Next pos char state category");
}
#endif
- // No matter what, handleNext alway correctly sets the break tag value.
- fLastStatusIndexValid = TRUE;
- fLastRuleStatusIndex = 0;
+ // handleNext alway sets the break tag value.
+ // Set the default for it.
+ fRuleStatusIndex = 0;
+ fDictionaryCharCount = 0;
+
// if we're already at the end of the text, return DONE.
- initialPosition = (int32_t)UTEXT_GETNATIVEINDEX(fText);
+ initialPosition = fPosition;
+ UTEXT_SETNATIVEINDEX(fText, initialPosition);
result = initialPosition;
c = UTEXT_NEXT32(fText);
- if (fData == NULL || c==U_SENTINEL) {
- return BreakIterator::DONE;
+ if (c==U_SENTINEL) {
+ fDone = TRUE;
+ return UBRK_DONE;
}
// Set the initial state for the state machine
@@ -1086,7 +841,7 @@
category = UTRIE2_GET16(fData->fTrie, c);
// Check the dictionary bit in the character's category.
- // Counter is only used by dictionary based iterators (subclasses).
+ // Counter is only used by dictionary based iteration.
// Chars that need to be handled by a dictionary have a flag bit set
// in their category values.
//
@@ -1098,7 +853,7 @@
}
#ifdef RBBI_DEBUG
- if (fTrace) {
+ if (gTrace) {
RBBIDebugPrintf(" %4ld ", utext_getNativeIndex(fText));
if (0x20<=c && c<0x7f) {
RBBIDebugPrintf("\"%c\" ", c);
@@ -1127,7 +882,7 @@
if (mode != RBBI_START) {
result = (int32_t)UTEXT_GETNATIVEINDEX(fText);
}
- fLastRuleStatusIndex = row->fTagIdx; // Remember the break status (tag) values.
+ fRuleStatusIndex = row->fTagIdx; // Remember the break status (tag) values.
}
int16_t completedRule = row->fAccepting;
@@ -1135,8 +890,8 @@
// Lookahead match is completed.
int32_t lookaheadResult = lookAheadMatches.getPosition(completedRule);
if (lookaheadResult >= 0) {
- fLastRuleStatusIndex = row->fTagIdx;
- UTEXT_SETNATIVEINDEX(fText, lookaheadResult);
+ fRuleStatusIndex = row->fTagIdx;
+ fPosition = lookaheadResult;
return lookaheadResult;
}
}
@@ -1165,8 +920,6 @@
mode = RBBI_RUN;
}
}
-
-
}
// The state machine is done. Check whether it found a match...
@@ -1175,15 +928,16 @@
// (This really indicates a defect in the break rules. They should always match
// at least one character.)
if (result == initialPosition) {
- UTEXT_SETNATIVEINDEX(fText, initialPosition);
- UTEXT_NEXT32(fText);
- result = (int32_t)UTEXT_GETNATIVEINDEX(fText);
+ utext_setNativeIndex(fText, initialPosition);
+ utext_next32(fText);
+ result = (int32_t)utext_getNativeIndex(fText);
+ fRuleStatusIndex = 0;
}
// Leave the iterator at our result position.
- UTEXT_SETNATIVEINDEX(fText, result);
+ fPosition = result;
#ifdef RBBI_DEBUG
- if (fTrace) {
+ if (gTrace) {
RBBIDebugPrintf("result = %d\n\n", result);
}
#endif
@@ -1196,13 +950,11 @@
//
// handlePrevious()
//
-// Iterate backwards, according to the logic of the reverse rules.
-// This version handles the exact style backwards rules.
-//
+// Iterate backwards using the safe reverse rules.
// The logic of this function is very similar to handleNext(), above.
//
//-----------------------------------------------------------------------------------
-int32_t RuleBasedBreakIterator::handlePrevious(const RBBIStateTable *statetable) {
+int32_t RuleBasedBreakIterator::handlePrevious(int32_t fromPosition) {
int32_t state;
uint16_t category = 0;
RBBIRunMode mode;
@@ -1212,19 +964,14 @@
int32_t result = 0;
int32_t initialPosition = 0;
+ const RBBIStateTable *stateTable = fData->fSafeRevTable;
+ UTEXT_SETNATIVEINDEX(fText, fromPosition);
#ifdef RBBI_DEBUG
- if (fTrace) {
+ if (gTrace) {
RBBIDebugPuts("Handle Previous pos char state category");
}
#endif
- // handlePrevious() never gets the rule status.
- // Flag the status as invalid; if the user ever asks for status, we will need
- // to back up, then re-find the break position using handleNext(), which does
- // get the status value.
- fLastStatusIndexValid = FALSE;
- fLastRuleStatusIndex = 0;
-
// if we're already at the start of the text, return DONE.
if (fText == NULL || fData == NULL || UTEXT_GETNATIVEINDEX(fText)==0) {
return BreakIterator::DONE;
@@ -1238,10 +985,10 @@
// Set the initial state for the state machine
state = START_STATE;
row = (RBBIStateTableRow *)
- (statetable->fTableData + (statetable->fRowLen * state));
+ (stateTable->fTableData + (stateTable->fRowLen * state));
category = 3;
mode = RBBI_RUN;
- if (statetable->fFlags & RBBI_BOF_REQUIRED) {
+ if (stateTable->fFlags & RBBI_BOF_REQUIRED) {
category = 2;
mode = RBBI_START;
}
@@ -1256,12 +1003,6 @@
// We have already run the loop one last time with the
// character set to the psueudo {eof} value. Now it is time
// to unconditionally bail out.
- if (result == initialPosition) {
- // Ran off start, no match found.
- // move one index one (towards the start, since we are doing a previous())
- UTEXT_SETNATIVEINDEX(fText, initialPosition);
- (void)UTEXT_PREVIOUS32(fText); // TODO: shouldn't be necessary. We're already at beginning. Check.
- }
break;
}
// Run the loop one last time with the fake end-of-input character category.
@@ -1280,22 +1021,13 @@
// Note: the 16 in UTRIE_GET16 refers to the size of the data being returned,
// not the size of the character going in, which is a UChar32.
//
+ // And off the dictionary flag bit. For reverse iteration it is not used.
category = UTRIE2_GET16(fData->fTrie, c);
-
- // Check the dictionary bit in the character's category.
- // Counter is only used by dictionary based iterators (subclasses).
- // Chars that need to be handled by a dictionary have a flag bit set
- // in their category values.
- //
- if ((category & 0x4000) != 0) {
- fDictionaryCharCount++;
- // And off the dictionary flag bit.
- category &= ~0x4000;
- }
+ category &= ~0x4000;
}
#ifdef RBBI_DEBUG
- if (fTrace) {
+ if (gTrace) {
RBBIDebugPrintf(" %4d ", (int32_t)utext_getNativeIndex(fText));
if (0x20<=c && c<0x7f) {
RBBIDebugPrintf("\"%c\" ", c);
@@ -1315,7 +1047,7 @@
U_ASSERT(category<fData->fHeader->fCatCount);
state = row->fNextState[category]; /*Not accessing beyond memory*/
row = (RBBIStateTableRow *)
- (statetable->fTableData + (statetable->fRowLen * state));
+ (stateTable->fTableData + (stateTable->fRowLen * state));
if (row->fAccepting == -1) {
// Match found, common case.
@@ -1369,10 +1101,8 @@
result = (int32_t)UTEXT_GETNATIVEINDEX(fText);
}
- // Leave the iterator at our result position.
- UTEXT_SETNATIVEINDEX(fText, result);
#ifdef RBBI_DEBUG
- if (fTrace) {
+ if (gTrace) {
RBBIDebugPrintf("result = %d\n\n", result);
}
#endif
@@ -1380,20 +1110,6 @@
}
-void
-RuleBasedBreakIterator::reset()
-{
- if (fCachedBreakPositions) {
- uprv_free(fCachedBreakPositions);
- }
- fCachedBreakPositions = NULL;
- fNumCachedBreakPositions = 0;
- fDictionaryCharCount = 0;
- fPositionInCache = 0;
-}
-
-
-
//-------------------------------------------------------------------------------
//
// getRuleStatus() Return the break rule tag associated with the current
@@ -1401,46 +1117,14 @@
// position by iterating forwards, the value will have been
// cached by the handleNext() function.
//
-// If no cached status value is available, the status is
-// found by doing a previous() followed by a next(), which
-// leaves the iterator where it started, and computes the
-// status while doing the next().
-//
//-------------------------------------------------------------------------------
-void RuleBasedBreakIterator::makeRuleStatusValid() {
- if (fLastStatusIndexValid == FALSE) {
- // No cached status is available.
- if (fText == NULL || current() == 0) {
- // At start of text, or there is no text. Status is always zero.
- fLastRuleStatusIndex = 0;
- fLastStatusIndexValid = TRUE;
- } else {
- // Not at start of text. Find status the tedious way.
- int32_t pa = current();
- previous();
- if (fNumCachedBreakPositions > 0) {
- reset(); // Blow off the dictionary cache
- }
- int32_t pb = next();
- if (pa != pb) {
- // note: the if (pa != pb) test is here only to eliminate warnings for
- // unused local variables on gcc. Logically, it isn't needed.
- U_ASSERT(pa == pb);
- }
- }
- }
- U_ASSERT(fLastRuleStatusIndex >= 0 && fLastRuleStatusIndex < fData->fStatusMaxIdx);
-}
-
int32_t RuleBasedBreakIterator::getRuleStatus() const {
- RuleBasedBreakIterator *nonConstThis = (RuleBasedBreakIterator *)this;
- nonConstThis->makeRuleStatusValid();
// fLastRuleStatusIndex indexes to the start of the appropriate status record
// (the number of status values.)
// This function returns the last (largest) of the array of status values.
- int32_t idx = fLastRuleStatusIndex + fData->fRuleStatusTable[fLastRuleStatusIndex];
+ int32_t idx = fRuleStatusIndex + fData->fRuleStatusTable[fRuleStatusIndex];
int32_t tagVal = fData->fRuleStatusTable[idx];
return tagVal;
@@ -1447,18 +1131,13 @@
}
-
-
int32_t RuleBasedBreakIterator::getRuleStatusVec(
- int32_t *fillInVec, int32_t capacity, UErrorCode &status)
-{
+ int32_t *fillInVec, int32_t capacity, UErrorCode &status) {
if (U_FAILURE(status)) {
return 0;
}
- RuleBasedBreakIterator *nonConstThis = (RuleBasedBreakIterator *)this;
- nonConstThis->makeRuleStatusValid();
- int32_t numVals = fData->fRuleStatusTable[fLastRuleStatusIndex];
+ int32_t numVals = fData->fRuleStatusTable[fRuleStatusIndex];
int32_t numValsToCopy = numVals;
if (numVals > capacity) {
status = U_BUFFER_OVERFLOW_ERROR;
@@ -1466,7 +1145,7 @@
}
int i;
for (i=0; i<numValsToCopy; i++) {
- fillInVec[i] = fData->fRuleStatusTable[fLastRuleStatusIndex + i + 1];
+ fillInVec[i] = fData->fRuleStatusTable[fRuleStatusIndex + i + 1];
}
return numVals;
}
@@ -1514,169 +1193,6 @@
return (RuleBasedBreakIterator *)clonedBI;
}
-
-//-------------------------------------------------------------------------------
-//
-// checkDictionary This function handles all processing of characters in
-// the "dictionary" set. It will determine the appropriate
-// course of action, and possibly set up a cache in the
-// process.
-//
-//-------------------------------------------------------------------------------
-int32_t RuleBasedBreakIterator::checkDictionary(int32_t startPos,
- int32_t endPos,
- UBool reverse) {
- // Reset the old break cache first.
- reset();
-
- // note: code segment below assumes that dictionary chars are in the
- // startPos-endPos range
- // value returned should be next character in sequence
- if ((endPos - startPos) <= 1) {
- return (reverse ? startPos : endPos);
- }
-
- // Starting from the starting point, scan towards the proposed result,
- // looking for the first dictionary character (which may be the one
- // we're on, if we're starting in the middle of a range).
- utext_setNativeIndex(fText, reverse ? endPos : startPos);
- if (reverse) {
- UTEXT_PREVIOUS32(fText);
- }
-
- int32_t rangeStart = startPos;
- int32_t rangeEnd = endPos;
-
- uint16_t category;
- int32_t current;
- UErrorCode status = U_ZERO_ERROR;
- UStack breaks(status);
- int32_t foundBreakCount = 0;
- UChar32 c = utext_current32(fText);
-
- category = UTRIE2_GET16(fData->fTrie, c);
-
- // Is the character we're starting on a dictionary character? If so, we
- // need to back up to include the entire run; otherwise the results of
- // the break algorithm will differ depending on where we start. Since
- // the result is cached and there is typically a non-dictionary break
- // within a small number of words, there should be little performance impact.
- if (category & 0x4000) {
- if (reverse) {
- do {
- utext_next32(fText); // TODO: recast to work directly with postincrement.
- c = utext_current32(fText);
- category = UTRIE2_GET16(fData->fTrie, c);
- } while (c != U_SENTINEL && (category & 0x4000));
- // Back up to the last dictionary character
- rangeEnd = (int32_t)UTEXT_GETNATIVEINDEX(fText);
- if (c == U_SENTINEL) {
- // c = fText->last32();
- // TODO: why was this if needed?
- c = UTEXT_PREVIOUS32(fText);
- }
- else {
- c = UTEXT_PREVIOUS32(fText);
- }
- }
- else {
- do {
- c = UTEXT_PREVIOUS32(fText);
- category = UTRIE2_GET16(fData->fTrie, c);
- }
- while (c != U_SENTINEL && (category & 0x4000));
- // Back up to the last dictionary character
- if (c == U_SENTINEL) {
- // c = fText->first32();
- c = utext_current32(fText);
- }
- else {
- utext_next32(fText);
- c = utext_current32(fText);
- }
- rangeStart = (int32_t)UTEXT_GETNATIVEINDEX(fText);;
- }
- category = UTRIE2_GET16(fData->fTrie, c);
- }
-
- // Loop through the text, looking for ranges of dictionary characters.
- // For each span, find the appropriate break engine, and ask it to find
- // any breaks within the span.
- // Note: we always do this in the forward direction, so that the break
- // cache is built in the right order.
- if (reverse) {
- utext_setNativeIndex(fText, rangeStart);
- c = utext_current32(fText);
- category = UTRIE2_GET16(fData->fTrie, c);
- }
- while(U_SUCCESS(status)) {
- while((current = (int32_t)UTEXT_GETNATIVEINDEX(fText)) < rangeEnd && (category & 0x4000) == 0) {
- utext_next32(fText); // TODO: tweak for post-increment operation
- c = utext_current32(fText);
- category = UTRIE2_GET16(fData->fTrie, c);
- }
- if (current >= rangeEnd) {
- break;
- }
-
- // We now have a dictionary character. Get the appropriate language object
- // to deal with it.
- const LanguageBreakEngine *lbe = getLanguageBreakEngine(c);
-
- // Ask the language object if there are any breaks. It will leave the text
- // pointer on the other side of its range, ready to search for the next one.
- if (lbe != NULL) {
- foundBreakCount += lbe->findBreaks(fText, rangeStart, rangeEnd, FALSE, fBreakType, breaks);
- }
-
- // Reload the loop variables for the next go-round
- c = utext_current32(fText);
- category = UTRIE2_GET16(fData->fTrie, c);
- }
-
- // If we found breaks, build a new break cache. The first and last entries must
- // be the original starting and ending position.
- if (foundBreakCount > 0) {
- U_ASSERT(foundBreakCount == breaks.size());
- int32_t totalBreaks = foundBreakCount;
- if (startPos < breaks.elementAti(0)) {
- totalBreaks += 1;
- }
- if (endPos > breaks.peeki()) {
- totalBreaks += 1;
- }
- fCachedBreakPositions = (int32_t *)uprv_malloc(totalBreaks * sizeof(int32_t));
- if (fCachedBreakPositions != NULL) {
- int32_t out = 0;
- fNumCachedBreakPositions = totalBreaks;
- if (startPos < breaks.elementAti(0)) {
- fCachedBreakPositions[out++] = startPos;
- }
- for (int32_t i = 0; i < foundBreakCount; ++i) {
- fCachedBreakPositions[out++] = breaks.elementAti(i);
- }
- if (endPos > fCachedBreakPositions[out-1]) {
- fCachedBreakPositions[out] = endPos;
- }
- // If there are breaks, then by definition, we are replacing the original
- // proposed break by one of the breaks we found. Use following() and
- // preceding() to do the work. They should never recurse in this case.
- if (reverse) {
- return preceding(endPos);
- }
- else {
- return following(startPos);
- }
- }
- // If the allocation failed, just fall through to the "no breaks found" case.
- }
-
- // If we get here, there were no language-based breaks. Set the text pointer
- // to the original proposed break.
- utext_setNativeIndex(fText, reverse ? startPos : endPos);
- return (reverse ? startPos : endPos);
-}
-
U_NAMESPACE_END
@@ -1824,9 +1340,11 @@
void RuleBasedBreakIterator::setBreakType(int32_t type) {
fBreakType = type;
- reset();
}
+void RuleBasedBreakIterator::dumpCache() {
+ fBreakCache->dumpCache();
+}
/**
* Returns the description used to create this iterator

Powered by Google App Engine
RSS Feeds Recent Issues | This issue
This is Rietveld f62528b