Index: icu4c/source/common/rbbi.cpp |
=================================================================== |
--- icu4c/source/common/rbbi.cpp (revision 40432) |
+++ icu4c/source/common/rbbi.cpp (working copy) |
@@ -7,7 +7,7 @@ |
*************************************************************************** |
*/ |
// |
-// file: rbbi.c Contains the implementation of the rule based break iterator |
+// file: rbbi.cpp Contains the implementation of the rule based break iterator |
// runtime engine and the API implementation for |
// class RuleBasedBreakIterator |
// |
@@ -23,15 +23,17 @@ |
#include "unicode/uchriter.h" |
#include "unicode/uclean.h" |
#include "unicode/udata.h" |
+ |
#include "brkeng.h" |
+#include "ucln_cmn.h" |
#include "cmemory.h" |
#include "cstring.h" |
#include "rbbidata.h" |
+#include "rbbi_cache.h" |
#include "rbbirb.h" |
#include "uassert.h" |
-#include "ucln_cmn.h" |
#include "umutex.h" |
-#include "uvector.h" |
+#include "uvectr32.h" |
// if U_LOCAL_SERVICE_HOOK is defined, then localsvc.cpp is expected to be included. |
#if U_LOCAL_SERVICE_HOOK |
@@ -39,16 +41,16 @@ |
#endif |
#ifdef RBBI_DEBUG |
-static UBool fTrace = FALSE; |
+static UBool gTrace = FALSE; |
#endif |
U_NAMESPACE_BEGIN |
// The state number of the starting state |
-#define START_STATE 1 |
+constexpr int32_t START_STATE = 1; |
// The state-transition value indicating "stop" |
-#define STOP_STATE 0 |
+constexpr int32_t STOP_STATE = 0; |
UOBJECT_DEFINE_RTTI_IMPLEMENTATION(RuleBasedBreakIterator) |
@@ -62,9 +64,8 @@ |
* Constructs a RuleBasedBreakIterator that uses the already-created |
* tables object that is passed in as a parameter. |
*/ |
-RuleBasedBreakIterator::RuleBasedBreakIterator(RBBIDataHeader* data, UErrorCode &status) |
-{ |
- init(); |
+RuleBasedBreakIterator::RuleBasedBreakIterator(RBBIDataHeader* data, UErrorCode &status) { |
+ init(status); |
fData = new RBBIDataWrapper(data, status); // status checked in constructor |
if (U_FAILURE(status)) {return;} |
if(fData == 0) { |
@@ -80,7 +81,7 @@ |
RuleBasedBreakIterator::RuleBasedBreakIterator(const uint8_t *compiledRules, |
uint32_t ruleLength, |
UErrorCode &status) { |
- init(); |
+ init(status); |
if (U_FAILURE(status)) { |
return; |
} |
@@ -110,7 +111,7 @@ |
//------------------------------------------------------------------------------- |
RuleBasedBreakIterator::RuleBasedBreakIterator(UDataMemory* udm, UErrorCode &status) |
{ |
- init(); |
+ init(status); |
fData = new RBBIDataWrapper(udm, status); // status checked in constructor |
if (U_FAILURE(status)) {return;} |
if(fData == 0) { |
@@ -130,7 +131,7 @@ |
UParseError &parseError, |
UErrorCode &status) |
{ |
- init(); |
+ init(status); |
if (U_FAILURE(status)) {return;} |
RuleBasedBreakIterator *bi = (RuleBasedBreakIterator *) |
RBBIRuleBuilder::createRuleBasedBreakIterator(rules, &parseError, status); |
@@ -152,7 +153,8 @@ |
// of rules. |
//------------------------------------------------------------------------------- |
RuleBasedBreakIterator::RuleBasedBreakIterator() { |
- init(); |
+ UErrorCode status = U_ZERO_ERROR; |
+ init(status); |
} |
@@ -165,7 +167,8 @@ |
RuleBasedBreakIterator::RuleBasedBreakIterator(const RuleBasedBreakIterator& other) |
: BreakIterator(other) |
{ |
- this->init(); |
+ UErrorCode status = U_ZERO_ERROR; |
+ this->init(status); |
*this = other; |
} |
@@ -180,7 +183,7 @@ |
} |
fCharIter = NULL; |
delete fSCharIter; |
- fCharIter = NULL; |
+ fSCharIter = NULL; |
delete fDCharIter; |
fDCharIter = NULL; |
@@ -190,18 +193,17 @@ |
fData->removeReference(); |
fData = NULL; |
} |
- if (fCachedBreakPositions) { |
- uprv_free(fCachedBreakPositions); |
- fCachedBreakPositions = NULL; |
- } |
- if (fLanguageBreakEngines) { |
- delete fLanguageBreakEngines; |
- fLanguageBreakEngines = NULL; |
- } |
- if (fUnhandledBreakEngine) { |
- delete fUnhandledBreakEngine; |
- fUnhandledBreakEngine = NULL; |
- } |
+ delete fBreakCache; |
+ fBreakCache = NULL; |
+ |
+ delete fDictionaryCache; |
+ fDictionaryCache = NULL; |
+ |
+ delete fLanguageBreakEngines; |
+ fLanguageBreakEngines = NULL; |
+ |
+ delete fUnhandledBreakEngine; |
+ fUnhandledBreakEngine = NULL; |
} |
/** |
@@ -215,7 +217,6 @@ |
} |
BreakIterator::operator=(that); |
- reset(); // Delete break cache information |
fBreakType = that.fBreakType; |
if (fLanguageBreakEngines != NULL) { |
delete fLanguageBreakEngines; |
@@ -245,6 +246,17 @@ |
fData = that.fData->addReference(); |
} |
+ fPosition = that.fPosition; |
+ fRuleStatusIndex = that.fRuleStatusIndex; |
+ fDone = that.fDone; |
+ |
+ // TODO: both the dictionary and the main cache need to be copied. |
+ // Current position could be within a dictionary range. Trying to continue |
+ // the iteration without the caches present would go to the rules, with |
+ // the assumption that the current position is on a rule boundary. |
+ fBreakCache->reset(fPosition, fRuleStatusIndex); |
+ fDictionaryCache->reset(); |
+ |
return *this; |
} |
@@ -256,15 +268,15 @@ |
// Initializes all fields, leaving the object in a consistent state. |
// |
//----------------------------------------------------------------------------- |
-void RuleBasedBreakIterator::init() { |
- UErrorCode status = U_ZERO_ERROR; |
- fText = utext_openUChars(NULL, NULL, 0, &status); |
+void RuleBasedBreakIterator::init(UErrorCode &status) { |
+ fText = NULL; |
fCharIter = NULL; |
fSCharIter = NULL; |
fDCharIter = NULL; |
fData = NULL; |
- fLastRuleStatusIndex = 0; |
- fLastStatusIndexValid = TRUE; |
+ fPosition = 0; |
+ fRuleStatusIndex = 0; |
+ fDone = false; |
fDictionaryCharCount = 0; |
fBreakType = UBRK_WORD; // Defaulting BreakType to word gives reasonable |
// dictionary behavior for Break Iterators that are |
@@ -271,18 +283,28 @@ |
// built from rules. Even better would be the ability to |
// declare the type in the rules. |
- fCachedBreakPositions = NULL; |
- fLanguageBreakEngines = NULL; |
- fUnhandledBreakEngine = NULL; |
- fNumCachedBreakPositions = 0; |
- fPositionInCache = 0; |
+ fLanguageBreakEngines = NULL; |
+ fUnhandledBreakEngine = NULL; |
+ fBreakCache = NULL; |
+ fDictionaryCache = NULL; |
+ if (U_FAILURE(status)) { |
+ return; |
+ } |
+ |
+ fText = utext_openUChars(NULL, NULL, 0, &status); |
+ fDictionaryCache = new DictionaryCache(this, status); |
+ fBreakCache = new BreakCache(this, status); |
+ if (U_SUCCESS(status) && (fText == NULL || fDictionaryCache == NULL || fBreakCache == NULL)) { |
+ status = U_MEMORY_ALLOCATION_ERROR; |
+ } |
+ |
#ifdef RBBI_DEBUG |
static UBool debugInitDone = FALSE; |
if (debugInitDone == FALSE) { |
char *debugEnv = getenv("U_RBBIDEBUG"); |
if (debugEnv && uprv_strstr(debugEnv, "trace")) { |
- fTrace = TRUE; |
+ gTrace = TRUE; |
} |
debugInitDone = TRUE; |
} |
@@ -312,6 +334,9 @@ |
if (typeid(*this) != typeid(that)) { |
return FALSE; |
} |
+ if (this == &that) { |
+ return TRUE; |
+ } |
// The base class BreakIterator carries no state that participates in equality, |
// and does not implement an equality function that would otherwise be |
@@ -326,6 +351,12 @@ |
return FALSE; |
}; |
+ if (!(fPosition == that2.fPosition && |
+ fRuleStatusIndex == that2.fRuleStatusIndex && |
+ fDone == that2.fDone)) { |
+ return FALSE; |
+ } |
+ |
if (that2.fData == fData || |
(fData != NULL && that2.fData != NULL && *that2.fData == *fData)) { |
// The two break iterators are using the same rules. |
@@ -352,7 +383,8 @@ |
if (U_FAILURE(status)) { |
return; |
} |
- reset(); |
+ fBreakCache->reset(); |
+ fDictionaryCache->reset(); |
fText = utext_clone(fText, ut, FALSE, TRUE, &status); |
// Set up a dummy CharacterIterator to be returned if anyone |
@@ -413,7 +445,8 @@ |
fCharIter = newText; |
UErrorCode status = U_ZERO_ERROR; |
- reset(); |
+ fBreakCache->reset(); |
+ fDictionaryCache->reset(); |
if (newText==NULL || newText->startIndex() != 0) { |
// startIndex !=0 wants to be an error, but there's no way to report it. |
// Make the iterator text be an empty string. |
@@ -432,7 +465,8 @@ |
void |
RuleBasedBreakIterator::setText(const UnicodeString& newText) { |
UErrorCode status = U_ZERO_ERROR; |
- reset(); |
+ fBreakCache->reset(); |
+ fDictionaryCache->reset(); |
fText = utext_openConstUnicodeString(fText, &newText, &status); |
// Set up a character iterator on the string. |
@@ -492,13 +526,12 @@ |
* @return The new iterator position, which is zero. |
*/ |
int32_t RuleBasedBreakIterator::first(void) { |
- reset(); |
- fLastRuleStatusIndex = 0; |
- fLastStatusIndexValid = TRUE; |
- //if (fText == NULL) |
- // return BreakIterator::DONE; |
- |
- utext_setNativeIndex(fText, 0); |
+ UErrorCode status = U_ZERO_ERROR; |
+ if (!fBreakCache->seek(0)) { |
+ fBreakCache->populateNear(0, status); |
+ } |
+ fBreakCache->current(); |
+ U_ASSERT(fPosition == 0); |
return 0; |
} |
@@ -507,17 +540,12 @@ |
* @return The text's past-the-end offset. |
*/ |
int32_t RuleBasedBreakIterator::last(void) { |
- reset(); |
- if (fText == NULL) { |
- fLastRuleStatusIndex = 0; |
- fLastStatusIndexValid = TRUE; |
- return BreakIterator::DONE; |
- } |
- |
- fLastStatusIndexValid = FALSE; |
- int32_t pos = (int32_t)utext_nativeLength(fText); |
- utext_setNativeIndex(fText, pos); |
- return pos; |
+ int32_t endPos = (int32_t)utext_nativeLength(fText); |
+ UBool endShouldBeBoundary = isBoundary(endPos); // Has side effect of setting iterator position. |
+ (void)endShouldBeBoundary; |
+ U_ASSERT(endShouldBeBoundary); |
+ U_ASSERT(fPosition == endPos); |
+ return endPos; |
} |
/** |
@@ -530,15 +558,18 @@ |
* the current one. |
*/ |
int32_t RuleBasedBreakIterator::next(int32_t n) { |
- int32_t result = current(); |
- while (n > 0) { |
- result = next(); |
- --n; |
+ int32_t result = 0; |
+ if (n > 0) { |
+ for (; n > 0 && result != UBRK_DONE; --n) { |
+ result = next(); |
+ } |
+ } else if (n < 0) { |
+ for (; n < 0 && result != UBRK_DONE; ++n) { |
+ result = previous(); |
+ } |
+ } else { |
+ result = current(); |
} |
- while (n < 0) { |
- result = previous(); |
- ++n; |
- } |
return result; |
} |
@@ -547,346 +578,67 @@ |
* @return The position of the first boundary after this one. |
*/ |
int32_t RuleBasedBreakIterator::next(void) { |
- // if we have cached break positions and we're still in the range |
- // covered by them, just move one step forward in the cache |
- if (fCachedBreakPositions != NULL) { |
- if (fPositionInCache < fNumCachedBreakPositions - 1) { |
- ++fPositionInCache; |
- int32_t pos = fCachedBreakPositions[fPositionInCache]; |
- utext_setNativeIndex(fText, pos); |
- return pos; |
- } |
- else { |
- reset(); |
- } |
- } |
- |
- int32_t startPos = current(); |
- fDictionaryCharCount = 0; |
- int32_t result = handleNext(fData->fForwardTable); |
- if (fDictionaryCharCount > 0) { |
- result = checkDictionary(startPos, result, FALSE); |
- } |
- return result; |
+ fBreakCache->next(); |
+ return fDone ? UBRK_DONE : fPosition; |
} |
/** |
- * Advances the iterator backwards, to the last boundary preceding this one. |
- * @return The position of the last boundary position preceding this one. |
+ * Move the iterator backwards, to the boundary preceding the current one. |
+ * |
+ * Starts from the current position within fText. |
+ * Starting position need not be on a boundary. |
+ * |
+ * @return The position of the boundary position immediately preceding the starting position. |
*/ |
int32_t RuleBasedBreakIterator::previous(void) { |
- int32_t result; |
- int32_t startPos; |
- |
- // if we have cached break positions and we're still in the range |
- // covered by them, just move one step backward in the cache |
- if (fCachedBreakPositions != NULL) { |
- if (fPositionInCache > 0) { |
- --fPositionInCache; |
- // If we're at the beginning of the cache, need to reevaluate the |
- // rule status |
- if (fPositionInCache <= 0) { |
- fLastStatusIndexValid = FALSE; |
- } |
- int32_t pos = fCachedBreakPositions[fPositionInCache]; |
- utext_setNativeIndex(fText, pos); |
- return pos; |
- } |
- else { |
- reset(); |
- } |
- } |
- |
- // if we're already sitting at the beginning of the text, return DONE |
- if (fText == NULL || (startPos = current()) == 0) { |
- fLastRuleStatusIndex = 0; |
- fLastStatusIndexValid = TRUE; |
- return BreakIterator::DONE; |
- } |
- |
- if (fData->fSafeRevTable != NULL || fData->fSafeFwdTable != NULL) { |
- result = handlePrevious(fData->fReverseTable); |
- if (fDictionaryCharCount > 0) { |
- result = checkDictionary(result, startPos, TRUE); |
- } |
- return result; |
- } |
- |
- // old rule syntax |
- // set things up. handlePrevious() will back us up to some valid |
- // break position before the current position (we back our internal |
- // iterator up one step to prevent handlePrevious() from returning |
- // the current position), but not necessarily the last one before |
- // where we started |
- |
- int32_t start = current(); |
- |
- (void)UTEXT_PREVIOUS32(fText); |
- int32_t lastResult = handlePrevious(fData->fReverseTable); |
- if (lastResult == UBRK_DONE) { |
- lastResult = 0; |
- utext_setNativeIndex(fText, 0); |
- } |
- result = lastResult; |
- int32_t lastTag = 0; |
- UBool breakTagValid = FALSE; |
- |
- // iterate forward from the known break position until we pass our |
- // starting point. The last break position before the starting |
- // point is our return value |
- |
- for (;;) { |
- result = next(); |
- if (result == BreakIterator::DONE || result >= start) { |
- break; |
- } |
- lastResult = result; |
- lastTag = fLastRuleStatusIndex; |
- breakTagValid = TRUE; |
- } |
- |
- // fLastBreakTag wants to have the value for section of text preceding |
- // the result position that we are to return (in lastResult.) If |
- // the backwards rules overshot and the above loop had to do two or more |
- // next()s to move up to the desired return position, we will have a valid |
- // tag value. But, if handlePrevious() took us to exactly the correct result position, |
- // we wont have a tag value for that position, which is only set by handleNext(). |
- |
- // Set the current iteration position to be the last break position |
- // before where we started, and then return that value. |
- utext_setNativeIndex(fText, lastResult); |
- fLastRuleStatusIndex = lastTag; // for use by getRuleStatus() |
- fLastStatusIndexValid = breakTagValid; |
- |
- // No need to check the dictionary; it will have been handled by |
- // next() |
- |
- return lastResult; |
+ UErrorCode status = U_ZERO_ERROR; |
+ fBreakCache->previous(status); |
+ return fDone ? UBRK_DONE : fPosition; |
} |
/** |
* Sets the iterator to refer to the first boundary position following |
* the specified position. |
- * @offset The position from which to begin searching for a break position. |
+ * @param startPos The position from which to begin searching for a break position. |
* @return The position of the first break after the current position. |
*/ |
-int32_t RuleBasedBreakIterator::following(int32_t offset) { |
- // if the offset passed in is already past the end of the text, |
- // just return DONE; if it's before the beginning, return the |
+int32_t RuleBasedBreakIterator::following(int32_t startPos) { |
+ // if the supplied position is before the beginning, return the |
// text's starting offset |
- if (fText == NULL || offset >= utext_nativeLength(fText)) { |
- last(); |
- return next(); |
- } |
- else if (offset < 0) { |
+ if (startPos < 0) { |
return first(); |
} |
// Move requested offset to a code point start. It might be on a trail surrogate, |
- // or on a trail byte if the input is UTF-8. |
- utext_setNativeIndex(fText, offset); |
- offset = (int32_t)utext_getNativeIndex(fText); |
+ // or on a trail byte if the input is UTF-8. Or it may be beyond the end of the text. |
+ utext_setNativeIndex(fText, startPos); |
+ startPos = (int32_t)utext_getNativeIndex(fText); |
- // if we have cached break positions and offset is in the range |
- // covered by them, use them |
- // TODO: could use binary search |
- // TODO: what if offset is outside range, but break is not? |
- if (fCachedBreakPositions != NULL) { |
- if (offset >= fCachedBreakPositions[0] |
- && offset < fCachedBreakPositions[fNumCachedBreakPositions - 1]) { |
- fPositionInCache = 0; |
- // We are guaranteed not to leave the array due to range test above |
- while (offset >= fCachedBreakPositions[fPositionInCache]) { |
- ++fPositionInCache; |
- } |
- int32_t pos = fCachedBreakPositions[fPositionInCache]; |
- utext_setNativeIndex(fText, pos); |
- return pos; |
- } |
- else { |
- reset(); |
- } |
- } |
- |
- // Set our internal iteration position (temporarily) |
- // to the position passed in. If this is the _beginning_ position, |
- // then we can just use next() to get our return value |
- |
- int32_t result = 0; |
- |
- if (fData->fSafeRevTable != NULL) { |
- // new rule syntax |
- utext_setNativeIndex(fText, offset); |
- // move forward one codepoint to prepare for moving back to a |
- // safe point. |
- // this handles offset being between a supplementary character |
- // TODO: is this still needed, with move to code point boundary handled above? |
- (void)UTEXT_NEXT32(fText); |
- // handlePrevious will move most of the time to < 1 boundary away |
- handlePrevious(fData->fSafeRevTable); |
- int32_t result = next(); |
- while (result <= offset) { |
- result = next(); |
- } |
- return result; |
- } |
- if (fData->fSafeFwdTable != NULL) { |
- // backup plan if forward safe table is not available |
- utext_setNativeIndex(fText, offset); |
- (void)UTEXT_PREVIOUS32(fText); |
- // handle next will give result >= offset |
- handleNext(fData->fSafeFwdTable); |
- // previous will give result 0 or 1 boundary away from offset, |
- // most of the time |
- // we have to |
- int32_t oldresult = previous(); |
- while (oldresult > offset) { |
- int32_t result = previous(); |
- if (result <= offset) { |
- return oldresult; |
- } |
- oldresult = result; |
- } |
- int32_t result = next(); |
- if (result <= offset) { |
- return next(); |
- } |
- return result; |
- } |
- // otherwise, we have to sync up first. Use handlePrevious() to back |
- // up to a known break position before the specified position (if |
- // we can determine that the specified position is a break position, |
- // we don't back up at all). This may or may not be the last break |
- // position at or before our starting position. Advance forward |
- // from here until we've passed the starting position. The position |
- // we stop on will be the first break position after the specified one. |
- // old rule syntax |
- |
- utext_setNativeIndex(fText, offset); |
- if (offset==0 || |
- (offset==1 && utext_getNativeIndex(fText)==0)) { |
- return next(); |
- } |
- result = previous(); |
- |
- while (result != BreakIterator::DONE && result <= offset) { |
- result = next(); |
- } |
- |
- return result; |
+ UErrorCode status = U_ZERO_ERROR; |
+ fBreakCache->following(startPos, status); |
+ return fDone ? UBRK_DONE : fPosition; |
} |
/** |
* Sets the iterator to refer to the last boundary position before the |
* specified position. |
- * @offset The position to begin searching for a break from. |
+ * @param offset The position to begin searching for a break from. |
* @return The position of the last boundary before the starting position. |
*/ |
int32_t RuleBasedBreakIterator::preceding(int32_t offset) { |
- // if the offset passed in is already past the end of the text, |
- // just return DONE; if it's before the beginning, return the |
- // text's starting offset |
if (fText == NULL || offset > utext_nativeLength(fText)) { |
return last(); |
} |
- else if (offset < 0) { |
- return first(); |
- } |
// Move requested offset to a code point start. It might be on a trail surrogate, |
// or on a trail byte if the input is UTF-8. |
+ |
utext_setNativeIndex(fText, offset); |
- offset = (int32_t)utext_getNativeIndex(fText); |
+ int32_t adjustedOffset = utext_getNativeIndex(fText); |
- // if we have cached break positions and offset is in the range |
- // covered by them, use them |
- if (fCachedBreakPositions != NULL) { |
- // TODO: binary search? |
- // TODO: What if offset is outside range, but break is not? |
- if (offset > fCachedBreakPositions[0] |
- && offset <= fCachedBreakPositions[fNumCachedBreakPositions - 1]) { |
- fPositionInCache = 0; |
- while (fPositionInCache < fNumCachedBreakPositions |
- && offset > fCachedBreakPositions[fPositionInCache]) |
- ++fPositionInCache; |
- --fPositionInCache; |
- // If we're at the beginning of the cache, need to reevaluate the |
- // rule status |
- if (fPositionInCache <= 0) { |
- fLastStatusIndexValid = FALSE; |
- } |
- utext_setNativeIndex(fText, fCachedBreakPositions[fPositionInCache]); |
- return fCachedBreakPositions[fPositionInCache]; |
- } |
- else { |
- reset(); |
- } |
- } |
- |
- // if we start by updating the current iteration position to the |
- // position specified by the caller, we can just use previous() |
- // to carry out this operation |
- |
- if (fData->fSafeFwdTable != NULL) { |
- // new rule syntax |
- utext_setNativeIndex(fText, offset); |
- int32_t newOffset = (int32_t)UTEXT_GETNATIVEINDEX(fText); |
- if (newOffset != offset) { |
- // Will come here if specified offset was not a code point boundary AND |
- // the underlying implmentation is using UText, which snaps any non-code-point-boundary |
- // indices to the containing code point. |
- // For breakitereator::preceding only, these non-code-point indices need to be moved |
- // up to refer to the following codepoint. |
- (void)UTEXT_NEXT32(fText); |
- offset = (int32_t)UTEXT_GETNATIVEINDEX(fText); |
- } |
- |
- // TODO: (synwee) would it be better to just check for being in the middle of a surrogate pair, |
- // rather than adjusting the position unconditionally? |
- // (Change would interact with safe rules.) |
- // TODO: change RBBI behavior for off-boundary indices to match that of UText? |
- // affects only preceding(), seems cleaner, but is slightly different. |
- (void)UTEXT_PREVIOUS32(fText); |
- handleNext(fData->fSafeFwdTable); |
- int32_t result = (int32_t)UTEXT_GETNATIVEINDEX(fText); |
- while (result >= offset) { |
- result = previous(); |
- } |
- return result; |
- } |
- if (fData->fSafeRevTable != NULL) { |
- // backup plan if forward safe table is not available |
- // TODO: check whether this path can be discarded |
- // It's probably OK to say that rules must supply both safe tables |
- // if they use safe tables at all. We have certainly never described |
- // to anyone how to work with just one safe table. |
- utext_setNativeIndex(fText, offset); |
- (void)UTEXT_NEXT32(fText); |
- |
- // handle previous will give result <= offset |
- handlePrevious(fData->fSafeRevTable); |
- |
- // next will give result 0 or 1 boundary away from offset, |
- // most of the time |
- // we have to |
- int32_t oldresult = next(); |
- while (oldresult < offset) { |
- int32_t result = next(); |
- if (result >= offset) { |
- return oldresult; |
- } |
- oldresult = result; |
- } |
- int32_t result = previous(); |
- if (result >= offset) { |
- return previous(); |
- } |
- return result; |
- } |
- |
- // old rule syntax |
- utext_setNativeIndex(fText, offset); |
- return previous(); |
+ UErrorCode status = U_ZERO_ERROR; |
+ fBreakCache->preceding(adjustedOffset, status); |
+ return fDone ? UBRK_DONE : fPosition; |
} |
/** |
@@ -893,21 +645,11 @@ |
* Returns true if the specfied position is a boundary position. As a side |
* effect, leaves the iterator pointing to the first boundary position at |
* or after "offset". |
+ * |
* @param offset the offset to check. |
* @return True if "offset" is a boundary position. |
*/ |
UBool RuleBasedBreakIterator::isBoundary(int32_t offset) { |
- // the beginning index of the iterator is always a boundary position by definition |
- if (offset == 0) { |
- first(); // For side effects on current position, tag values. |
- return TRUE; |
- } |
- |
- if (offset == (int32_t)utext_nativeLength(fText)) { |
- last(); // For side effects on current position, tag values. |
- return TRUE; |
- } |
- |
// out-of-range indexes are never boundary positions |
if (offset < 0) { |
first(); // For side effects on current position, tag values. |
@@ -914,29 +656,42 @@ |
return FALSE; |
} |
- if (offset > utext_nativeLength(fText)) { |
- last(); // For side effects on current position, tag values. |
+ // Adjust offset to be on a code point boundary and not beyond the end of the text. |
+ // Note that isBoundary() is always be false for offsets that are not on code point boundaries. |
+ // But we still need the side effect of leaving iteration at the following boundary. |
+ |
+ utext_setNativeIndex(fText, offset); |
+ int32_t adjustedOffset = utext_getNativeIndex(fText); |
+ |
+ bool result = false; |
+ UErrorCode status = U_ZERO_ERROR; |
+ if (fBreakCache->seek(adjustedOffset) || fBreakCache->populateNear(adjustedOffset, status)) { |
+ result = (fBreakCache->current() == offset); |
+ } |
+ |
+ if (result && adjustedOffset < offset && utext_char32At(fText, offset) == U_SENTINEL) { |
+ // Original offset is beyond the end of the text. Return FALSE, it's not a boundary, |
+ // but the iteration position remains set to the end of the text, which is a boundary. |
return FALSE; |
} |
- |
- // otherwise, we can use following() on the position before the specified |
- // one and return true if the position we get back is the one the user |
- // specified |
- utext_previous32From(fText, offset); |
- int32_t backOne = (int32_t)UTEXT_GETNATIVEINDEX(fText); |
- UBool result = following(backOne) == offset; |
+ if (!result) { |
+ // Not on a boundary. isBoundary() must leave iterator on the following boundary. |
+ // Cache->seek(), above, left us on the preceding boundary, so advance one. |
+ next(); |
+ } |
return result; |
} |
+ |
/** |
* Returns the current iteration position. |
* @return The current iteration position. |
*/ |
int32_t RuleBasedBreakIterator::current(void) const { |
- int32_t pos = (int32_t)UTEXT_GETNATIVEINDEX(fText); |
- return pos; |
+ return fPosition; |
} |
+ |
//======================================================================= |
// implementation |
//======================================================================= |
@@ -1003,15 +758,11 @@ |
//----------------------------------------------------------------------------------- |
// |
-// handleNext(stateTable) |
-// This method is the actual implementation of the rbbi next() method. |
-// This method initializes the state machine to state 1 |
-// and advances through the text character by character until we reach the end |
-// of the text or the state machine transitions to state 0. We update our return |
-// value every time the state machine passes through an accepting state. |
+// handleNext() |
+// Run the state machine to find a boundary |
// |
//----------------------------------------------------------------------------------- |
-int32_t RuleBasedBreakIterator::handleNext(const RBBIStateTable *statetable) { |
+int32_t RuleBasedBreakIterator::handleNext() { |
int32_t state; |
uint16_t category = 0; |
RBBIRunMode mode; |
@@ -1021,25 +772,29 @@ |
LookAheadResults lookAheadMatches; |
int32_t result = 0; |
int32_t initialPosition = 0; |
+ const RBBIStateTable *statetable = fData->fForwardTable; |
const char *tableData = statetable->fTableData; |
uint32_t tableRowLen = statetable->fRowLen; |
- |
#ifdef RBBI_DEBUG |
- if (fTrace) { |
+ if (gTrace) { |
RBBIDebugPuts("Handle Next pos char state category"); |
} |
#endif |
- // No matter what, handleNext alway correctly sets the break tag value. |
- fLastStatusIndexValid = TRUE; |
- fLastRuleStatusIndex = 0; |
+ // handleNext alway sets the break tag value. |
+ // Set the default for it. |
+ fRuleStatusIndex = 0; |
+ fDictionaryCharCount = 0; |
+ |
// if we're already at the end of the text, return DONE. |
- initialPosition = (int32_t)UTEXT_GETNATIVEINDEX(fText); |
+ initialPosition = fPosition; |
+ UTEXT_SETNATIVEINDEX(fText, initialPosition); |
result = initialPosition; |
c = UTEXT_NEXT32(fText); |
- if (fData == NULL || c==U_SENTINEL) { |
- return BreakIterator::DONE; |
+ if (c==U_SENTINEL) { |
+ fDone = TRUE; |
+ return UBRK_DONE; |
} |
// Set the initial state for the state machine |
@@ -1086,7 +841,7 @@ |
category = UTRIE2_GET16(fData->fTrie, c); |
// Check the dictionary bit in the character's category. |
- // Counter is only used by dictionary based iterators (subclasses). |
+ // Counter is only used by dictionary based iteration. |
// Chars that need to be handled by a dictionary have a flag bit set |
// in their category values. |
// |
@@ -1098,7 +853,7 @@ |
} |
#ifdef RBBI_DEBUG |
- if (fTrace) { |
+ if (gTrace) { |
RBBIDebugPrintf(" %4ld ", utext_getNativeIndex(fText)); |
if (0x20<=c && c<0x7f) { |
RBBIDebugPrintf("\"%c\" ", c); |
@@ -1127,7 +882,7 @@ |
if (mode != RBBI_START) { |
result = (int32_t)UTEXT_GETNATIVEINDEX(fText); |
} |
- fLastRuleStatusIndex = row->fTagIdx; // Remember the break status (tag) values. |
+ fRuleStatusIndex = row->fTagIdx; // Remember the break status (tag) values. |
} |
int16_t completedRule = row->fAccepting; |
@@ -1135,8 +890,8 @@ |
// Lookahead match is completed. |
int32_t lookaheadResult = lookAheadMatches.getPosition(completedRule); |
if (lookaheadResult >= 0) { |
- fLastRuleStatusIndex = row->fTagIdx; |
- UTEXT_SETNATIVEINDEX(fText, lookaheadResult); |
+ fRuleStatusIndex = row->fTagIdx; |
+ fPosition = lookaheadResult; |
return lookaheadResult; |
} |
} |
@@ -1165,8 +920,6 @@ |
mode = RBBI_RUN; |
} |
} |
- |
- |
} |
// The state machine is done. Check whether it found a match... |
@@ -1175,15 +928,16 @@ |
// (This really indicates a defect in the break rules. They should always match |
// at least one character.) |
if (result == initialPosition) { |
- UTEXT_SETNATIVEINDEX(fText, initialPosition); |
- UTEXT_NEXT32(fText); |
- result = (int32_t)UTEXT_GETNATIVEINDEX(fText); |
+ utext_setNativeIndex(fText, initialPosition); |
+ utext_next32(fText); |
+ result = (int32_t)utext_getNativeIndex(fText); |
+ fRuleStatusIndex = 0; |
} |
// Leave the iterator at our result position. |
- UTEXT_SETNATIVEINDEX(fText, result); |
+ fPosition = result; |
#ifdef RBBI_DEBUG |
- if (fTrace) { |
+ if (gTrace) { |
RBBIDebugPrintf("result = %d\n\n", result); |
} |
#endif |
@@ -1196,13 +950,11 @@ |
// |
// handlePrevious() |
// |
-// Iterate backwards, according to the logic of the reverse rules. |
-// This version handles the exact style backwards rules. |
-// |
+// Iterate backwards using the safe reverse rules. |
// The logic of this function is very similar to handleNext(), above. |
// |
//----------------------------------------------------------------------------------- |
-int32_t RuleBasedBreakIterator::handlePrevious(const RBBIStateTable *statetable) { |
+int32_t RuleBasedBreakIterator::handlePrevious(int32_t fromPosition) { |
int32_t state; |
uint16_t category = 0; |
RBBIRunMode mode; |
@@ -1212,19 +964,14 @@ |
int32_t result = 0; |
int32_t initialPosition = 0; |
+ const RBBIStateTable *stateTable = fData->fSafeRevTable; |
+ UTEXT_SETNATIVEINDEX(fText, fromPosition); |
#ifdef RBBI_DEBUG |
- if (fTrace) { |
+ if (gTrace) { |
RBBIDebugPuts("Handle Previous pos char state category"); |
} |
#endif |
- // handlePrevious() never gets the rule status. |
- // Flag the status as invalid; if the user ever asks for status, we will need |
- // to back up, then re-find the break position using handleNext(), which does |
- // get the status value. |
- fLastStatusIndexValid = FALSE; |
- fLastRuleStatusIndex = 0; |
- |
// if we're already at the start of the text, return DONE. |
if (fText == NULL || fData == NULL || UTEXT_GETNATIVEINDEX(fText)==0) { |
return BreakIterator::DONE; |
@@ -1238,10 +985,10 @@ |
// Set the initial state for the state machine |
state = START_STATE; |
row = (RBBIStateTableRow *) |
- (statetable->fTableData + (statetable->fRowLen * state)); |
+ (stateTable->fTableData + (stateTable->fRowLen * state)); |
category = 3; |
mode = RBBI_RUN; |
- if (statetable->fFlags & RBBI_BOF_REQUIRED) { |
+ if (stateTable->fFlags & RBBI_BOF_REQUIRED) { |
category = 2; |
mode = RBBI_START; |
} |
@@ -1256,12 +1003,6 @@ |
// We have already run the loop one last time with the |
// character set to the psueudo {eof} value. Now it is time |
// to unconditionally bail out. |
- if (result == initialPosition) { |
- // Ran off start, no match found. |
- // move one index one (towards the start, since we are doing a previous()) |
- UTEXT_SETNATIVEINDEX(fText, initialPosition); |
- (void)UTEXT_PREVIOUS32(fText); // TODO: shouldn't be necessary. We're already at beginning. Check. |
- } |
break; |
} |
// Run the loop one last time with the fake end-of-input character category. |
@@ -1280,22 +1021,13 @@ |
// Note: the 16 in UTRIE_GET16 refers to the size of the data being returned, |
// not the size of the character going in, which is a UChar32. |
// |
+ // And off the dictionary flag bit. For reverse iteration it is not used. |
category = UTRIE2_GET16(fData->fTrie, c); |
- |
- // Check the dictionary bit in the character's category. |
- // Counter is only used by dictionary based iterators (subclasses). |
- // Chars that need to be handled by a dictionary have a flag bit set |
- // in their category values. |
- // |
- if ((category & 0x4000) != 0) { |
- fDictionaryCharCount++; |
- // And off the dictionary flag bit. |
- category &= ~0x4000; |
- } |
+ category &= ~0x4000; |
} |
#ifdef RBBI_DEBUG |
- if (fTrace) { |
+ if (gTrace) { |
RBBIDebugPrintf(" %4d ", (int32_t)utext_getNativeIndex(fText)); |
if (0x20<=c && c<0x7f) { |
RBBIDebugPrintf("\"%c\" ", c); |
@@ -1315,7 +1047,7 @@ |
U_ASSERT(category<fData->fHeader->fCatCount); |
state = row->fNextState[category]; /*Not accessing beyond memory*/ |
row = (RBBIStateTableRow *) |
- (statetable->fTableData + (statetable->fRowLen * state)); |
+ (stateTable->fTableData + (stateTable->fRowLen * state)); |
if (row->fAccepting == -1) { |
// Match found, common case. |
@@ -1369,10 +1101,8 @@ |
result = (int32_t)UTEXT_GETNATIVEINDEX(fText); |
} |
- // Leave the iterator at our result position. |
- UTEXT_SETNATIVEINDEX(fText, result); |
#ifdef RBBI_DEBUG |
- if (fTrace) { |
+ if (gTrace) { |
RBBIDebugPrintf("result = %d\n\n", result); |
} |
#endif |
@@ -1380,20 +1110,6 @@ |
} |
-void |
-RuleBasedBreakIterator::reset() |
-{ |
- if (fCachedBreakPositions) { |
- uprv_free(fCachedBreakPositions); |
- } |
- fCachedBreakPositions = NULL; |
- fNumCachedBreakPositions = 0; |
- fDictionaryCharCount = 0; |
- fPositionInCache = 0; |
-} |
- |
- |
- |
//------------------------------------------------------------------------------- |
// |
// getRuleStatus() Return the break rule tag associated with the current |
@@ -1401,46 +1117,14 @@ |
// position by iterating forwards, the value will have been |
// cached by the handleNext() function. |
// |
-// If no cached status value is available, the status is |
-// found by doing a previous() followed by a next(), which |
-// leaves the iterator where it started, and computes the |
-// status while doing the next(). |
-// |
//------------------------------------------------------------------------------- |
-void RuleBasedBreakIterator::makeRuleStatusValid() { |
- if (fLastStatusIndexValid == FALSE) { |
- // No cached status is available. |
- if (fText == NULL || current() == 0) { |
- // At start of text, or there is no text. Status is always zero. |
- fLastRuleStatusIndex = 0; |
- fLastStatusIndexValid = TRUE; |
- } else { |
- // Not at start of text. Find status the tedious way. |
- int32_t pa = current(); |
- previous(); |
- if (fNumCachedBreakPositions > 0) { |
- reset(); // Blow off the dictionary cache |
- } |
- int32_t pb = next(); |
- if (pa != pb) { |
- // note: the if (pa != pb) test is here only to eliminate warnings for |
- // unused local variables on gcc. Logically, it isn't needed. |
- U_ASSERT(pa == pb); |
- } |
- } |
- } |
- U_ASSERT(fLastRuleStatusIndex >= 0 && fLastRuleStatusIndex < fData->fStatusMaxIdx); |
-} |
- |
int32_t RuleBasedBreakIterator::getRuleStatus() const { |
- RuleBasedBreakIterator *nonConstThis = (RuleBasedBreakIterator *)this; |
- nonConstThis->makeRuleStatusValid(); |
// fLastRuleStatusIndex indexes to the start of the appropriate status record |
// (the number of status values.) |
// This function returns the last (largest) of the array of status values. |
- int32_t idx = fLastRuleStatusIndex + fData->fRuleStatusTable[fLastRuleStatusIndex]; |
+ int32_t idx = fRuleStatusIndex + fData->fRuleStatusTable[fRuleStatusIndex]; |
int32_t tagVal = fData->fRuleStatusTable[idx]; |
return tagVal; |
@@ -1447,18 +1131,13 @@ |
} |
- |
- |
int32_t RuleBasedBreakIterator::getRuleStatusVec( |
- int32_t *fillInVec, int32_t capacity, UErrorCode &status) |
-{ |
+ int32_t *fillInVec, int32_t capacity, UErrorCode &status) { |
if (U_FAILURE(status)) { |
return 0; |
} |
- RuleBasedBreakIterator *nonConstThis = (RuleBasedBreakIterator *)this; |
- nonConstThis->makeRuleStatusValid(); |
- int32_t numVals = fData->fRuleStatusTable[fLastRuleStatusIndex]; |
+ int32_t numVals = fData->fRuleStatusTable[fRuleStatusIndex]; |
int32_t numValsToCopy = numVals; |
if (numVals > capacity) { |
status = U_BUFFER_OVERFLOW_ERROR; |
@@ -1466,7 +1145,7 @@ |
} |
int i; |
for (i=0; i<numValsToCopy; i++) { |
- fillInVec[i] = fData->fRuleStatusTable[fLastRuleStatusIndex + i + 1]; |
+ fillInVec[i] = fData->fRuleStatusTable[fRuleStatusIndex + i + 1]; |
} |
return numVals; |
} |
@@ -1514,169 +1193,6 @@ |
return (RuleBasedBreakIterator *)clonedBI; |
} |
- |
-//------------------------------------------------------------------------------- |
-// |
-// checkDictionary This function handles all processing of characters in |
-// the "dictionary" set. It will determine the appropriate |
-// course of action, and possibly set up a cache in the |
-// process. |
-// |
-//------------------------------------------------------------------------------- |
-int32_t RuleBasedBreakIterator::checkDictionary(int32_t startPos, |
- int32_t endPos, |
- UBool reverse) { |
- // Reset the old break cache first. |
- reset(); |
- |
- // note: code segment below assumes that dictionary chars are in the |
- // startPos-endPos range |
- // value returned should be next character in sequence |
- if ((endPos - startPos) <= 1) { |
- return (reverse ? startPos : endPos); |
- } |
- |
- // Starting from the starting point, scan towards the proposed result, |
- // looking for the first dictionary character (which may be the one |
- // we're on, if we're starting in the middle of a range). |
- utext_setNativeIndex(fText, reverse ? endPos : startPos); |
- if (reverse) { |
- UTEXT_PREVIOUS32(fText); |
- } |
- |
- int32_t rangeStart = startPos; |
- int32_t rangeEnd = endPos; |
- |
- uint16_t category; |
- int32_t current; |
- UErrorCode status = U_ZERO_ERROR; |
- UStack breaks(status); |
- int32_t foundBreakCount = 0; |
- UChar32 c = utext_current32(fText); |
- |
- category = UTRIE2_GET16(fData->fTrie, c); |
- |
- // Is the character we're starting on a dictionary character? If so, we |
- // need to back up to include the entire run; otherwise the results of |
- // the break algorithm will differ depending on where we start. Since |
- // the result is cached and there is typically a non-dictionary break |
- // within a small number of words, there should be little performance impact. |
- if (category & 0x4000) { |
- if (reverse) { |
- do { |
- utext_next32(fText); // TODO: recast to work directly with postincrement. |
- c = utext_current32(fText); |
- category = UTRIE2_GET16(fData->fTrie, c); |
- } while (c != U_SENTINEL && (category & 0x4000)); |
- // Back up to the last dictionary character |
- rangeEnd = (int32_t)UTEXT_GETNATIVEINDEX(fText); |
- if (c == U_SENTINEL) { |
- // c = fText->last32(); |
- // TODO: why was this if needed? |
- c = UTEXT_PREVIOUS32(fText); |
- } |
- else { |
- c = UTEXT_PREVIOUS32(fText); |
- } |
- } |
- else { |
- do { |
- c = UTEXT_PREVIOUS32(fText); |
- category = UTRIE2_GET16(fData->fTrie, c); |
- } |
- while (c != U_SENTINEL && (category & 0x4000)); |
- // Back up to the last dictionary character |
- if (c == U_SENTINEL) { |
- // c = fText->first32(); |
- c = utext_current32(fText); |
- } |
- else { |
- utext_next32(fText); |
- c = utext_current32(fText); |
- } |
- rangeStart = (int32_t)UTEXT_GETNATIVEINDEX(fText);; |
- } |
- category = UTRIE2_GET16(fData->fTrie, c); |
- } |
- |
- // Loop through the text, looking for ranges of dictionary characters. |
- // For each span, find the appropriate break engine, and ask it to find |
- // any breaks within the span. |
- // Note: we always do this in the forward direction, so that the break |
- // cache is built in the right order. |
- if (reverse) { |
- utext_setNativeIndex(fText, rangeStart); |
- c = utext_current32(fText); |
- category = UTRIE2_GET16(fData->fTrie, c); |
- } |
- while(U_SUCCESS(status)) { |
- while((current = (int32_t)UTEXT_GETNATIVEINDEX(fText)) < rangeEnd && (category & 0x4000) == 0) { |
- utext_next32(fText); // TODO: tweak for post-increment operation |
- c = utext_current32(fText); |
- category = UTRIE2_GET16(fData->fTrie, c); |
- } |
- if (current >= rangeEnd) { |
- break; |
- } |
- |
- // We now have a dictionary character. Get the appropriate language object |
- // to deal with it. |
- const LanguageBreakEngine *lbe = getLanguageBreakEngine(c); |
- |
- // Ask the language object if there are any breaks. It will leave the text |
- // pointer on the other side of its range, ready to search for the next one. |
- if (lbe != NULL) { |
- foundBreakCount += lbe->findBreaks(fText, rangeStart, rangeEnd, FALSE, fBreakType, breaks); |
- } |
- |
- // Reload the loop variables for the next go-round |
- c = utext_current32(fText); |
- category = UTRIE2_GET16(fData->fTrie, c); |
- } |
- |
- // If we found breaks, build a new break cache. The first and last entries must |
- // be the original starting and ending position. |
- if (foundBreakCount > 0) { |
- U_ASSERT(foundBreakCount == breaks.size()); |
- int32_t totalBreaks = foundBreakCount; |
- if (startPos < breaks.elementAti(0)) { |
- totalBreaks += 1; |
- } |
- if (endPos > breaks.peeki()) { |
- totalBreaks += 1; |
- } |
- fCachedBreakPositions = (int32_t *)uprv_malloc(totalBreaks * sizeof(int32_t)); |
- if (fCachedBreakPositions != NULL) { |
- int32_t out = 0; |
- fNumCachedBreakPositions = totalBreaks; |
- if (startPos < breaks.elementAti(0)) { |
- fCachedBreakPositions[out++] = startPos; |
- } |
- for (int32_t i = 0; i < foundBreakCount; ++i) { |
- fCachedBreakPositions[out++] = breaks.elementAti(i); |
- } |
- if (endPos > fCachedBreakPositions[out-1]) { |
- fCachedBreakPositions[out] = endPos; |
- } |
- // If there are breaks, then by definition, we are replacing the original |
- // proposed break by one of the breaks we found. Use following() and |
- // preceding() to do the work. They should never recurse in this case. |
- if (reverse) { |
- return preceding(endPos); |
- } |
- else { |
- return following(startPos); |
- } |
- } |
- // If the allocation failed, just fall through to the "no breaks found" case. |
- } |
- |
- // If we get here, there were no language-based breaks. Set the text pointer |
- // to the original proposed break. |
- utext_setNativeIndex(fText, reverse ? startPos : endPos); |
- return (reverse ? startPos : endPos); |
-} |
- |
U_NAMESPACE_END |
@@ -1824,9 +1340,11 @@ |
void RuleBasedBreakIterator::setBreakType(int32_t type) { |
fBreakType = type; |
- reset(); |
} |
+void RuleBasedBreakIterator::dumpCache() { |
+ fBreakCache->dumpCache(); |
+} |
/** |
* Returns the description used to create this iterator |