remove special status of partial tokens

Full-word tokens are no longer marked by a space at the
beginning of the token. Use the new Partial token category
instead. This removes a couple of special casing, we don't
really need.

The word table still has the space for compatibility reasons,
so the tokenizer code needs to get rid of it when loading the
tokens.
This commit is contained in:
Sarah Hoffmann
2021-07-14 22:17:17 +02:00
parent 6070c3d1d5
commit 143ff14466
5 changed files with 34 additions and 69 deletions

View File

@@ -355,15 +355,15 @@ class Geocode
$aNewWordsetSearches = array();
foreach ($aWordsetSearches as $oCurrentSearch) {
// Tokens with full name matches.
foreach ($oValidTokens->get(' '.$sToken) as $oSearchTerm) {
$aNewSearches = $oCurrentSearch->extendWithFullTerm(
foreach ($oValidTokens->get($sToken) as $oSearchTerm) {
$aNewSearches = $oCurrentSearch->extendWithSearchTerm(
$sToken,
$oSearchTerm,
$sPhraseType,
$iToken == 0 && $iPhrase == 0,
$iPhrase == 0,
$iToken + 1 == count($aWordset)
&& $iPhrase + 1 == count($aPhrases)
&& $iPhrase + 1 == count($aPhrases),
$iPhrase
);
foreach ($aNewSearches as $oSearch) {
@@ -372,27 +372,6 @@ class Geocode
}
}
}
// Look for partial matches.
// Note that there is no point in adding country terms here
// because country is omitted in the address.
if ($sPhraseType != 'country') {
// Allow searching for a word - but at extra cost
foreach ($oValidTokens->get($sToken) as $oSearchTerm) {
$aNewSearches = $oCurrentSearch->extendWithPartialTerm(
$sToken,
$oSearchTerm,
(bool) $sPhraseType,
$iPhrase,
$oValidTokens->get(' '.$sToken)
);
foreach ($aNewSearches as $oSearch) {
if ($oSearch->getRank() < $this->iMaxRank) {
$aNewWordsetSearches[] = $oSearch;
}
}
}
}
}
// Sort and cut
usort($aNewWordsetSearches, array('Nominatim\SearchDescription', 'bySearchRank'));