hans/Nominatim
1
0
Fork 1
mirror of https://github.com/osm-search/Nominatim.git synced 2026-02-16 15:47:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

remove special status of partial tokens

Browse Source 
Full-word tokens are no longer marked by a space at the
beginning of the token. Use the new Partial token category
instead. This removes a couple of special casing, we don't
really need.

The word table still has the space for compatibility reasons,
so the tokenizer code needs to get rid of it when loading the
tokens.
This commit is contained in:
Sarah Hoffmann
2021-07-14 22:17:17 +02:00
parent 6070c3d1d5
commit 143ff14466
5 changed files with 34 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
lib-php/TokenList.php
View File

@@ -18,15 +18,6 @@ require_once(CONST_LibDir.'/SpecialSearchOperator.php');
* tokens do not have a common base class. All tokens need to have a field
* with the word id that points to an entry in the `word` database table
* but otherwise the information saved about a token can be very different.
*
* There are two different kinds of token words: full words and partial terms.
*
* Full words start with a space. They represent a complete name of a place.
* All special tokens are normally full words.
*
* Partial terms have no space at the beginning. They may represent a part of
* a name of a place (e.g. in the name 'World Trade Center' a partial term
* would be 'Trade' or 'Trade Center'). They are only used in TokenWord.
*/
class TokenList
{
@@ -65,7 +56,7 @@ class TokenList
*/
public function containsAny($sWord)
{
return isset($this->aTokens[$sWord]) || isset($this->aTokens[' '.$sWord]);
return isset($this->aTokens[$sWord]);
}
/**
@@ -87,7 +78,7 @@ class TokenList
foreach ($this->aTokens as $aTokenList) {
foreach ($aTokenList as $oToken) {
if (is_a($oToken, '\Nominatim\Token\Word') && !$oToken->bPartial) {
if (is_a($oToken, '\Nominatim\Token\Word')) {
$ids[$oToken->iId] = $oToken->iId;
}
}