diff --git a/src/nominatim_api/search/db_searches/place_search.py b/src/nominatim_api/search/db_searches/place_search.py
index 3ef223be..e647632d 100644
--- a/src/nominatim_api/search/db_searches/place_search.py
+++ b/src/nominatim_api/search/db_searches/place_search.py
@@ -63,8 +63,7 @@ class PlaceSearch(base.AbstractSearch):
                         t.c.name_vector, t.c.nameaddress_vector,
                         sa.case((t.c.importance > 0, t.c.importance),
                                 else_=0.40001-(sa.cast(t.c.search_rank, sa.Float())/75))
-                          .label('importance'),
-                        penalty.label('penalty'))
+                          .label('importance'))
 
         for lookup in self.lookups:
             sql = sql.where(lookup.sql_condition(t))
@@ -88,10 +87,10 @@ class PlaceSearch(base.AbstractSearch):
                 sql = sql.where(t.c.centroid
                                    .intersects(VIEWBOX_PARAM,
                                                use_index=details.viewbox.area < 0.2))
-            elif not self.postcodes and self.expected_count >= 10000:
-                sql = sql.where(t.c.centroid
-                                   .intersects(VIEWBOX2_PARAM,
-                                               use_index=details.viewbox.area < 0.5))
+            else:
+                penalty += sa.case((t.c.centroid.intersects(VIEWBOX_PARAM, use_index=False), 0.0),
+                                   (t.c.centroid.intersects(VIEWBOX2_PARAM, use_index=False), 0.5),
+                                   else_=1.0)
 
         if details.near is not None and details.near_radius is not None:
             if details.near_radius < 0.1:
@@ -110,6 +109,8 @@ class PlaceSearch(base.AbstractSearch):
             sql = sql.where(sa.or_(t.c.address_rank <= MAX_RANK_PARAM,
                                    t.c.search_rank <= MAX_RANK_PARAM))
 
+        sql = sql.add_columns(penalty.label('penalty'))
+
         inner = sql.limit(5000 if self.qualifiers else 1000)\
                    .order_by(sa.desc(sa.text('importance')))\
                    .subquery()
@@ -121,8 +122,8 @@ class PlaceSearch(base.AbstractSearch):
         # If the query is not an address search or has a geographic preference,
         # preselect most important items to restrict the number of places
         # that need to be looked up in placex.
-        if (details.viewbox is None or details.bounded_viewbox)\
-           and (details.near is None or details.near_radius is not None)\
+        if (details.viewbox is None or not details.bounded_viewbox)\
+           and (details.near is None or details.near_radius is None)\
            and not self.qualifiers:
             sql = sql.add_columns(sa.func.first_value(inner.c.penalty - inner.c.importance)
                                     .over(order_by=inner.c.penalty - inner.c.importance)
@@ -166,11 +167,6 @@ class PlaceSearch(base.AbstractSearch):
                 # The postcode search needs to get priority here.
                 penalty += sa.case((t.c.postcode.in_(self.postcodes.values), 0.0), else_=1.0)
 
-        if details.viewbox is not None and not details.bounded_viewbox:
-            penalty += sa.case((t.c.geometry.intersects(VIEWBOX_PARAM, use_index=False), 0.0),
-                               (t.c.geometry.intersects(VIEWBOX2_PARAM, use_index=False), 0.5),
-                               else_=1.0)
-
         if details.near is not None:
             sql = sql.add_columns((-tsearch.c.centroid.ST_Distance(NEAR_PARAM))
                                   .label('importance'))
diff --git a/test/python/api/search/test_search_places.py b/test/python/api/search/test_search_places.py
index 31d1a778..7ea5d28e 100644
--- a/test/python/api/search/test_search_places.py
+++ b/test/python/api/search/test_search_places.py
@@ -156,9 +156,11 @@ class TestNameOnlySearches:
         assert result.place_id == 333
         assert len(geom['coordinates']) == npoints
 
-    @pytest.mark.parametrize('viewbox', ['5.0,4.0,6.0,5.0', '5.7,4.0,6.0,5.0'])
-    @pytest.mark.parametrize('wcount,rids', [(2, [100, 101]), (20000, [100])])
-    def test_prefer_viewbox(self, apiobj, frontend, viewbox, wcount, rids):
+    @pytest.mark.parametrize('viewbox,rids', [('5.0,4.0,6.0,5.0', [100]),
+                                              ('5.7,4.0,6.0,5.0', [100, 101]),
+                                              ('10.0,10.0,11.0,11.0', [101, 100])])
+    @pytest.mark.parametrize('wcount', [2, 50000])
+    def test_prefer_viewbox(self, apiobj, frontend, viewbox, rids, wcount):
         lookup = FieldLookup('name_vector', [1, 2], LookupAll)
         ranking = FieldRanking('name_vector', 0.2, [RankedTokens(0.0, [21])])