hans/Nominatim
1
0
Fork 1
mirror of https://github.com/osm-search/Nominatim.git synced 2026-02-26 11:08:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'PSR2-fixes-for-wikidata' of https://github.com/mtmail/Nominatim into mtmail-misc-psr-fixes

Browse Source
This commit is contained in:
Sarah Hoffmann
2016-09-15 21:51:48 +02:00
parent 93c26a0e65 b243aa84e2
commit e686ce8680
3 changed files with 95 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
tests-php/bootstrap.php
View File

@@ -1,2 +1 @@
<?php <?php

27
wikidata/parse.php
View File

@@ -20,14 +20,11 @@ if ($hFile) {
while (($sLine = fgets($hFile, 4000000)) !== false) { while (($sLine = fgets($hFile, 4000000)) !== false) {
if (substr($sLine, 0, 11) == ' <title>') { if (substr($sLine, 0, 11) == ' <title>') {
$sTitle = substr($sLine, 11, -9); $sTitle = substr($sLine, 11, -9);
} } elseif (substr($sLine, 0, 8) == ' <ns>') {
else if (substr($sLine, 0, 8) == ' <ns>') {
$iNS = (int)substr($sLine, 8, -6); $iNS = (int)substr($sLine, 8, -6);
} } elseif (substr($sLine, 0, 8) == ' <id>') {
else if (substr($sLine, 0, 8) == ' <id>') {
$iID = (int)substr($sLine, 8, -6); $iID = (int)substr($sLine, 8, -6);
} } elseif (substr($sLine, 0, 33) == ' <text xml:space="preserve">') {
else if (substr($sLine, 0, 33) == ' <text xml:space="preserve">') {
if ($iNS == -2) continue; if ($iNS == -2) continue;
if ($iNS == -1) continue; if ($iNS == -1) continue;
if ($iNS == 1) continue; if ($iNS == 1) continue;
@@ -102,26 +99,24 @@ if ($hFile) {
if (isset($aArticle['claims'])) { if (isset($aArticle['claims'])) {
//
foreach ($aArticle['claims'] as $iClaim => $aClaim) { foreach ($aArticle['claims'] as $iClaim => $aClaim) {
//
$bFail = false; $bFail = false;
if ($aClaim['m'][0] == 'novalue') continue; if ($aClaim['m'][0] == 'novalue') continue;
if ($aClaim['m'][0] == 'somevalue') continue; if ($aClaim['m'][0] == 'somevalue') continue;
$iPID = (int)$aClaim['m'][1]; $iPID = (int)$aClaim['m'][1];
if ($aClaim['m'][0] != 'value') $bFail = true; if ($aClaim['m'][0] != 'value') $bFail = true;
if ($aClaim['m'][2]== 'wikibase-entityid') { if ($aClaim['m'][2]== 'wikibase-entityid') {
//
if ($aClaim['m'][3]['entity-type'] != 'item') $bFail = true; if ($aClaim['m'][3]['entity-type'] != 'item') $bFail = true;
fputcsv($hFileEntityProperty, array($iID, $iClaim, $iPID, null, $aClaim['m'][3]['numeric-id'], null, null)); fputcsv($hFileEntityProperty, array($iID, $iClaim, $iPID, null, $aClaim['m'][3]['numeric-id'], null, null));
// echo "insert into entity_property values (nextval('seq_entity_property'),".$iID.",".$iPID.",null,".$aClaim['m'][3]['numeric-id'].",null);\n"; // echo "insert into entity_property values (nextval('seq_entity_property'),".$iID.",".$iPID.",null,".$aClaim['m'][3]['numeric-id'].",null);\n";
} elseif ($aClaim['m'][2] == 'globecoordinate') { } elseif ($aClaim['m'][2] == 'globecoordinate') {
//
if ($aClaim['m'][3]['globe'] != 'http://www.wikidata.org/entity/Q2') $bFail = true; if ($aClaim['m'][3]['globe'] != 'http://www.wikidata.org/entity/Q2') $bFail = true;
fputcsv($hFileEntityProperty, array($iID, $iClaim, $iPID, null, null, "SRID=4326;POINT(".((float) $aClaim['m'][3]['longitude'])." ".((float)$aClaim['m'][3]['latitude']).")", null)); fputcsv($hFileEntityProperty, array($iID, $iClaim, $iPID, null, null, "SRID=4326;POINT(".((float) $aClaim['m'][3]['longitude'])." ".((float)$aClaim['m'][3]['latitude']).")", null));
// echo "insert into entity_property values (nextval('seq_entity_property'),".$iID.",".$iPID.",null,null,ST_SetSRID(ST_MakePoint(".((float)$aClaim['m'][3]['longitude']).", ".((float)$aClaim['m'][3]['latitude'])."),4326));\n"; // echo "insert into entity_property values (nextval('seq_entity_property'),".$iID.",".$iPID.",null,null,ST_SetSRID(ST_MakePoint(".((float)$aClaim['m'][3]['longitude']).", ".((float)$aClaim['m'][3]['latitude'])."),4326));\n";
} elseif ($aClaim['m'][2] == 'time') { } elseif ($aClaim['m'][2] == 'time') {
// TODO! // TODO!
/* /*
@@ -136,7 +131,6 @@ if ($hFile) {
// $bFail = true; // $bFail = true;
} }
} elseif ( $aClaim['m'][3]['calendarmodel'] != 'http://www.wikidata.org/entity/Q1985786') { } elseif ( $aClaim['m'][3]['calendarmodel'] != 'http://www.wikidata.org/entity/Q1985786') {
/ *
// Julian // Julian
if (preg_match('#(\\+|-)0*([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2}:[0-9]{2}:[0-9]{2})Z#', $aClaim['m'][3]['time'], $aMatch)) { if (preg_match('#(\\+|-)0*([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2}:[0-9]{2}:[0-9]{2})Z#', $aClaim['m'][3]['time'], $aMatch)) {
var_dump($aMatch); var_dump($aMatch);
@@ -148,20 +142,15 @@ var_dump($iDayCount, jdtogregorian($iDayCount));
exit; exit;
} }
exit; exit;
* /
} else { } else {
// $bFail = true; // $bFail = true;
} }
*/ */
} elseif ($aClaim['m'][2] == 'string') { } elseif ($aClaim['m'][2] == 'string') {
// echo "insert into entity_property values (nextval('seq_entity_property'),".$iID.",".$iPID.",'".pg_escape_string($aClaim['m'][3])."',null,null);\n"; // echo "insert into entity_property values (nextval('seq_entity_property'),".$iID.",".$iPID.",'".pg_escape_string($aClaim['m'][3])."',null,null);\n";
fputcsv($hFileEntityProperty, array($iID, $iClaim, $iPID, $aClaim['m'][3], null, null, null)); fputcsv($hFileEntityProperty, array($iID, $iClaim, $iPID, $aClaim['m'][3], null, null, null));
} else { } else {
$bFail = true; $bFail = true;
} }
// Don't care about sources: if ($aClaim['refs'] != array()) $bFail = true; // Don't care about sources: if ($aClaim['refs'] != array()) $bFail = true;
@@ -172,9 +161,7 @@ exit;
} else { } else {
// process // process
} }
} }
} }
} }
} }

44
wikidata/pull_wikipedia_logs.php
View File

@@ -1,17 +1,14 @@
<?php <?php
for($iTimestamp = mktime(0, 0, 0, 5, 1, 2013); $iTimestamp < mktime(0, 0, 0, 6, 15, 2013); $iTimestamp += 24*60*60) for ($iTimestamp = mktime(0, 0, 0, 5, 1, 2013); $iTimestamp < mktime(0, 0, 0, 6, 15, 2013); $iTimestamp += 24*60*60) {
{
$sYear = date("Y", $iTimestamp); $sYear = date("Y", $iTimestamp);
$sMonth = date("Y-m", $iTimestamp); $sMonth = date("Y-m", $iTimestamp);
$sDay = date("Ymd", $iTimestamp); $sDay = date("Ymd", $iTimestamp);
for($iHour = 0; $iHour < 24; $iHour++) for ($iHour = 0; $iHour < 24; $iHour++) {
{
$sFilename = sprintf("pagecounts-".$sDay."-%02d0000", $iHour); $sFilename = sprintf("pagecounts-".$sDay."-%02d0000", $iHour);
echo $sFilename."\n"; echo $sFilename."\n";
if (!file_exists($sFilename.'.gz')) if (!file_exists($sFilename.'.gz')) {
{
exec('wget http://dumps.wikimedia.org/other/pagecounts-raw/'.$sYear.'/'.$sMonth.'/'.$sFilename.'.gz'); exec('wget http://dumps.wikimedia.org/other/pagecounts-raw/'.$sYear.'/'.$sMonth.'/'.$sFilename.'.gz');
} }
@@ -25,57 +22,42 @@ for($iTimestamp = mktime(0, 0, 0, 5, 1, 2013); $iTimestamp < mktime(0, 0, 0, 6,
$sPrevLine = true; $sPrevLine = true;
$sDayLine = true; $sDayLine = true;
do do {
{ if ($sPrevKey === $sDayKey) {
if ($sPrevKey === $sDayKey)
{
if ($sPrevLine !== true) fputs($hNewTotals, "$sPrevKey ".($iPrevValue+$iDayValue)."\n"); if ($sPrevLine !== true) fputs($hNewTotals, "$sPrevKey ".($iPrevValue+$iDayValue)."\n");
$sPrevLine = true; $sPrevLine = true;
$sDayLine = true; $sDayLine = true;
} } elseif ($sDayKey !== false && ($sPrevKey > $sDayKey || $sPrevKey === false)) {
else if ($sDayKey !== false && ($sPrevKey > $sDayKey || $sPrevKey === false))
{
fputs($hNewTotals, "$sDayKey ".($iDayValue)."\n"); fputs($hNewTotals, "$sDayKey ".($iDayValue)."\n");
$sDayLine = true; $sDayLine = true;
} } elseif ($sPrevKey !== false && ($sDayKey > $sPrevKey || $sDayKey === false)) {
else if ($sPrevKey !== false && ($sDayKey > $sPrevKey || $sDayKey === false))
{
fputs($hNewTotals, "$sPrevKey ".($iPrevValue)."\n"); fputs($hNewTotals, "$sPrevKey ".($iPrevValue)."\n");
$sPrevLine = true; $sPrevLine = true;
} }
if ($sPrevLine === true) if ($sPrevLine === true) {
{
$sPrevLine = $hPrevTotals?fgets($hPrevTotals, 4096):false; $sPrevLine = $hPrevTotals?fgets($hPrevTotals, 4096):false;
if ($sPrevLine !== false) if ($sPrevLine !== false) {
{
$aPrevLine = explode(' ', $sPrevLine); $aPrevLine = explode(' ', $sPrevLine);
$sPrevKey = $aPrevLine[0].' '.$aPrevLine[1]; $sPrevKey = $aPrevLine[0].' '.$aPrevLine[1];
$iPrevValue = (int)$aPrevLine[2]; $iPrevValue = (int)$aPrevLine[2];
} } else {
else
{
$sPrevKey = false; $sPrevKey = false;
$iPrevValue = 0; $iPrevValue = 0;
} }
} }
if ($sDayLine === true) if ($sDayLine === true) {
{
$sDayLine = $hDayTotals?fgets($hDayTotals, 4096):false; $sDayLine = $hDayTotals?fgets($hDayTotals, 4096):false;
if ($sDayLine !== false) if ($sDayLine !== false) {
{
preg_match('#^([a-z]{2}) ([^ :]+) ([0-9]+) [0-9]+$#', $sDayLine, $aMatch); preg_match('#^([a-z]{2}) ([^ :]+) ([0-9]+) [0-9]+$#', $sDayLine, $aMatch);
$sDayKey = $aMatch[1].' '.$aMatch[2]; $sDayKey = $aMatch[1].' '.$aMatch[2];
$iDayValue = (int)$aMatch[3]; $iDayValue = (int)$aMatch[3];
} } else {
else
{
$sDayKey = false; $sDayKey = false;
$iDayValue = 0; $iDayValue = 0;
} }
} }
} while ($sPrevLine !== false || $sDayLine !== false); } while ($sPrevLine !== false || $sDayLine !== false);
@fclose($hPrevTotals); @fclose($hPrevTotals);