tabs-to-spaces

This commit is contained in:
Marc Tobias Metten
2016-09-04 03:19:48 +02:00
parent cd6dcfa574
commit 832547f192
55 changed files with 8236 additions and 8232 deletions

View File

@@ -17,172 +17,172 @@ $iNS = false;
$iID = false;
if ($hFile) {
while (($sLine = fgets($hFile, 4000000)) !== false) {
if (substr($sLine, 0, 11) == ' <title>') {
$sTitle = substr($sLine, 11, -9);
}
else if (substr($sLine, 0, 8) == ' <ns>') {
$iNS = (int)substr($sLine, 8, -6);
}
else if (substr($sLine, 0, 8) == ' <id>') {
$iID = (int)substr($sLine, 8, -6);
}
else if (substr($sLine, 0, 33) == ' <text xml:space="preserve">') {
if ($iNS == -2) continue;
if ($iNS == -1) continue;
if ($iNS == 1) continue;
if ($iNS == 2) continue;
if ($iNS == 3) continue;
if ($iNS == 4) continue;
if ($iNS == 5) continue;
if ($iNS == 6) continue;
if ($iNS == 7) continue;
if ($iNS == 8) continue;
if ($iNS == 9) continue;
if ($iNS == 10) continue;
if ($iNS == 11) continue;
if ($iNS == 12) continue;
if ($iNS == 13) continue;
if ($iNS == 14) continue;
if ($iNS == 15) continue;
if ($iNS == 121) continue;
if ($iNS == 123) continue;
if ($iNS == 829) continue;
if ($iNS == 1198) continue;
if ($iNS == 1199) continue;
$sText = html_entity_decode(substr($sLine, 33, -8), ENT_COMPAT, 'UTF-8');
$aArticle = json_decode($sText, true);
while (($sLine = fgets($hFile, 4000000)) !== false) {
if (substr($sLine, 0, 11) == ' <title>') {
$sTitle = substr($sLine, 11, -9);
}
else if (substr($sLine, 0, 8) == ' <ns>') {
$iNS = (int)substr($sLine, 8, -6);
}
else if (substr($sLine, 0, 8) == ' <id>') {
$iID = (int)substr($sLine, 8, -6);
}
else if (substr($sLine, 0, 33) == ' <text xml:space="preserve">') {
if ($iNS == -2) continue;
if ($iNS == -1) continue;
if ($iNS == 1) continue;
if ($iNS == 2) continue;
if ($iNS == 3) continue;
if ($iNS == 4) continue;
if ($iNS == 5) continue;
if ($iNS == 6) continue;
if ($iNS == 7) continue;
if ($iNS == 8) continue;
if ($iNS == 9) continue;
if ($iNS == 10) continue;
if ($iNS == 11) continue;
if ($iNS == 12) continue;
if ($iNS == 13) continue;
if ($iNS == 14) continue;
if ($iNS == 15) continue;
if ($iNS == 121) continue;
if ($iNS == 123) continue;
if ($iNS == 829) continue;
if ($iNS == 1198) continue;
if ($iNS == 1199) continue;
$sText = html_entity_decode(substr($sLine, 33, -8), ENT_COMPAT, 'UTF-8');
$aArticle = json_decode($sText, true);
if (array_diff(array_keys($aArticle), array("label", "description", "aliases", "links", "entity", "claims", "datatype")) != array()) {
// DEBUG
var_dump($sTitle);
var_dump(array_keys($aArticle));
var_dump($aArticle);
exit;
}
if (array_diff(array_keys($aArticle), array("label", "description", "aliases", "links", "entity", "claims", "datatype")) != array()) {
// DEBUG
var_dump($sTitle);
var_dump(array_keys($aArticle));
var_dump($aArticle);
exit;
}
$iPID = $iQID = null;
if ($aArticle['entity'][0] == 'p') {
$iPID = (int)substr($aArticle['entity'], 1);
} else if ($aArticle['entity'][0] == 'q') {
$iQID = (int)substr($aArticle['entity'], 1);
} else {
continue;
}
$iPID = $iQID = null;
if ($aArticle['entity'][0] == 'p') {
$iPID = (int)substr($aArticle['entity'], 1);
} else if ($aArticle['entity'][0] == 'q') {
$iQID = (int)substr($aArticle['entity'], 1);
} else {
continue;
}
echo ".";
echo ".";
fputcsv($hFileEntity, array($iID,$sTitle,$iPID,$iQID,@$aArticle['datatype']));
fputcsv($hFileEntity, array($iID,$sTitle,$iPID,$iQID,@$aArticle['datatype']));
foreach($aArticle['label'] as $sLang => $sLabel) {
fputcsv($hFileEntityLabel, array($iID,$sLang,$sLabel));
// echo "insert into entity_label values (".$iID.",'".pg_escape_string($sLang)."','".pg_escape_string($sLabel)."');\n";
}
foreach($aArticle['label'] as $sLang => $sLabel) {
fputcsv($hFileEntityLabel, array($iID,$sLang,$sLabel));
// echo "insert into entity_label values (".$iID.",'".pg_escape_string($sLang)."','".pg_escape_string($sLabel)."');\n";
}
foreach($aArticle['description'] as $sLang => $sLabel) {
fputcsv($hFileEntityDescription, array($iID,$sLang,$sLabel));
// echo "insert into entity_description values (".$iID.",'".pg_escape_string($sLang)."','".pg_escape_string($sLabel)."');\n";
}
foreach($aArticle['description'] as $sLang => $sLabel) {
fputcsv($hFileEntityDescription, array($iID,$sLang,$sLabel));
// echo "insert into entity_description values (".$iID.",'".pg_escape_string($sLang)."','".pg_escape_string($sLabel)."');\n";
}
foreach($aArticle['aliases'] as $sLang => $aLabels) {
$aUniqueAlias = array();
foreach($aLabels as $sLabel) {
if (!isset($aUniqueAlias[$sLabel]) && $sLabel) {
fputcsv($hFileEntityAlias, array($iID,$sLang,$sLabel));
// echo "insert into entity_alias values (".$iID.",'".pg_escape_string($sLang)."','".pg_escape_string($sLabel)."');\n";
$aUniqueAlias[$sLabel] = true;
}
}
}
foreach($aArticle['aliases'] as $sLang => $aLabels) {
$aUniqueAlias = array();
foreach($aLabels as $sLabel) {
if (!isset($aUniqueAlias[$sLabel]) && $sLabel) {
fputcsv($hFileEntityAlias, array($iID,$sLang,$sLabel));
// echo "insert into entity_alias values (".$iID.",'".pg_escape_string($sLang)."','".pg_escape_string($sLabel)."');\n";
$aUniqueAlias[$sLabel] = true;
}
}
}
foreach($aArticle['links'] as $sLang => $sLabel) {
fputcsv($hFileEntityLink, array($iID,$sLang,$sLabel));
// echo "insert into entity_link values (".$iID.",'".pg_escape_string($sLang)."','".pg_escape_string($sLabel)."');\n";
}
foreach($aArticle['links'] as $sLang => $sLabel) {
fputcsv($hFileEntityLink, array($iID,$sLang,$sLabel));
// echo "insert into entity_link values (".$iID.",'".pg_escape_string($sLang)."','".pg_escape_string($sLabel)."');\n";
}
if (isset($aArticle['claims'])) {
if (isset($aArticle['claims'])) {
foreach($aArticle['claims'] as $iClaim => $aClaim) {
foreach($aArticle['claims'] as $iClaim => $aClaim) {
$bFail = false;
if ($aClaim['m'][0] == 'novalue') continue;
if ($aClaim['m'][0] == 'somevalue') continue;
$iPID = (int)$aClaim['m'][1];
if ($aClaim['m'][0] != 'value') $bFail = true;
if ($aClaim['m'][2]== 'wikibase-entityid') {
$bFail = false;
if ($aClaim['m'][0] == 'novalue') continue;
if ($aClaim['m'][0] == 'somevalue') continue;
$iPID = (int)$aClaim['m'][1];
if ($aClaim['m'][0] != 'value') $bFail = true;
if ($aClaim['m'][2]== 'wikibase-entityid') {
if ($aClaim['m'][3]['entity-type'] != 'item') $bFail = true;
fputcsv($hFileEntityProperty, array($iID,$iClaim,$iPID,null,$aClaim['m'][3]['numeric-id'],null,null));
// echo "insert into entity_property values (nextval('seq_entity_property'),".$iID.",".$iPID.",null,".$aClaim['m'][3]['numeric-id'].",null);\n";
if ($aClaim['m'][3]['entity-type'] != 'item') $bFail = true;
fputcsv($hFileEntityProperty, array($iID,$iClaim,$iPID,null,$aClaim['m'][3]['numeric-id'],null,null));
// echo "insert into entity_property values (nextval('seq_entity_property'),".$iID.",".$iPID.",null,".$aClaim['m'][3]['numeric-id'].",null);\n";
} elseif ($aClaim['m'][2] == 'globecoordinate') {
} elseif ($aClaim['m'][2] == 'globecoordinate') {
if ($aClaim['m'][3]['globe'] != 'http://www.wikidata.org/entity/Q2') $bFail = true;
fputcsv($hFileEntityProperty, array($iID,$iClaim,$iPID,null,null,"SRID=4326;POINT(".((float)$aClaim['m'][3]['longitude'])." ".((float)$aClaim['m'][3]['latitude']).")",null));
// echo "insert into entity_property values (nextval('seq_entity_property'),".$iID.",".$iPID.",null,null,ST_SetSRID(ST_MakePoint(".((float)$aClaim['m'][3]['longitude']).", ".((float)$aClaim['m'][3]['latitude'])."),4326));\n";
if ($aClaim['m'][3]['globe'] != 'http://www.wikidata.org/entity/Q2') $bFail = true;
fputcsv($hFileEntityProperty, array($iID,$iClaim,$iPID,null,null,"SRID=4326;POINT(".((float)$aClaim['m'][3]['longitude'])." ".((float)$aClaim['m'][3]['latitude']).")",null));
// echo "insert into entity_property values (nextval('seq_entity_property'),".$iID.",".$iPID.",null,null,ST_SetSRID(ST_MakePoint(".((float)$aClaim['m'][3]['longitude']).", ".((float)$aClaim['m'][3]['latitude'])."),4326));\n";
} elseif ($aClaim['m'][2] == 'time') {
// TODO!
} elseif ($aClaim['m'][2] == 'time') {
// TODO!
/*
if ($aClaim['m'][3]['calendarmodel'] == 'http://www.wikidata.org/entity/Q1985727') {
// Gregorian
if (preg_match('#(\\+|-)0*([0-9]{4})-([0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2})Z#', $aClaim['m'][3]['time'], $aMatch)) {
if ((int)$aMatch[2] < 4700 && ) {
$sDateString = $aMatch[2].'-'.$aMatch[3].($aClaim['m'][3]['timezone']>=0?'+':'').$aClaim['m'][3]['timezone'].($aMatch[1]=='-'?' bc':'');
fputcsv($hFileEntityProperty, array($iID,$iClaim,$iPID,null,null,null,$sDateString));
}
} else {
// $bFail = true;
}
} elseif ( $aClaim['m'][3]['calendarmodel'] != 'http://www.wikidata.org/entity/Q1985786') {
if ($aClaim['m'][3]['calendarmodel'] == 'http://www.wikidata.org/entity/Q1985727') {
// Gregorian
if (preg_match('#(\\+|-)0*([0-9]{4})-([0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2})Z#', $aClaim['m'][3]['time'], $aMatch)) {
if ((int)$aMatch[2] < 4700 && ) {
$sDateString = $aMatch[2].'-'.$aMatch[3].($aClaim['m'][3]['timezone']>=0?'+':'').$aClaim['m'][3]['timezone'].($aMatch[1]=='-'?' bc':'');
fputcsv($hFileEntityProperty, array($iID,$iClaim,$iPID,null,null,null,$sDateString));
}
} else {
// $bFail = true;
}
} elseif ( $aClaim['m'][3]['calendarmodel'] != 'http://www.wikidata.org/entity/Q1985786') {
/ *
// Julian
if (preg_match('#(\\+|-)0*([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2}:[0-9]{2}:[0-9]{2})Z#', $aClaim['m'][3]['time'], $aMatch)) {
// Julian
if (preg_match('#(\\+|-)0*([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2}:[0-9]{2}:[0-9]{2})Z#', $aClaim['m'][3]['time'], $aMatch)) {
var_dump($aMatch);
exit;
$iDayCount = juliantojd(2, 11, 1732);
var_dump($iDayCount, jdtogregorian($iDayCount));
} else {
$bFail = true;
} else {
$bFail = true;
exit;
}
}
exit;
* /
} else {
// $bFail = true;
}
} else {
// $bFail = true;
}
*/
} elseif ($aClaim['m'][2] == 'string') {
} elseif ($aClaim['m'][2] == 'string') {
// echo "insert into entity_property values (nextval('seq_entity_property'),".$iID.",".$iPID.",'".pg_escape_string($aClaim['m'][3])."',null,null);\n";
fputcsv($hFileEntityProperty, array($iID,$iClaim,$iPID,$aClaim['m'][3],null,null,null));
// echo "insert into entity_property values (nextval('seq_entity_property'),".$iID.",".$iPID.",'".pg_escape_string($aClaim['m'][3])."',null,null);\n";
fputcsv($hFileEntityProperty, array($iID,$iClaim,$iPID,$aClaim['m'][3],null,null,null));
} else {
} else {
$bFail = true;
$bFail = true;
}
}
// Don't care about sources: if ($aClaim['refs'] != array()) $bFail = true;
// Don't care about sources: if ($aClaim['refs'] != array()) $bFail = true;
if ($bFail) {
var_dump($sTitle);
var_dump($aClaim);
} else {
// process
}
if ($bFail) {
var_dump($sTitle);
var_dump($aClaim);
} else {
// process
}
}
}
}
}
}
fclose($hFile);
fclose($hFileEntity);
fclose($hFileEntityLabel);
fclose($hFileEntityDescription);
fclose($hFileEntityAlias);
fclose($hFileEntityLink);
fclose($hFileEntityProperty);
}
}
}
fclose($hFile);
fclose($hFileEntity);
fclose($hFileEntityLabel);
fclose($hFileEntityDescription);
fclose($hFileEntityAlias);
fclose($hFileEntityLink);
fclose($hFileEntityProperty);
}

View File

@@ -1,91 +1,91 @@
<?php
for($iTimestamp = mktime(0, 0, 0, 5, 1, 2013); $iTimestamp < mktime(0, 0, 0, 6, 15, 2013); $iTimestamp += 24*60*60)
{
$sYear = date("Y", $iTimestamp);
$sMonth = date("Y-m", $iTimestamp);
$sDay = date("Ymd", $iTimestamp);
for($iTimestamp = mktime(0, 0, 0, 5, 1, 2013); $iTimestamp < mktime(0, 0, 0, 6, 15, 2013); $iTimestamp += 24*60*60)
{
$sYear = date("Y", $iTimestamp);
$sMonth = date("Y-m", $iTimestamp);
$sDay = date("Ymd", $iTimestamp);
for($iHour = 0; $iHour < 24; $iHour++)
{
$sFilename = sprintf("pagecounts-".$sDay."-%02d0000", $iHour);
echo $sFilename."\n";
if (!file_exists($sFilename.'.gz'))
{
exec('wget http://dumps.wikimedia.org/other/pagecounts-raw/'.$sYear.'/'.$sMonth.'/'.$sFilename.'.gz');
}
for($iHour = 0; $iHour < 24; $iHour++)
{
$sFilename = sprintf("pagecounts-".$sDay."-%02d0000", $iHour);
echo $sFilename."\n";
if (!file_exists($sFilename.'.gz'))
{
exec('wget http://dumps.wikimedia.org/other/pagecounts-raw/'.$sYear.'/'.$sMonth.'/'.$sFilename.'.gz');
}
exec('gzip -dc '.$sFilename.'.gz'.' | grep -e "^[a-z]\{2\} [^ :]\+ [0-9]\+" > hour.txt');
exec('gzip -dc '.$sFilename.'.gz'.' | grep -e "^[a-z]\{2\} [^ :]\+ [0-9]\+" > hour.txt');
$hPrevTotals = @fopen("totals.txt", "r");
$hDayTotals = @fopen("hour.txt", "r");
$hNewTotals = @fopen("newtotals.txt", "w");
$hPrevTotals = @fopen("totals.txt", "r");
$hDayTotals = @fopen("hour.txt", "r");
$hNewTotals = @fopen("newtotals.txt", "w");
$sPrevKey = $sDayKey = true;
$sPrevLine = true;
$sDayLine = true;
$sPrevKey = $sDayKey = true;
$sPrevLine = true;
$sDayLine = true;
do
{
if ($sPrevKey === $sDayKey)
{
if ($sPrevLine !== true) fputs($hNewTotals, "$sPrevKey ".($iPrevValue+$iDayValue)."\n");
$sPrevLine = true;
$sDayLine = true;
}
else if ($sDayKey !== false && ($sPrevKey > $sDayKey || $sPrevKey === false))
{
fputs($hNewTotals, "$sDayKey ".($iDayValue)."\n");
$sDayLine = true;
}
else if ($sPrevKey !== false && ($sDayKey > $sPrevKey || $sDayKey === false))
{
fputs($hNewTotals, "$sPrevKey ".($iPrevValue)."\n");
$sPrevLine = true;
}
do
{
if ($sPrevKey === $sDayKey)
{
if ($sPrevLine !== true) fputs($hNewTotals, "$sPrevKey ".($iPrevValue+$iDayValue)."\n");
$sPrevLine = true;
$sDayLine = true;
}
else if ($sDayKey !== false && ($sPrevKey > $sDayKey || $sPrevKey === false))
{
fputs($hNewTotals, "$sDayKey ".($iDayValue)."\n");
$sDayLine = true;
}
else if ($sPrevKey !== false && ($sDayKey > $sPrevKey || $sDayKey === false))
{
fputs($hNewTotals, "$sPrevKey ".($iPrevValue)."\n");
$sPrevLine = true;
}
if ($sPrevLine === true)
{
$sPrevLine = $hPrevTotals?fgets($hPrevTotals, 4096):false;
if ($sPrevLine !== false)
{
$aPrevLine = explode(' ', $sPrevLine);
$sPrevKey = $aPrevLine[0].' '.$aPrevLine[1];
$iPrevValue = (int)$aPrevLine[2];
}
else
{
$sPrevKey = false;
$iPrevValue = 0;
}
}
if ($sPrevLine === true)
{
$sPrevLine = $hPrevTotals?fgets($hPrevTotals, 4096):false;
if ($sPrevLine !== false)
{
$aPrevLine = explode(' ', $sPrevLine);
$sPrevKey = $aPrevLine[0].' '.$aPrevLine[1];
$iPrevValue = (int)$aPrevLine[2];
}
else
{
$sPrevKey = false;
$iPrevValue = 0;
}
}
if ($sDayLine === true)
{
$sDayLine = $hDayTotals?fgets($hDayTotals, 4096):false;
if ($sDayLine !== false)
{
preg_match('#^([a-z]{2}) ([^ :]+) ([0-9]+) [0-9]+$#', $sDayLine, $aMatch);
$sDayKey = $aMatch[1].' '.$aMatch[2];
$iDayValue = (int)$aMatch[3];
}
else
{
$sDayKey = false;
$iDayValue = 0;
}
}
if ($sDayLine === true)
{
$sDayLine = $hDayTotals?fgets($hDayTotals, 4096):false;
if ($sDayLine !== false)
{
preg_match('#^([a-z]{2}) ([^ :]+) ([0-9]+) [0-9]+$#', $sDayLine, $aMatch);
$sDayKey = $aMatch[1].' '.$aMatch[2];
$iDayValue = (int)$aMatch[3];
}
else
{
$sDayKey = false;
$iDayValue = 0;
}
}
} while ($sPrevLine !== false || $sDayLine !== false);
} while ($sPrevLine !== false || $sDayLine !== false);
@fclose($hPrevTotals);
@fclose($hDayTotals);
@fclose($hNewTotals);
@fclose($hPrevTotals);
@fclose($hDayTotals);
@fclose($hNewTotals);
@unlink("totals.txt");
rename("newtotals.txt", "totals.txt");
}
}
@unlink("totals.txt");
rename("newtotals.txt", "totals.txt");
}
}
// Notes:
/*