MPAQ/BotSolo
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Latest version

Browse Source 
Signed-off-by: Bob <bob@mpaq.org>
This commit is contained in:
Bob
2026-08-04 14:04:38 -07:00
parent abbf2f3775
commit 5be9916dfc
1 changed files with 324 additions and 232 deletions
Download Patch File Download Diff File Expand all files Collapse all files
BotSolo.php
+324 -232
View File
@@ -3,22 +3,21 @@ ini_set('display_errors',1);ini_set('display_startup_errors',1);error_reporting(
date_default_timezone_set("America/Chicago"); date_default_timezone_set("America/Chicago");
$StartTime=microtime(true);$homeless=$new_post=$Count=$count=0; $StartTime=microtime(true);$homeless=$new_post=$Count=$count=0;
$home_dir=dirname(__FILE__); $home_dir=dirname(__FILE__);
include_once('/config/folder/.BotSolo.php'); include_once('/media/10T/ADMIN/.BotSolo.php');
$lockFile="/tmp/RSSSolo.lock"; $lockFile="/tmp/RSSSolo.lock";
if (file_exists($lockFile)){ if (file_exists($lockFile)){
$locked=$caller." has been locked at ".date('Y:m:d H:i:s'); $locked=$caller." has been locked";
if (DEBUG==0){send_group($caller,"tidybot@lobby.mpaq.org",$locked,"mpaq.org");} if (DEBUG==0){send_group($caller,"system@lobby.mpaq.org",$locked,"mpaq.org");}
exit;
} }
if (DEBUG==0){file_put_contents($lockFile,getmypid());}else{if (file_exists($lockFile)){unlink($lockFile);}} if (DEBUG==0){file_put_contents($lockFile,getmypid());}else{if (file_exists($lockFile)){unlink($lockFile);}}
$ERROR=NULL;$caller="BotRSS".$BOT['MyID']; $RID=$ERROR=NULL;
$sync=BaseAPI."synczero";
$options=['ssl' => ['verify_peer' => false,'verify_peer_name' => false]]; $options=['ssl' => ['verify_peer' => false,'verify_peer_name' => false]];
$context=stream_context_create($options); $context=stream_context_create($options);
$json=file_get_contents($sync,false,$context); $json=file_get_contents(SYNC,false,$context);
$Site=json_decode($json); $Site=json_decode($json);
$websites=$next=0; $websites=$next=0;
while(isset($Site[$next])){$next++;}$last=$next=$next-1;$next=0; while(isset($Site[$next])){$next++;}$last=$next=$next-1;$next=0;
echo "\t\tUsing ".SYNC."\n";
for ($site=$next;$next<=$last;$next++){ for ($site=$next;$next<=$last;$next++){
$websites++;$data['G2G']=0; $websites++;$data['G2G']=0;
$data['RSSindex']=$Site[$next]->index; $data['RSSindex']=$Site[$next]->index;
@@ -27,192 +26,209 @@ for ($site=$next;$next<=$last;$next++){
$data['SiteName']=$Site[$next]->sitename; $data['SiteName']=$Site[$next]->sitename;
$data['datetime']=$Site[$next]->datetime; $data['datetime']=$Site[$next]->datetime;
$data['hits']=$Site[$next]->hits; $data['hits']=$Site[$next]->hits;
if ($Site[$next]->botid==0){$data['botid']=$BOT['MyID'];}else{$data['botid']=$Site[$next]->botid;} if ($Site[$next]->botid==0){
$data['botid']=$BOT['MyID'];
}else{
$data['botid']=$Site[$next]->botid;
}
$data['feed']=$Site[$next]->url; $data['feed']=$Site[$next]->url;
$data['SiteNotes']=$Site[$next]->site_notes; $data['SiteNotes']=$Site[$next]->site_notes;
$HTMLCode=test_channel($data['feed']); $HTMLCode=test_channel($data['feed']);
if ($HTMLCode>399){ if ($HTMLCode>399){
$data['DataType']=2; $data['DataType']=2;
$data['SITES']="UPDATE tidysearch.sites SET botid='".$HTMLCode."' WHERE sites.index=".$data['SiteID'].";"; $data['SITES']="UPDATE tidysearch.sites SET botid='".$HTMLCode."',datetime=NULL WHERE sites.index=".$data['SiteID'].";";
$data['G2G']=send_2_server($BOT,$data); $data['G2G']=send_2_server($BOT,$data);
$data['DataType']=3; $data['DataType']=3;
$data['RSS']="UPDATE tidysearch.sites_rss SET botid='".$HTMLCode."' WHERE sites_rss.index=".$data['RSSindex'].";"; $data['RSS']="UPDATE tidysearch.sites_rss SET botid='".$HTMLCode."' WHERE sites_rss.index=".$data['RSSindex'].";";
$data['G2G']=send_2_server($BOT,$data); $data['G2G']=send_2_server($BOT,$data);
echo "Disabled Feed SiteID: ".$data['SiteID']." RSS: ".$data['RSSindex']."\n"; echo "Disabled Feed SiteID: ".$data['SiteID']." RSS: ".$data['RSSindex']."\n";
} }
if ($data['RSSindex']>0){$RID=trim($RID." ".pad_5($data['RSSindex']));}
$rss=new DOMDocument(); $rss=new DOMDocument();
@$rss->load($data['feed']); @$rss->load($data['feed']);
$feed = array(); $feed = array();
if (!$rss->getElementsByTagName('item')){var_dump($data);exit;} if ($rss->getElementsByTagName('item')){
foreach ($rss->getElementsByTagName('item') as $node){ foreach ($rss->getElementsByTagName('item') as $node){
$data['G2G']=$newLink=0; $data['G2G']=$newLink=0;
if (isset($node->getElementsByTagName('link')->item(0)->nodeValue)){ if (isset($node->getElementsByTagName('link')->item(0)->nodeValue)){
$data['link']=$node->getElementsByTagName('link')->item(0)->nodeValue; $data['link']=$node->getElementsByTagName('link')->item(0)->nodeValue;
$data['Source_Code']=mb_trim($data['link'],"/"); $data['Source_Code']=mb_trim($data['link'],"/");
if (strripos($data['Source_Code'],'?')){ if (strripos($data['Source_Code'],'?')){
$data['Source_Code']=substr($data['Source_Code'],0,strpos($data['Source_Code'],'?')); $data['Source_Code']=substr($data['Source_Code'],0,strpos($data['Source_Code'],'?'));
$data['Source_Code']=str_ireplace("?",'',$data['Source_Code']); $data['Source_Code']=str_ireplace("?",'',$data['Source_Code']);
}
$data['Source_Code']=str_ireplace("'","%27",$data['Source_Code']);
$data['Source_Code']=trim($data['Source_Code']);
$data['Source_Code']=mb_trim($data['Source_Code'],"/");
$sc=BaseAPI."source_code?sc=".$data['Source_Code'];
$sc=file_get_contents($sc,false,$context);
$sc=json_decode($sc);
if (!isset($sc[0])){
if (isset($node->getElementsByTagName('title')->item(0)->nodeValue)){
$data['Title']=clean_string($node->getElementsByTagName('title')->item(0)->nodeValue);
}else{$data['Title']=NULL;}
if (isset($node->getElementsByTagName('description')->item(0)->nodeValue)){
$RSSdata['desc']=clean_string($node->getElementsByTagName('description')->item(0)->nodeValue);
if (isset($RSSdata['desc']) and $RSSdata['desc']!==""){
$data['Description']=strip_tags($RSSdata['desc'],"<A>");
$data['Description']=str_ireplace("\n","",$data['Description']);
$data['Description']=str_ireplace("[&#8230;]","...",$data['Description']);
$data['Description']=substr($data['Description'],0,1000);
if (str_contains($data['Description'],'utm_source')){
$data['Description']=substr($data['Description'],0,strpos($data['Description'],'?'));
$data['Description']=str_ireplace("?",'',$data['Description']);
/* Look in $data['Description'] for
?utm_source=social&utm_medium=feed&utm_campaign=profeed
rel="nofollow"
*/
}
for ($x=0;$x<=50;$x++){$data['Description']=str_ireplace(' ',' ',$data['Description']);}
$data['Description']=trim($data['Description']);
$data['Description']=mb_trim($data['Description']);
}else{$data['Description']=NULL;}
}else{$data['Description']=NULL;}
if (isset($node->getElementsByTagName('pubDate')->item(0)->nodeValue)){
$data['date']=$node->getElementsByTagName('pubDate')->item(0)->nodeValue;
}elseif (isset($node->getElementsByTagName('dc:date')->item(0)->nodeValue)){
$data['date']=$node->getElementsByTagName('dc:date')->item(0)->nodeValue;
} }
if (isset($data['date']) and !is_null($data['date'])){ $data['Source_Code']=str_ireplace("'","%27",$data['Source_Code']);
$tmp=date('Y-m-d H:m:i',strtotime($data['date'])); $data['Source_Code']=trim($data['Source_Code']);
if (verifyDateTime($tmp)){ $data['Source_Code']=mb_trim($data['Source_Code'],"/");
$data['IndexDate']=date('Y-m-d H:i:s',strtotime($tmp)); $sc=BaseAPI."/api/v1/source_code?sc=".$data['Source_Code'];
}elseif (verifyDate($tmp)){ // Error keeps popping up HERE, GRR
$data['IndexDate']=date('Y-m-d H:i:s',strtotime($tmp)); $sc=@file_get_contents($sc,false,$context);
// Error keeps popping up HERE, GRR
$sc=json_decode($sc);
if (!isset($sc[0])){
if (isset($node->getElementsByTagName('title')->item(0)->nodeValue)){
$data['Title']=clean_string($node->getElementsByTagName('title')->item(0)->nodeValue);
}else{$data['Title']=NULL;$data['G2G']=-1;}
if (isset($node->getElementsByTagName('description')->item(0)->nodeValue)){
$RSSdata['desc']=clean_string($node->getElementsByTagName('description')->item(0)->nodeValue);
if (isset($RSSdata['desc']) and $RSSdata['desc']!==""){
$data['Description']=strip_tags($RSSdata['desc'],"<A>");
$data['Description']=str_ireplace("\n","",$data['Description']);
$data['Description']=str_ireplace("[&#8230;]","...",$data['Description']);
$data['Description']=substr($data['Description'],0,1000);
if (str_contains($data['Description'],'utm_source')){
$data['Description']=substr($data['Description'],0,strpos($data['Description'],'?'));
$data['Description']=str_ireplace("?",'',$data['Description']);
}
for ($x=0;$x<=50;$x++){$data['Description']=str_ireplace(' ',' ',$data['Description']);}
$data['Description']=trim($data['Description']);
$data['Description']=mb_trim($data['Description']);
}else{$data['Description']=NULL;$data['G2G']=-1;}
}else{$data['Description']=NULL;$data['G2G']=-1;}
if (isset($node->getElementsByTagName('pubDate')->item(0)->nodeValue)){
$data['date']=$node->getElementsByTagName('pubDate')->item(0)->nodeValue;
}elseif (isset($node->getElementsByTagName('dc:date')->item(0)->nodeValue)){
$data['date']=$node->getElementsByTagName('dc:date')->item(0)->nodeValue;
} }
}else{$data['IndexDate']=NULL;} if (isset($data['date']) and !is_null($data['date'])){
if (is_null($data['link'])){ $tmp=date('Y-m-d H:m:i',strtotime($data['date']));
log_files("Missing Link SiteID: ".$data['SiteID']."\t@ ".__LINE__."\n",LOG."Missing_Data.log"); if (verifyDateTime($tmp)){
$flag++;$newLink=0; $data['IndexDate']=date('Y-m-d H:i:s',strtotime($tmp));
} }elseif (verifyDate($tmp)){
if (!is_null($data['Title']) and !is_null($data['Description']) and !is_null($data['Source_Code'])){ $data['IndexDate']=date('Y-m-d H:i:s',strtotime($tmp));
$data['KWords']=KWords($data['Title'],$data['Description'],$data['SiteName']); }
$data['Description']=strip_tags($data['Description']); }else{$data['IndexDate']=NULL;$data['G2G']=-1;}
} if (is_null($data['link'])){
if (isset($data['IndexDate']) and !is_null($data['IndexDate'])){ log_files("Missing Link SiteID: ".$data['SiteID']."\t@ ".__LINE__."\n",LOG."Missing_Data.log");
if (!isset($new_date)){ $flag++;$newLink=0;
$new_date=date('Y-m-d H:m:i',strtotime($data['IndexDate'])); }
if (!is_null($data['Title']) and !is_null($data['Description']) and !is_null($data['Source_Code'])){
$data['KWords']=KWords($data['Title'],$data['Description'],$data['SiteName']);
$data['Description']=strip_tags($data['Description']);
}else{$data['G2G']=-1;}
if (isset($data['IndexDate']) and !is_null($data['IndexDate'])){
if (!isset($new_date)){
$new_date=date('Y-m-d H:m:i',strtotime($data['IndexDate']));
}else{
if (!is_null($new_date)){
if (strtotime($new_date)<strtotime($data['IndexDate'])){
$new_date=date('Y-m-d H:m:i',strtotime($data['IndexDate']));
}
}
}
}else{ }else{
if (!is_null($new_date)){ $data['IndexDate']=NULL;
if (strtotime($new_date)<strtotime($data['IndexDate'])){ }
$new_date=date('Y-m-d H:m:i',strtotime($data['IndexDate'])); if (is_null($data['datetime']) or $data['datetime']=="" or !isset($data['datetime'])){$data['datetime']="Never";$data['G2G']=-1;}
if (!is_null($data['Title']) and !is_null($data['Description'])){
$data['Title']=mb_trim($data['Title']);
$data['Title']=clean_string($data['Title']);
$data['Description']=mb_trim($data['Description']);
$data['Description']=clean_string($data['Description']);
}
if (DEBUG==1){
log_files("SiteID:\t\t".pad_8($data['SiteID']),LOG.pad_8($data['SiteID']).'-TidySearch.log');
log_files("RemoteID:\t".$data['RemoteID'],LOG.pad_8($data['SiteID']).'-TidySearch.log');
log_files("BotID:\t\t".pad_3($BOT['MyID']),LOG.pad_8($data['SiteID']).'-TidySearch.log');
if (isset($new_date)){log_files("Last Update:\t".$new_date,LOG.pad_8($data['SiteID']).'-TidySearch.log');}
log_files("IndexDate:\t".$data['IndexDate'],LOG.pad_8($data['SiteID']).'-TidySearch.log');
log_files("Site_Name:\t".$data['SiteName'],LOG.pad_8($data['SiteID']).'-TidySearch.log');
log_files("URL:\t\t".$data['feed'],LOG.pad_8($data['SiteID']).'-TidySearch.log');
log_files("Source_Code:\t".$data['Source_Code'],LOG.pad_8($data['SiteID']).'-TidySearch.log');
if (!is_null($data['Title']) and !is_null($data['Description']) and !is_null($data['KWords'])){
log_files("Title:\t\t".$data['Title'],LOG.pad_8($data['SiteID']).'-TidySearch.log');
log_files("Description:\t".$data['Description'],LOG.pad_8($data['SiteID']).'-TidySearch.log');
log_files("KWords:\t\t".$data['KWords'],LOG.pad_8($data['SiteID']).'-TidySearch.log');
}
log_files("SiteNotes:\t".$data['SiteNotes'],LOG.pad_8($data['SiteID']).'-TidySearch.log');
}
if (!is_null($data['Title']) and !is_null($data['Description']) and !is_null($data['Source_Code']) and !is_null($data['IndexDate'])){
$tmp1=htmlentities($data['Title'],ENT_QUOTES|ENT_HTML5,'UTF-8');
$tmp1="'".str_ireplace('\'','&apos;',$tmp1)."'";
$data['Description']=strip_tags($data['Description']);
$tmp2=htmlentities($data['Description'],ENT_QUOTES|ENT_HTML5,'UTF-8');
$tmp2="'".str_ireplace('\'','&apos;',$tmp2)."'";
$data['DataType']=1;
$data['PAGES']="INSERT INTO tidysearch.pages (site_id,date,title,description,kwords,source_code,botid,hostid,lan_code,rank)VALUES(".pad_5($data['SiteID']).",'".$data['IndexDate']."',".$tmp1.",".$tmp2.",'".$data['KWords']."','".$data['Source_Code']."',".$BOT['MyID'].",3,NULL,0);";
if (DEBUG==1){log_files("PAGES:\t\t".$data['PAGES'],LOG.pad_8($data['SiteID']).'-TidySearch.log');}
if (!is_null($data['Title']) and !is_null($data['Description']) and !is_null($data['KWords']) and DEBUG==0){
$data['G2G']=send_2_server($BOT,$data);
if ($data['G2G']>10){
$G2G=$data['G2G'];$Count++;
}else{$G2G=$data['G2G'];}
}else{
$data['G2G']=-3;
}
if (!is_null($data['KWords']) and $data['G2G']>10){
if (str_contains($data['KWords'],'homeless') or str_contains($data['KWords'],'houseless') or str_contains($data['KWords'],'homelessness')){
if ($data['G2G']>10){$data['PageID']=$data['G2G'];}
$homeless++;
if (DEBUG==0){$data['caller']=$caller;Fediverse ($data);}
} }
} }
} }
}else{ if (!is_null($data['Title'])){
$data['IndexDate']=NULL; $display=pad_space(html_entity_decode($data['Title'],ENT_QUOTES|ENT_HTML5,'UTF-8'));
}
if (DEBUG==1){
if (is_null($data['datetime']) or $data['datetime']=="" or !isset($data['datetime'])){$data['datetime']="Never";}
log_files("SiteID:\t\t".pad_8($data['SiteID']),LOG.pad_8($data['SiteID']).'-TidySearch.log');
log_files("RemoteID:\t".$data['RemoteID'],LOG.pad_8($data['SiteID']).'-TidySearch.log');
log_files("BotID:\t\t".pad_3($BOT['MyID']),LOG.pad_8($data['SiteID']).'-TidySearch.log');
log_files("Site_Name:\t".$data['SiteName'],LOG.pad_8($data['SiteID']).'-TidySearch.log');
if (isset($new_date)){log_files("Last Update:\t".$new_date,LOG.pad_8($data['SiteID']).'-TidySearch.log');}
log_files("URL:\t\t".$data['feed'],LOG.pad_8($data['SiteID']).'-TidySearch.log');
log_files("Source_Code:\t".$data['Source_Code'],LOG.pad_8($data['SiteID']).'-TidySearch.log');
log_files("IndexDate:\t".$data['IndexDate'],LOG.pad_8($data['SiteID']).'-TidySearch.log');
if (!is_null($data['Title']) and !is_null($data['Description']) and !is_null($data['KWords'])){
log_files("Title:\t\t".$data['Title'],LOG.pad_8($data['SiteID']).'-TidySearch.log');
log_files("Description:\t".$data['Description'],LOG.pad_8($data['SiteID']).'-TidySearch.log');
log_files("KWords:\t\t".$data['KWords'],LOG.pad_8($data['SiteID']).'-TidySearch.log');
}
log_files("SiteNotes:\t".$data['SiteNotes'],LOG.pad_8($data['SiteID']).'-TidySearch.log');
}
if (!is_null($data['Title']) and !is_null($data['Description']) and !is_null($data['Source_Code']) and !is_null($data['IndexDate'])){
$tmp1=htmlentities($data['Title'],ENT_QUOTES|ENT_HTML5,'UTF-8');
$tmp1="'".str_ireplace('\'','&apos;',$tmp1)."'";
$data['Description']=strip_tags($data['Description']);
$tmp2=htmlentities($data['Description'],ENT_QUOTES|ENT_HTML5,'UTF-8');
$tmp2="'".str_ireplace('\'','&apos;',$tmp2)."'";
$data['DataType']=1;
$data['PAGES']="INSERT INTO tidysearch.pages (site_id,date,title,description,kwords,source_code,botid,hostid,lan_code,rank)VALUES(".pad_5($data['SiteID']).",'".$data['IndexDate']."',".$tmp1.",".$tmp2.",'".$data['KWords']."','".$data['Source_Code']."',".$BOT['MyID'].",3,NULL,0);";
if (DEBUG==1){log_files("PAGES:\t\t".$data['PAGES'],LOG.pad_8($data['SiteID']).'-TidySearch.log');}
if (!is_null($data['Title']) and !is_null($data['Description']) and !is_null($data['KWords']) and DEBUG==0){
$data['G2G']=send_2_server($BOT,$data);
if ($data['G2G']>10){
$G2G=$data['G2G'];$Count++;
log_files($data['PAGES']." -- Page Idex: ".$data['G2G'],SQL.pad_8($data['SiteID']).'-TidySearch.sql');
}
}else{ }else{
$data['G2G']=0; $display=pad_space(" ");
} }
if (!is_null($data['KWords'])){ if (!isset($data['G2G'])){$data['G2G']=-1;}
if (str_contains($data['KWords'],'homeless') or str_contains($data['KWords'],'houseless') or str_contains($data['KWords'],'homelessness')){ echo pad_5($data['SiteID'])." ".pad_5($data['RSSindex'])." ".pad_3($BOT['MyID'])." ".pad_3($Count)." ".$data['IndexDate']." - ".$display."\t".memory_get_usage()."\t".pad_r(time_check($StartTime))."\tG2G: ".$data['G2G']."\n";
$homeless++; if (DEBUG==1){log_files("\t\t******",LOG.pad_8($data['SiteID']).'-TidySearch.log');}
if (DEBUG==0){Fediverse ($data);}
}
}
}
if (!is_null($data['Title'])){
$display=pad_space(html_entity_decode($data['Title'],ENT_QUOTES|ENT_HTML5,'UTF-8'));
}else{ }else{
$display=pad_space(" "); $display=pad_space("No New Data");
if (!isset($data['G2G'])){$data['G2G']=-10;}
echo pad_5($data['SiteID'])." ".pad_5($data['RSSindex'])." ".pad_3($BOT['MyID'])." ".pad_3($Count)." - ".$display."\t".memory_get_usage()."\t".pad_r(time_check($StartTime))."\tG2G: ".$data['G2G']."\n";
} }
echo pad_5($data['SiteID'])." ".pad_5($data['RSSindex'])." ".pad_3($BOT['MyID'])." ".pad_3($Count)." ".$data['IndexDate']." - ".$display."\t".memory_get_usage()."\t".time_check($StartTime)."\tG2G: ".$data['G2G']."\n";
if (DEBUG==1){log_files("\t\t******",LOG.pad_8($data['SiteID']).'-TidySearch.log');}
}else{
$display=pad_space("No New Data");
echo pad_5($data['SiteID'])." ".pad_5($data['RSSindex'])." ".pad_3($BOT['MyID'])." ".pad_3($Count)." - ".$display."\t".memory_get_usage()."\t".time_check($StartTime)."\tG2G: ".$data['G2G']."\n";
} }
} }
} if ($Count>0){
if ($Count>0){ $hits=$data['hits']+$Count;
$hits=$data['hits']+$Count; $data['DataType']=3;
$data['PAGES']=$data['SITES']=$data['RSS']=NULL; $data['RSS']="UPDATE tidysearch.sites_rss SET datetime='".$new_date."',hits=".$hits." WHERE sites_rss.index=".$data['RSSindex'].";";
$data['DataType']=2; if (DEBUG==0){$data['G2G']=send_2_server($BOT,$data);}
$data['SITES']="UPDATE tidysearch.sites SET datetime='".$new_date."' WHERE sites.index=".$data['SiteID'].";"; $count=$count+$Count;
if (DEBUG==0){$data['G2G']=send_2_server($BOT,$data);} }else{
$data['DataType']=3; $data['DataType']=3;
$data['RSS']="UPDATE tidysearch.sites_rss SET datetime='".$new_date."',hits=".$hits." WHERE sites_rss.index=".$data['RSSindex'].";"; $date=date('Y-m-d H:i:s');
if (DEBUG==0){$data['G2G']=send_2_server($BOT,$data);} $data['RSS']="UPDATE tidysearch.sites_rss SET datetime='".$date."' WHERE sites_rss.index=".$data['RSSindex'].";";
log_files($data['SITES']."\n".$data['RSS'],SQL.pad_8($data['SiteID']).'-TidySearch.sql'); if (DEBUG==0){$data['G2G']=send_2_server($BOT,$data);}
$count=$count+$Count; }
}else{
$data['DataType']=3;
$date=date('Y-m-d H:i:s');
$data['RSS']="UPDATE tidysearch.sites_rss SET datetime='".$date."' WHERE sites_rss.index=".$data['RSSindex'].";";
if (DEBUG==0){$data['G2G']=send_2_server($BOT,$data);}
} }
$Count=0; $Count=0;
$RSSindex=$data['RSSindex']; $RSSindex=$data['RSSindex'];
unset($data); unset($data);
echo "\n"; echo "\n";
} }
$display="BotID: ".pad_3($BOT['MyID'])."\nSites: ".pad_3($websites)." Feeds: ".pad_3($count)."\nHomeless: ".pad_3($homeless)."\n".time_check($StartTime); $data['DataType']=4;
if (DEBUG==0 and $count>0){send_group("NewsBot".$BOT['MyID'],"tidybot@lobby.mpaq.org",$display,"mpaq.org");} $now=date('Y-m-d');
$proc_time=time_check($StartTime);
$data['Stats']=$now.":&:".$BOT['MyID'].":&:".$websites.":&:".$count.":&:".$homeless.":&:".$proc_time;
$data['SiteID']=0;
if (DEBUG==0){$data['G2G']=send_2_server($BOT,$data);}
$display="\nSites: ".pad_3($websites)." Links: ".pad_3($count)." Home: ".pad_3($homeless)."\nRSS_IDs: ".$RID."\n".round($proc_time,2);
if (isset($G2G) and $G2G>0){ if (isset($G2G) and $G2G>0){
echo "\nData Sent: ".$display."\t\tDone\007\n";sleep(1);echo "\007"; $display=$caller." Data Sent:".$display;
}else{ }else{
echo "\nNo Data Sent: ".$display."\t\tDone\007\n";sleep(1);echo "\007"; $display=$caller." No Data Sent:".$display;
} }
if (DEBUG==0){send_group($caller,"tidybot@lobby.mpaq.org",$display,"mpaq.org");}
echo $display."\t\tDone\007\n";sleep(1);echo "\007";
if (file_exists($lockFile)){unlink($lockFile);} if (file_exists($lockFile)){unlink($lockFile);}
//////////////////////////// The End //////////////////////////// //////////////////////////// The End ////////////////////////////
//////////////////////////// The End //////////////////////////// //////////////////////////// The End ////////////////////////////
//////////////////////////// The End //////////////////////////// //////////////////////////// The End ////////////////////////////
function Fediverse ($data){ function Fediverse ($data){
include_once('/media/10T/ADMIN/.BotSolo.php'); include_once('/media/10T/ADMIN/.BotSolo.php');
include_once('/var/www/00_00/class/Mastodon.php');
$caller=$data['caller'];
$display=strip_tags($data['Description']); $display=strip_tags($data['Description']);
$display=substr($display,0,1000); $display=substr($display,0,1000);
$report=$data['Title']."\n\n".$display."\n"; $report=$data['Title']."\n\n".$display."\n";
unset($display); unset($display);
$report=str_ireplace('&#8230; Continue reading &#8594;','...',$report); $report=str_ireplace('&#8230; Continue reading &#8594;','...',$report);
$report=str_ireplace('&#8217;','\'',$report); $report=str_ireplace('&#8217;','\'',$report);
$report=str_ireplace('&#8216;','\'',$report);
$report=str_replace('homelessness','#homelessness',$report); $report=str_replace('homelessness','#homelessness',$report);
$report=str_replace('houseless','#houseless',$report); $report=str_replace('houseless','#houseless',$report);
$report=str_replace('homeless','#homeless',$report); $report=str_replace('homeless','#homeless',$report);
@@ -220,41 +236,33 @@ function Fediverse ($data){
$report=str_replace('Houseless','#Houseless',$report); $report=str_replace('Houseless','#Houseless',$report);
$report=str_replace('Homeless','#Homeless',$report); $report=str_replace('Homeless','#Homeless',$report);
$report=str_ireplace('##','#',$report); $report=str_ireplace('##','#',$report);
$report=str_ireplace('&#160;','',$report); $report=$report."\n".$data['Source_Code']."\n".date('M j,Y',strtotime($data['IndexDate']))."\n\n#TidySearch Bot".$data['botid']." (".$data['SiteID'].") For more visit http://search.mpaq.org";
$report=$report.date('M j,Y',strtotime($data['IndexDate']))." #Homeless #Poverty\n".$data['Source_Code']."\n\n#TidySearch Bot".$data['botid']." (".$data['SiteID'].") For more posts, follow @newsbot @ https://beamship.mpaq.org and Visit http://search.mpaq.org for more."; $report=$report." #Homeless #Poverty\n";
for ($x=0;$x<=50;$x++){$report=str_ireplace(' ',' ',$report);} for ($x=0;$x<=50;$x++){$report=str_ireplace(' ',' ',$report);}
/* if (!is_null($report) and isset($MTOKEN) and isset($M_URL)){
if (!is_null($report)){ $mastodon=new MastodonAPI($MTOKEN,$M_URL);
$headers = [ $status_data=[
'Authorization: Bearer '.TOKEN, 'status'=> $report,
'Content-Type: multipart/form-data', 'visibility'=> 'public',
'language'=> 'en',
]; ];
$ch = curl_init(); $status=$mastodon->postStatus($status_data);
curl_setopt($ch, CURLOPT_URL, Mast_URL."/api/v1/statuses"); }
curl_setopt($ch, CURLOPT_CUSTOMREQUEST, "Post"); if (isset($status['id']) and isset($data['PageID'])){
curl_setopt($ch, CURLOPT_POSTFIELDS, $report); $send=$caller.": NewsFeed Posted ".$data['SiteID']."-".$data['PageID']." ".M_URL."/@newsbot/statuses/".$status['id'];
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); }elseif (isset($data['PageID'])){
curl_setopt($ch, CURLOPT_HTTPHEADER, $headers); $send=$caller." ".$data['SiteID']."-".$data['PageID'].": NewsFeed \n".$report;
curl_setopt($ch, CURLOPT_USERAGENT,"MPAQ"); }else{
var_dump($ch); $send=$caller." ".$data['SiteID'].": NewsFeed \n".$report;
$reply = curl_exec($ch); }
echo $report."\n\n".$reply."\n";exit; $send=str_ireplace('&rsquo;','',$send);
if (!$reply) {return json_encode(['ok'=>false, 'curl_error_code' => curl_errno($ch), 'curl_error' => curl_error($ch)]);} $send=str_ireplace('\'','',$send);
return json_decode($reply, true); send_group($caller,"news@lobby.mpaq.org",$send,"mpaq.org");
*/ log_files($send,LOG."Mastodon.log");
log_files($report,LOG.pad_3($data['SiteID'])."-Homeless.log");
send_group("NewsBot".$data['botid'],"news@lobby.mpaq.org",$report,"mpaq.org");
log_files("\t*****",LOG.pad_3($data['SiteID'])."-Homeless.log");
// }
} }
function time_check($StartTime){ function time_check($StartTime){
$tmp=microtime(true)-$StartTime; $tmp=microtime(true)-$StartTime;
if ($tmp>60){ return round($tmp,3);
$tmp=$tmp/60;$return=pad_2(round($tmp,2))."M";
}else{
$return=pad_2(round($tmp,2))."S";
}
return $return;
} }
function log_files($notes,$name){ function log_files($notes,$name){
$entry=$notes."\n"; $entry=$notes."\n";
@@ -274,37 +282,28 @@ function sort_words($input){
} }
function verifyDateTime($date, $format = 'Y-m-d H:i:s'){ function verifyDateTime($date, $format = 'Y-m-d H:i:s'){
if (strtotime($date)<strtotime(date('Y-m-d H:i:s'))){ if (strtotime($date)<strtotime(date('Y-m-d H:i:s'))){
$dd = DateTime::createFromFormat($format, $date); // $year=date('Y',strtotime($date));
return $dd && $dd->format($format) === $date; // if ($year>1970){
$dd = DateTime::createFromFormat($format, $date);
return $dd && $dd->format($format) === $date;
// }else{
// return null;
// }
}else{ }else{
return null; return null;
} }
} }
function verifyDate($date, $format = 'Y-m-d'){ function verifyDate($date, $format = 'Y-m-d'){
if (strtotime($date)<strtotime(date('Y-m-d'))){ // $year=date('Y',strtotime($date));
$dd = DateTime::createFromFormat($format, $date); // if ($year>1970){
return $dd && $dd->format($format) === $date; $dd = DateTime::createFromFormat($format, $date);
}else{ return $dd && $dd->format($format) === $date;
// }else{
// return null;
// }
// }else{
return null; return null;
} // }
}
function test_channel($url){
ini_set('display_errors',1);ini_set('display_startup_errors',1);error_reporting(E_ALL);
include('/media/10T/ADMIN/.tidysearch.php');
$curl_test = array(
CURLOPT_HEADER => true, // we want headers
CURLOPT_NOBODY => true, // we don't need body
CURLOPT_RETURNTRANSFER => 1,
CURLOPT_TIMEOUT => 2, // timeout on response
CURLOPT_USERAGENT => "TidySearch Engine: visit http://search.mpaq.org for details.", // who am i
CURLOPT_CONNECTTIMEOUT => 2, // timeout on connect
CURLOPT_MAXREDIRS => 5, // stop after 10 redirects
);
$ch=curl_init($url);
curl_setopt_array($ch,$curl_test);
$output=curl_exec($ch);
$httpcode=curl_getinfo($ch, CURLINFO_HTTP_CODE);
return $httpcode;
} }
function send_2_server($BOT,$data){ function send_2_server($BOT,$data){
include_once('/media/10T/ADMIN/.BotSolo.php'); include_once('/media/10T/ADMIN/.BotSolo.php');
@@ -312,6 +311,10 @@ function send_2_server($BOT,$data){
if (!isset($data['SITES'])){$data['SITES']=NULL;} if (!isset($data['SITES'])){$data['SITES']=NULL;}
if (!isset($data['RSS'])){$data['RSS']=NULL;} if (!isset($data['RSS'])){$data['RSS']=NULL;}
if (!isset($data['Source_Code'])){$data['Source_Code']=NULL;} if (!isset($data['Source_Code'])){$data['Source_Code']=NULL;}
if (!isset($data['SiteID'])){$data['SiteID']=NULL;}
if (!isset($data['RSSindex'])){$data['RSSindex']=NULL;}
if (!isset($data['SiteName'])){$data['SiteName']=NULL;}
if (!isset($data['Stats'])){$data['Stats']=NULL;}
if (!isset($data['DataType'])){return NULL;} if (!isset($data['DataType'])){return NULL;}
$data2send=array( $data2send=array(
'BotPubKey' => $BOT['PubKey'], 'BotPubKey' => $BOT['PubKey'],
@@ -324,21 +327,24 @@ function send_2_server($BOT,$data){
'DataSites' => encryptText($data['SITES'],$BOT['PVTKey']), 'DataSites' => encryptText($data['SITES'],$BOT['PVTKey']),
'DataRSS' => encryptText($data['RSS'],$BOT['PVTKey']), 'DataRSS' => encryptText($data['RSS'],$BOT['PVTKey']),
'Source_Code' => encryptText($data['Source_Code'],$BOT['PVTKey']), 'Source_Code' => encryptText($data['Source_Code'],$BOT['PVTKey']),
'Stats' => encryptText($data['Stats'],$BOT['PVTKey']),
); );
// $sendto=BaseAPI."collect_ts"; $sendto=BaseAPI."/collect_ts.php";
$sendto="https://212.102.59.80:250/collect_ts.php"; if (isset($sendto) and isset($data2send) and isset($BOT)){
$responce=mb_trim(server($sendto,$data2send,$BOT['MyID'],)); $server=server($sendto,$data2send,$BOT['MyID']);
if (is_numeric($responce) and $responce>0){ if (isset($server)){$responce=mb_trim($server);}else{echo "MPAQ Network error, abort.\n";exit;}
$send="API Server ".$sendto." responded with: ".$responce; if (is_numeric($responce) and $responce>0){
return $responce; $send="API Server ".$sendto." responded with: ".$responce;
}else{ return $responce;
if (isset($data['SiteAddress'])){ }else{
$send="Line ".__LINE__." API Responce Error!!! Aborting...\n".$data['LocalID']." ".mb_trim($data['SiteAddress'])." from ".$sendto."\n\n".$responce; if (isset($data['SiteAddress'])){
echo $send."\n"; $send="Line ".__LINE__." API Responce Error!!! Aborting...\n".$data['LocalID']." ".mb_trim($data['SiteAddress'])." from ".$sendto."\n\n".$responce;
var_dump($data); echo $send."\n";
echo "\007";sleep(1);echo "\007";sleep(1);echo "\007";sleep(1); if (DEBUG==0){send_group($caller,"system@lobby.mpaq.org",$locked,"mpaq.org");}
exit; }
} }
}else{
return -4;
} }
} }
function server($sendto,$data,$botid){ function server($sendto,$data,$botid){
@@ -347,7 +353,8 @@ function server($sendto,$data,$botid){
"Content-Type: application/json", "Content-Type: application/json",
"Authorization: Bearer your_token", "Authorization: Bearer your_token",
); );
$config['useragent']='TidyBot (TidySearch http://search.mpaq.org)'; $config['useragent']='TidyBot (TidySearch (beta) http://search.mpaq.org)';
$config['cookie_file']=LOG.md5($sendto).'.txt';
$dir=dirname(__FILE__); $dir=dirname(__FILE__);
$ch=curl_init($sendto); $ch=curl_init($sendto);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
@@ -362,7 +369,6 @@ function server($sendto,$data,$botid){
curl_setopt($ch, CURLOPT_POSTFIELDS, http_build_query($data)); curl_setopt($ch, CURLOPT_POSTFIELDS, http_build_query($data));
$logFile=fopen(LOG."Curl.log", "a"); $logFile=fopen(LOG."Curl.log", "a");
curl_setopt($ch, CURLOPT_STDERR, $logFile); curl_setopt($ch, CURLOPT_STDERR, $logFile);
$config['cookie_file']=LOG.md5($sendto).'.txt';
curl_setopt($ch, CURLOPT_COOKIEFILE, $config['cookie_file']); curl_setopt($ch, CURLOPT_COOKIEFILE, $config['cookie_file']);
curl_setopt($ch, CURLOPT_COOKIEJAR, $config['cookie_file']); curl_setopt($ch, CURLOPT_COOKIEJAR, $config['cookie_file']);
curl_setopt($ch, CURLOPT_USERAGENT, $config['useragent']); curl_setopt($ch, CURLOPT_USERAGENT, $config['useragent']);
@@ -370,11 +376,10 @@ function server($sendto,$data,$botid){
$response=curl_exec($ch); $response=curl_exec($ch);
if (curl_errno($ch)){ if (curl_errno($ch)){
$send=__LINE__." sendto: ".$sendto."\nresponse: ".$response."\ncurl_error: ".curl_error($ch); $send=__LINE__." sendto: ".$sendto."\nresponse: ".$response."\ncurl_error: ".curl_error($ch);
// log_files($send,LOG.'API_Response.log'); log_files($send,LOG.'API_Response.log');
exit;
}else{ }else{
$send=__LINE__." sendto: ".$sendto."\nresponse: ".$response."\ncurl_error: ".curl_error($ch); $send=__LINE__." sendto: ".$sendto." ".$response;
// log_files($send,LOG.'API_Response.log'); log_files($send,LOG.'API_Response.log');
return trim($response); return trim($response);
} }
} }
@@ -389,6 +394,16 @@ function pad_2($ThisNumber){if (isset($ThisNumber)){return str_pad($ThisNumber,2
function pad_3($ThisNumber){if (isset($ThisNumber)){return str_pad($ThisNumber,3,"0",STR_PAD_LEFT);}else{return "000";}} function pad_3($ThisNumber){if (isset($ThisNumber)){return str_pad($ThisNumber,3,"0",STR_PAD_LEFT);}else{return "000";}}
function pad_5($ThisNumber){if (isset($ThisNumber)){return str_pad($ThisNumber,5,"0",STR_PAD_LEFT);}else{return "00000";}} function pad_5($ThisNumber){if (isset($ThisNumber)){return str_pad($ThisNumber,5,"0",STR_PAD_LEFT);}else{return "00000";}}
function pad_8($ThisNumber){if (isset($ThisNumber)){return str_pad($ThisNumber,8,"0",STR_PAD_LEFT);}else{return "00000000";}} function pad_8($ThisNumber){if (isset($ThisNumber)){return str_pad($ThisNumber,8,"0",STR_PAD_LEFT);}else{return "00000000";}}
function pad_r($ThisNumber){
if (isset($ThisNumber)){
$ThisNumber=str_pad($ThisNumber,4,"0",STR_PAD_LEFT);
$ThisNumber=str_pad($ThisNumber,4,"0",STR_PAD_RIGHT);
//echo $ThisNumber."\n";exit;
return $ThisNumber;
}else{
return "0000.0000";
}
}
function pad_space($This){if (strlen($This)>20){$result=substr($This,0,20);}else{$result=str_pad($This,20," ");}return $result;} function pad_space($This){if (strlen($This)>20){$result=substr($This,0,20);}else{$result=str_pad($This,20," ");}return $result;}
function KWords($title,$desc,$site){ function KWords($title,$desc,$site){
$INDEX_List=array( $INDEX_List=array(
@@ -460,21 +475,98 @@ function KWords($title,$desc,$site){
}else{return NULL;} }else{return NULL;}
} }
function send_group($BName,$Group,$Send,$Server){ function send_group($BName,$Group,$Send,$Server){
if (isset($_SERVER['SERVER_NAME'])){if ($_SERVER['SERVER_NAME']=="localhost"){exit;}} if (isset($BName) and isset($Group) and isset($Send) and isset($Server)){
$ERROR=NULL; $ERROR=NULL;
$ERROR=shell_exec("echo '".$Send."' | sendxmpp --username netbot --password 4cUTyfWwN5wc7St --jserver ".$Server." --resource ".$BName." --message-type=headline --chatroom ".$Group.""); $ERROR=shell_exec("echo '".$Send."' | sendxmpp --username netbot --password 4cUTyfWwN5wc7St --jserver ".$Server." --resource ".$BName." --message-type=headline --chatroom ".$Group."");
return $ERROR; if (!is_null($ERROR)){echo $ERROR."\n";exit;}else{return NULL;}
// echo $ERROR."\n";exit;
}
return NULL;
}
/////////////////////////////////////////////////////////
/*
class GhostAPI{
private $token;
private $instance_url;
public function __construct($token, $instance_url){
$this->token = $token;
$this->instance_url = $instance_url;
}
public function postStatus($status){
return $this->callAPI('/api/v1/post', 'POST', $status);
}
public function callAPI($endpoint, $method, $data){
$headers = [
'Authorization: Bearer '.$this->token,
'Content-Type: multipart/form-data',
];
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, $this->instance_url.$endpoint);
curl_setopt($ch, CURLOPT_CUSTOMREQUEST, $method);
curl_setopt($ch, CURLOPT_POSTFIELDS, $data);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
curl_setopt($ch, CURLOPT_USERAGENT,"MPAQ");
$reply = curl_exec($ch);
if (!$reply) {
return json_encode(['ok'=>false, 'curl_error_code' => curl_errno($ch), 'curl_error' => curl_error($ch)]);
}
return json_decode($reply, true);
}
} }
/* /*
class MastodonAPI{
// $token=MTOKEN;
// $instance_url=M_URL;
private $token;
private $instance_url;
public function __construct($token, $instance_url){
$this->token = $token;
$this->instance_url = $instance_url;
}
public function postStatus($status){return $this->callAPI('/api/v1/statuses', 'POST', $status);}
public function uploadMedia($media){return $this->callAPI('/api/v1/media', 'POST', $media);}
UPDATE tidysearch.sites SET url='',name='' WHERE sites.index=1538; public function callAPI($endpoint, $method, $data){
$headers = [
'Authorization: Bearer '.$this->token,
'Content-Type: multipart/form-data',
];
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, $this->instance_url.$endpoint);
curl_setopt($ch, CURLOPT_CUSTOMREQUEST, $method);
curl_setopt($ch, CURLOPT_POSTFIELDS, $data);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
curl_setopt($ch, CURLOPT_USERAGENT, "curl");
$reply = curl_exec($ch);
if (!$reply) {
log_files(curl_errno($ch)."\n\t".$instance_url."@".$instance_url,LOG."MastodonAPI.log");
return json_encode(['ok'=>false, 'curl_error_code' => curl_errno($ch), 'curl_error' => curl_error($ch)]);
}
log_files(curl_errno($ch)."\n\t".$instance_url."@".$instance_url,LOG."MastodonAPI.log");
return json_decode($reply, true);
}
}
/*
UPDATE tidysearch.pages SET _lang=null WHERE sites._lang="";
UPDATE tidysearch.sites_rss SET url='https://ojrock.mpaq.org/feed' WHERE sites_rss.index=437;
UPDATE tidysearch.sites_rss SET botid='404' WHERE sites_rss.index=587; DELETE FROM tidysearch.pages WHERE pages.title is null;
UPDATE tidysearch.sites SET botid=0,hits=0,name='',url='' WHERE sites.index=xxx; DELETE FROM tidysearch.pages WHERE pages.title LIKE "";
UPDATE tidysearch.sites_rss SET botid=0,hits=0,site_name='',url='' WHERE sites_rss.index=xxx;
UPDATE tidysearch.sites SET botid=0 WHERE sites.botid=9000; DELETE FROM tidysearch.pages WHERE pages.date = NULL;
UPDATE tidysearch.sites_rss SET botid=0 WHERE sites_rss.botid=9000;
UPDATE tidysearch.sites SET botid=0 WHERE sites.botid=9001; delete from tidysearch.pages where kwords is null;
UPDATE tidysearch.sites_rss SET botid=0 WHERE sites_rss.botid=9001; SELECT * FROM tidysearch.pages where pages.description like '';
SELECT * FROM tidysearch.pages ORDER BY pages.date DESC LIMIT 10;
https://news.mpaq.org/sitemap-posts.xml
https://news.mpaq.org/.ghost/activitypub/nodeinfo/2.1
OpenFire chatter HWrBVELMAh8YYdk
https://bots.mpaq.org/.well-known/webfinger?resource=acct:@bob@mpaq.org
*/ */
?> ?>