diff --git a/BotSolo.php b/BotSolo.php index 7a6c5dd..3271d8b 100644 --- a/BotSolo.php +++ b/BotSolo.php @@ -3,22 +3,21 @@ ini_set('display_errors',1);ini_set('display_startup_errors',1);error_reporting( date_default_timezone_set("America/Chicago"); $StartTime=microtime(true);$homeless=$new_post=$Count=$count=0; $home_dir=dirname(__FILE__); -include_once('/config/folder/.BotSolo.php'); +include_once('/media/10T/ADMIN/.BotSolo.php'); $lockFile="/tmp/RSSSolo.lock"; if (file_exists($lockFile)){ - $locked=$caller." has been locked at ".date('Y:m:d H:i:s'); - if (DEBUG==0){send_group($caller,"tidybot@lobby.mpaq.org",$locked,"mpaq.org");} - exit; + $locked=$caller." has been locked"; + if (DEBUG==0){send_group($caller,"system@lobby.mpaq.org",$locked,"mpaq.org");} } if (DEBUG==0){file_put_contents($lockFile,getmypid());}else{if (file_exists($lockFile)){unlink($lockFile);}} -$ERROR=NULL;$caller="BotRSS".$BOT['MyID']; -$sync=BaseAPI."synczero"; +$RID=$ERROR=NULL; $options=['ssl' => ['verify_peer' => false,'verify_peer_name' => false]]; $context=stream_context_create($options); -$json=file_get_contents($sync,false,$context); +$json=file_get_contents(SYNC,false,$context); $Site=json_decode($json); $websites=$next=0; while(isset($Site[$next])){$next++;}$last=$next=$next-1;$next=0; +echo "\t\tUsing ".SYNC."\n"; for ($site=$next;$next<=$last;$next++){ $websites++;$data['G2G']=0; $data['RSSindex']=$Site[$next]->index; @@ -27,192 +26,209 @@ for ($site=$next;$next<=$last;$next++){ $data['SiteName']=$Site[$next]->sitename; $data['datetime']=$Site[$next]->datetime; $data['hits']=$Site[$next]->hits; - if ($Site[$next]->botid==0){$data['botid']=$BOT['MyID'];}else{$data['botid']=$Site[$next]->botid;} + if ($Site[$next]->botid==0){ + $data['botid']=$BOT['MyID']; + }else{ + $data['botid']=$Site[$next]->botid; + } $data['feed']=$Site[$next]->url; $data['SiteNotes']=$Site[$next]->site_notes; $HTMLCode=test_channel($data['feed']); if ($HTMLCode>399){ $data['DataType']=2; - $data['SITES']="UPDATE tidysearch.sites SET botid='".$HTMLCode."' WHERE sites.index=".$data['SiteID'].";"; + $data['SITES']="UPDATE tidysearch.sites SET botid='".$HTMLCode."',datetime=NULL WHERE sites.index=".$data['SiteID'].";"; $data['G2G']=send_2_server($BOT,$data); $data['DataType']=3; $data['RSS']="UPDATE tidysearch.sites_rss SET botid='".$HTMLCode."' WHERE sites_rss.index=".$data['RSSindex'].";"; $data['G2G']=send_2_server($BOT,$data); echo "Disabled Feed SiteID: ".$data['SiteID']." RSS: ".$data['RSSindex']."\n"; } + if ($data['RSSindex']>0){$RID=trim($RID." ".pad_5($data['RSSindex']));} $rss=new DOMDocument(); @$rss->load($data['feed']); $feed = array(); - if (!$rss->getElementsByTagName('item')){var_dump($data);exit;} - foreach ($rss->getElementsByTagName('item') as $node){ - $data['G2G']=$newLink=0; - if (isset($node->getElementsByTagName('link')->item(0)->nodeValue)){ - $data['link']=$node->getElementsByTagName('link')->item(0)->nodeValue; - $data['Source_Code']=mb_trim($data['link'],"/"); - if (strripos($data['Source_Code'],'?')){ - $data['Source_Code']=substr($data['Source_Code'],0,strpos($data['Source_Code'],'?')); - $data['Source_Code']=str_ireplace("?",'',$data['Source_Code']); - } - $data['Source_Code']=str_ireplace("'","%27",$data['Source_Code']); - $data['Source_Code']=trim($data['Source_Code']); - $data['Source_Code']=mb_trim($data['Source_Code'],"/"); - $sc=BaseAPI."source_code?sc=".$data['Source_Code']; - $sc=file_get_contents($sc,false,$context); - $sc=json_decode($sc); - if (!isset($sc[0])){ - if (isset($node->getElementsByTagName('title')->item(0)->nodeValue)){ - $data['Title']=clean_string($node->getElementsByTagName('title')->item(0)->nodeValue); - }else{$data['Title']=NULL;} - if (isset($node->getElementsByTagName('description')->item(0)->nodeValue)){ - $RSSdata['desc']=clean_string($node->getElementsByTagName('description')->item(0)->nodeValue); - if (isset($RSSdata['desc']) and $RSSdata['desc']!==""){ - $data['Description']=strip_tags($RSSdata['desc'],""); - $data['Description']=str_ireplace("\n","",$data['Description']); - $data['Description']=str_ireplace("[…]","...",$data['Description']); - $data['Description']=substr($data['Description'],0,1000); - if (str_contains($data['Description'],'utm_source')){ - $data['Description']=substr($data['Description'],0,strpos($data['Description'],'?')); - $data['Description']=str_ireplace("?",'',$data['Description']); -/* Look in $data['Description'] for - ?utm_source=social&utm_medium=feed&utm_campaign=profeed - rel="nofollow" -*/ - } - for ($x=0;$x<=50;$x++){$data['Description']=str_ireplace(' ',' ',$data['Description']);} - $data['Description']=trim($data['Description']); - $data['Description']=mb_trim($data['Description']); - }else{$data['Description']=NULL;} - }else{$data['Description']=NULL;} - if (isset($node->getElementsByTagName('pubDate')->item(0)->nodeValue)){ - $data['date']=$node->getElementsByTagName('pubDate')->item(0)->nodeValue; - }elseif (isset($node->getElementsByTagName('dc:date')->item(0)->nodeValue)){ - $data['date']=$node->getElementsByTagName('dc:date')->item(0)->nodeValue; + if ($rss->getElementsByTagName('item')){ + foreach ($rss->getElementsByTagName('item') as $node){ + $data['G2G']=$newLink=0; + if (isset($node->getElementsByTagName('link')->item(0)->nodeValue)){ + $data['link']=$node->getElementsByTagName('link')->item(0)->nodeValue; + $data['Source_Code']=mb_trim($data['link'],"/"); + if (strripos($data['Source_Code'],'?')){ + $data['Source_Code']=substr($data['Source_Code'],0,strpos($data['Source_Code'],'?')); + $data['Source_Code']=str_ireplace("?",'',$data['Source_Code']); } - if (isset($data['date']) and !is_null($data['date'])){ - $tmp=date('Y-m-d H:m:i',strtotime($data['date'])); - if (verifyDateTime($tmp)){ - $data['IndexDate']=date('Y-m-d H:i:s',strtotime($tmp)); - }elseif (verifyDate($tmp)){ - $data['IndexDate']=date('Y-m-d H:i:s',strtotime($tmp)); + $data['Source_Code']=str_ireplace("'","%27",$data['Source_Code']); + $data['Source_Code']=trim($data['Source_Code']); + $data['Source_Code']=mb_trim($data['Source_Code'],"/"); + $sc=BaseAPI."/api/v1/source_code?sc=".$data['Source_Code']; +// Error keeps popping up HERE, GRR + $sc=@file_get_contents($sc,false,$context); +// Error keeps popping up HERE, GRR + $sc=json_decode($sc); + if (!isset($sc[0])){ + if (isset($node->getElementsByTagName('title')->item(0)->nodeValue)){ + $data['Title']=clean_string($node->getElementsByTagName('title')->item(0)->nodeValue); + }else{$data['Title']=NULL;$data['G2G']=-1;} + if (isset($node->getElementsByTagName('description')->item(0)->nodeValue)){ + $RSSdata['desc']=clean_string($node->getElementsByTagName('description')->item(0)->nodeValue); + if (isset($RSSdata['desc']) and $RSSdata['desc']!==""){ + $data['Description']=strip_tags($RSSdata['desc'],""); + $data['Description']=str_ireplace("\n","",$data['Description']); + $data['Description']=str_ireplace("[…]","...",$data['Description']); + $data['Description']=substr($data['Description'],0,1000); + if (str_contains($data['Description'],'utm_source')){ + $data['Description']=substr($data['Description'],0,strpos($data['Description'],'?')); + $data['Description']=str_ireplace("?",'',$data['Description']); + } + for ($x=0;$x<=50;$x++){$data['Description']=str_ireplace(' ',' ',$data['Description']);} + $data['Description']=trim($data['Description']); + $data['Description']=mb_trim($data['Description']); + }else{$data['Description']=NULL;$data['G2G']=-1;} + }else{$data['Description']=NULL;$data['G2G']=-1;} + if (isset($node->getElementsByTagName('pubDate')->item(0)->nodeValue)){ + $data['date']=$node->getElementsByTagName('pubDate')->item(0)->nodeValue; + }elseif (isset($node->getElementsByTagName('dc:date')->item(0)->nodeValue)){ + $data['date']=$node->getElementsByTagName('dc:date')->item(0)->nodeValue; } - }else{$data['IndexDate']=NULL;} - if (is_null($data['link'])){ - log_files("Missing Link SiteID: ".$data['SiteID']."\t@ ".__LINE__."\n",LOG."Missing_Data.log"); - $flag++;$newLink=0; - } - if (!is_null($data['Title']) and !is_null($data['Description']) and !is_null($data['Source_Code'])){ - $data['KWords']=KWords($data['Title'],$data['Description'],$data['SiteName']); - $data['Description']=strip_tags($data['Description']); - } - if (isset($data['IndexDate']) and !is_null($data['IndexDate'])){ - if (!isset($new_date)){ - $new_date=date('Y-m-d H:m:i',strtotime($data['IndexDate'])); + if (isset($data['date']) and !is_null($data['date'])){ + $tmp=date('Y-m-d H:m:i',strtotime($data['date'])); + if (verifyDateTime($tmp)){ + $data['IndexDate']=date('Y-m-d H:i:s',strtotime($tmp)); + }elseif (verifyDate($tmp)){ + $data['IndexDate']=date('Y-m-d H:i:s',strtotime($tmp)); + } + }else{$data['IndexDate']=NULL;$data['G2G']=-1;} + if (is_null($data['link'])){ + log_files("Missing Link SiteID: ".$data['SiteID']."\t@ ".__LINE__."\n",LOG."Missing_Data.log"); + $flag++;$newLink=0; + } + if (!is_null($data['Title']) and !is_null($data['Description']) and !is_null($data['Source_Code'])){ + $data['KWords']=KWords($data['Title'],$data['Description'],$data['SiteName']); + $data['Description']=strip_tags($data['Description']); + }else{$data['G2G']=-1;} + if (isset($data['IndexDate']) and !is_null($data['IndexDate'])){ + if (!isset($new_date)){ + $new_date=date('Y-m-d H:m:i',strtotime($data['IndexDate'])); + }else{ + if (!is_null($new_date)){ + if (strtotime($new_date)10){ + $G2G=$data['G2G'];$Count++; + }else{$G2G=$data['G2G'];} + }else{ + $data['G2G']=-3; + } + if (!is_null($data['KWords']) and $data['G2G']>10){ + if (str_contains($data['KWords'],'homeless') or str_contains($data['KWords'],'houseless') or str_contains($data['KWords'],'homelessness')){ + if ($data['G2G']>10){$data['PageID']=$data['G2G'];} + $homeless++; + if (DEBUG==0){$data['caller']=$caller;Fediverse ($data);} } } } - }else{ - $data['IndexDate']=NULL; - } - if (DEBUG==1){ - if (is_null($data['datetime']) or $data['datetime']=="" or !isset($data['datetime'])){$data['datetime']="Never";} - log_files("SiteID:\t\t".pad_8($data['SiteID']),LOG.pad_8($data['SiteID']).'-TidySearch.log'); - log_files("RemoteID:\t".$data['RemoteID'],LOG.pad_8($data['SiteID']).'-TidySearch.log'); - log_files("BotID:\t\t".pad_3($BOT['MyID']),LOG.pad_8($data['SiteID']).'-TidySearch.log'); - log_files("Site_Name:\t".$data['SiteName'],LOG.pad_8($data['SiteID']).'-TidySearch.log'); - if (isset($new_date)){log_files("Last Update:\t".$new_date,LOG.pad_8($data['SiteID']).'-TidySearch.log');} - log_files("URL:\t\t".$data['feed'],LOG.pad_8($data['SiteID']).'-TidySearch.log'); - log_files("Source_Code:\t".$data['Source_Code'],LOG.pad_8($data['SiteID']).'-TidySearch.log'); - log_files("IndexDate:\t".$data['IndexDate'],LOG.pad_8($data['SiteID']).'-TidySearch.log'); - if (!is_null($data['Title']) and !is_null($data['Description']) and !is_null($data['KWords'])){ - log_files("Title:\t\t".$data['Title'],LOG.pad_8($data['SiteID']).'-TidySearch.log'); - log_files("Description:\t".$data['Description'],LOG.pad_8($data['SiteID']).'-TidySearch.log'); - log_files("KWords:\t\t".$data['KWords'],LOG.pad_8($data['SiteID']).'-TidySearch.log'); - } - log_files("SiteNotes:\t".$data['SiteNotes'],LOG.pad_8($data['SiteID']).'-TidySearch.log'); - } - if (!is_null($data['Title']) and !is_null($data['Description']) and !is_null($data['Source_Code']) and !is_null($data['IndexDate'])){ - $tmp1=htmlentities($data['Title'],ENT_QUOTES|ENT_HTML5,'UTF-8'); - $tmp1="'".str_ireplace('\'',''',$tmp1)."'"; - $data['Description']=strip_tags($data['Description']); - $tmp2=htmlentities($data['Description'],ENT_QUOTES|ENT_HTML5,'UTF-8'); - $tmp2="'".str_ireplace('\'',''',$tmp2)."'"; - $data['DataType']=1; - $data['PAGES']="INSERT INTO tidysearch.pages (site_id,date,title,description,kwords,source_code,botid,hostid,lan_code,rank)VALUES(".pad_5($data['SiteID']).",'".$data['IndexDate']."',".$tmp1.",".$tmp2.",'".$data['KWords']."','".$data['Source_Code']."',".$BOT['MyID'].",3,NULL,0);"; - if (DEBUG==1){log_files("PAGES:\t\t".$data['PAGES'],LOG.pad_8($data['SiteID']).'-TidySearch.log');} - if (!is_null($data['Title']) and !is_null($data['Description']) and !is_null($data['KWords']) and DEBUG==0){ - $data['G2G']=send_2_server($BOT,$data); - if ($data['G2G']>10){ - $G2G=$data['G2G'];$Count++; - log_files($data['PAGES']." -- Page Idex: ".$data['G2G'],SQL.pad_8($data['SiteID']).'-TidySearch.sql'); - } + if (!is_null($data['Title'])){ + $display=pad_space(html_entity_decode($data['Title'],ENT_QUOTES|ENT_HTML5,'UTF-8')); }else{ - $data['G2G']=0; + $display=pad_space(" "); } - if (!is_null($data['KWords'])){ - if (str_contains($data['KWords'],'homeless') or str_contains($data['KWords'],'houseless') or str_contains($data['KWords'],'homelessness')){ - $homeless++; - if (DEBUG==0){Fediverse ($data);} - } - } - } - if (!is_null($data['Title'])){ - $display=pad_space(html_entity_decode($data['Title'],ENT_QUOTES|ENT_HTML5,'UTF-8')); + if (!isset($data['G2G'])){$data['G2G']=-1;} + echo pad_5($data['SiteID'])." ".pad_5($data['RSSindex'])." ".pad_3($BOT['MyID'])." ".pad_3($Count)." ".$data['IndexDate']." - ".$display."\t".memory_get_usage()."\t".pad_r(time_check($StartTime))."\tG2G: ".$data['G2G']."\n"; + if (DEBUG==1){log_files("\t\t******",LOG.pad_8($data['SiteID']).'-TidySearch.log');} }else{ - $display=pad_space(" "); + $display=pad_space("No New Data"); + if (!isset($data['G2G'])){$data['G2G']=-10;} + echo pad_5($data['SiteID'])." ".pad_5($data['RSSindex'])." ".pad_3($BOT['MyID'])." ".pad_3($Count)." - ".$display."\t".memory_get_usage()."\t".pad_r(time_check($StartTime))."\tG2G: ".$data['G2G']."\n"; } - echo pad_5($data['SiteID'])." ".pad_5($data['RSSindex'])." ".pad_3($BOT['MyID'])." ".pad_3($Count)." ".$data['IndexDate']." - ".$display."\t".memory_get_usage()."\t".time_check($StartTime)."\tG2G: ".$data['G2G']."\n"; - if (DEBUG==1){log_files("\t\t******",LOG.pad_8($data['SiteID']).'-TidySearch.log');} - }else{ - $display=pad_space("No New Data"); - echo pad_5($data['SiteID'])." ".pad_5($data['RSSindex'])." ".pad_3($BOT['MyID'])." ".pad_3($Count)." - ".$display."\t".memory_get_usage()."\t".time_check($StartTime)."\tG2G: ".$data['G2G']."\n"; } } - } - if ($Count>0){ - $hits=$data['hits']+$Count; - $data['PAGES']=$data['SITES']=$data['RSS']=NULL; - $data['DataType']=2; - $data['SITES']="UPDATE tidysearch.sites SET datetime='".$new_date."' WHERE sites.index=".$data['SiteID'].";"; - if (DEBUG==0){$data['G2G']=send_2_server($BOT,$data);} - $data['DataType']=3; - $data['RSS']="UPDATE tidysearch.sites_rss SET datetime='".$new_date."',hits=".$hits." WHERE sites_rss.index=".$data['RSSindex'].";"; - if (DEBUG==0){$data['G2G']=send_2_server($BOT,$data);} - log_files($data['SITES']."\n".$data['RSS'],SQL.pad_8($data['SiteID']).'-TidySearch.sql'); - $count=$count+$Count; - }else{ - $data['DataType']=3; - $date=date('Y-m-d H:i:s'); - $data['RSS']="UPDATE tidysearch.sites_rss SET datetime='".$date."' WHERE sites_rss.index=".$data['RSSindex'].";"; - if (DEBUG==0){$data['G2G']=send_2_server($BOT,$data);} + if ($Count>0){ + $hits=$data['hits']+$Count; + $data['DataType']=3; + $data['RSS']="UPDATE tidysearch.sites_rss SET datetime='".$new_date."',hits=".$hits." WHERE sites_rss.index=".$data['RSSindex'].";"; + if (DEBUG==0){$data['G2G']=send_2_server($BOT,$data);} + $count=$count+$Count; + }else{ + $data['DataType']=3; + $date=date('Y-m-d H:i:s'); + $data['RSS']="UPDATE tidysearch.sites_rss SET datetime='".$date."' WHERE sites_rss.index=".$data['RSSindex'].";"; + if (DEBUG==0){$data['G2G']=send_2_server($BOT,$data);} + } } $Count=0; $RSSindex=$data['RSSindex']; unset($data); echo "\n"; } -$display="BotID: ".pad_3($BOT['MyID'])."\nSites: ".pad_3($websites)." Feeds: ".pad_3($count)."\nHomeless: ".pad_3($homeless)."\n".time_check($StartTime); -if (DEBUG==0 and $count>0){send_group("NewsBot".$BOT['MyID'],"tidybot@lobby.mpaq.org",$display,"mpaq.org");} +$data['DataType']=4; +$now=date('Y-m-d'); +$proc_time=time_check($StartTime); +$data['Stats']=$now.":&:".$BOT['MyID'].":&:".$websites.":&:".$count.":&:".$homeless.":&:".$proc_time; +$data['SiteID']=0; +if (DEBUG==0){$data['G2G']=send_2_server($BOT,$data);} +$display="\nSites: ".pad_3($websites)." Links: ".pad_3($count)." Home: ".pad_3($homeless)."\nRSS_IDs: ".$RID."\n".round($proc_time,2); if (isset($G2G) and $G2G>0){ - echo "\nData Sent: ".$display."\t\tDone\007\n";sleep(1);echo "\007"; + $display=$caller." Data Sent:".$display; }else{ - echo "\nNo Data Sent: ".$display."\t\tDone\007\n";sleep(1);echo "\007"; + $display=$caller." No Data Sent:".$display; } +if (DEBUG==0){send_group($caller,"tidybot@lobby.mpaq.org",$display,"mpaq.org");} +echo $display."\t\tDone\007\n";sleep(1);echo "\007"; if (file_exists($lockFile)){unlink($lockFile);} //////////////////////////// The End //////////////////////////// //////////////////////////// The End //////////////////////////// //////////////////////////// The End //////////////////////////// function Fediverse ($data){ include_once('/media/10T/ADMIN/.BotSolo.php'); + include_once('/var/www/00_00/class/Mastodon.php'); + $caller=$data['caller']; $display=strip_tags($data['Description']); $display=substr($display,0,1000); $report=$data['Title']."\n\n".$display."\n"; unset($display); $report=str_ireplace('… Continue reading →','...',$report); $report=str_ireplace('’','\'',$report); + $report=str_ireplace('‘','\'',$report); $report=str_replace('homelessness','#homelessness',$report); $report=str_replace('houseless','#houseless',$report); $report=str_replace('homeless','#homeless',$report); @@ -220,41 +236,33 @@ function Fediverse ($data){ $report=str_replace('Houseless','#Houseless',$report); $report=str_replace('Homeless','#Homeless',$report); $report=str_ireplace('##','#',$report); - $report=str_ireplace(' ','',$report); - $report=$report.date('M j,Y',strtotime($data['IndexDate']))." #Homeless #Poverty\n".$data['Source_Code']."\n\n#TidySearch Bot".$data['botid']." (".$data['SiteID'].") For more posts, follow @newsbot @ https://beamship.mpaq.org and Visit http://search.mpaq.org for more."; + $report=$report."\n".$data['Source_Code']."\n".date('M j,Y',strtotime($data['IndexDate']))."\n\n#TidySearch Bot".$data['botid']." (".$data['SiteID'].") For more visit http://search.mpaq.org"; + $report=$report." #Homeless #Poverty\n"; for ($x=0;$x<=50;$x++){$report=str_ireplace(' ',' ',$report);} -/* - if (!is_null($report)){ - $headers = [ - 'Authorization: Bearer '.TOKEN, - 'Content-Type: multipart/form-data', + if (!is_null($report) and isset($MTOKEN) and isset($M_URL)){ + $mastodon=new MastodonAPI($MTOKEN,$M_URL); + $status_data=[ + 'status'=> $report, + 'visibility'=> 'public', + 'language'=> 'en', ]; - $ch = curl_init(); - curl_setopt($ch, CURLOPT_URL, Mast_URL."/api/v1/statuses"); - curl_setopt($ch, CURLOPT_CUSTOMREQUEST, "Post"); - curl_setopt($ch, CURLOPT_POSTFIELDS, $report); - curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); - curl_setopt($ch, CURLOPT_HTTPHEADER, $headers); - curl_setopt($ch, CURLOPT_USERAGENT,"MPAQ"); -var_dump($ch); - $reply = curl_exec($ch); -echo $report."\n\n".$reply."\n";exit; - if (!$reply) {return json_encode(['ok'=>false, 'curl_error_code' => curl_errno($ch), 'curl_error' => curl_error($ch)]);} - return json_decode($reply, true); -*/ - log_files($report,LOG.pad_3($data['SiteID'])."-Homeless.log"); - send_group("NewsBot".$data['botid'],"news@lobby.mpaq.org",$report,"mpaq.org"); - log_files("\t*****",LOG.pad_3($data['SiteID'])."-Homeless.log"); -// } + $status=$mastodon->postStatus($status_data); + } + if (isset($status['id']) and isset($data['PageID'])){ + $send=$caller.": NewsFeed Posted ".$data['SiteID']."-".$data['PageID']." ".M_URL."/@newsbot/statuses/".$status['id']; + }elseif (isset($data['PageID'])){ + $send=$caller." ".$data['SiteID']."-".$data['PageID'].": NewsFeed \n".$report; + }else{ + $send=$caller." ".$data['SiteID'].": NewsFeed \n".$report; + } + $send=str_ireplace('’','',$send); + $send=str_ireplace('\'','',$send); + send_group($caller,"news@lobby.mpaq.org",$send,"mpaq.org"); + log_files($send,LOG."Mastodon.log"); } function time_check($StartTime){ $tmp=microtime(true)-$StartTime; - if ($tmp>60){ - $tmp=$tmp/60;$return=pad_2(round($tmp,2))."M"; - }else{ - $return=pad_2(round($tmp,2))."S"; - } - return $return; + return round($tmp,3); } function log_files($notes,$name){ $entry=$notes."\n"; @@ -274,37 +282,28 @@ function sort_words($input){ } function verifyDateTime($date, $format = 'Y-m-d H:i:s'){ if (strtotime($date)format($format) === $date; +// $year=date('Y',strtotime($date)); +// if ($year>1970){ + $dd = DateTime::createFromFormat($format, $date); + return $dd && $dd->format($format) === $date; +// }else{ +// return null; +// } }else{ return null; } } function verifyDate($date, $format = 'Y-m-d'){ - if (strtotime($date)format($format) === $date; - }else{ +// $year=date('Y',strtotime($date)); +// if ($year>1970){ + $dd = DateTime::createFromFormat($format, $date); + return $dd && $dd->format($format) === $date; +// }else{ +// return null; +// } +// }else{ return null; - } -} -function test_channel($url){ - ini_set('display_errors',1);ini_set('display_startup_errors',1);error_reporting(E_ALL); - include('/media/10T/ADMIN/.tidysearch.php'); - $curl_test = array( - CURLOPT_HEADER => true, // we want headers - CURLOPT_NOBODY => true, // we don't need body - CURLOPT_RETURNTRANSFER => 1, - CURLOPT_TIMEOUT => 2, // timeout on response - CURLOPT_USERAGENT => "TidySearch Engine: visit http://search.mpaq.org for details.", // who am i - CURLOPT_CONNECTTIMEOUT => 2, // timeout on connect - CURLOPT_MAXREDIRS => 5, // stop after 10 redirects - ); - $ch=curl_init($url); - curl_setopt_array($ch,$curl_test); - $output=curl_exec($ch); - $httpcode=curl_getinfo($ch, CURLINFO_HTTP_CODE); - return $httpcode; +// } } function send_2_server($BOT,$data){ include_once('/media/10T/ADMIN/.BotSolo.php'); @@ -312,6 +311,10 @@ function send_2_server($BOT,$data){ if (!isset($data['SITES'])){$data['SITES']=NULL;} if (!isset($data['RSS'])){$data['RSS']=NULL;} if (!isset($data['Source_Code'])){$data['Source_Code']=NULL;} + if (!isset($data['SiteID'])){$data['SiteID']=NULL;} + if (!isset($data['RSSindex'])){$data['RSSindex']=NULL;} + if (!isset($data['SiteName'])){$data['SiteName']=NULL;} + if (!isset($data['Stats'])){$data['Stats']=NULL;} if (!isset($data['DataType'])){return NULL;} $data2send=array( 'BotPubKey' => $BOT['PubKey'], @@ -324,21 +327,24 @@ function send_2_server($BOT,$data){ 'DataSites' => encryptText($data['SITES'],$BOT['PVTKey']), 'DataRSS' => encryptText($data['RSS'],$BOT['PVTKey']), 'Source_Code' => encryptText($data['Source_Code'],$BOT['PVTKey']), + 'Stats' => encryptText($data['Stats'],$BOT['PVTKey']), ); -// $sendto=BaseAPI."collect_ts"; - $sendto="https://212.102.59.80:250/collect_ts.php"; - $responce=mb_trim(server($sendto,$data2send,$BOT['MyID'],)); - if (is_numeric($responce) and $responce>0){ - $send="API Server ".$sendto." responded with: ".$responce; - return $responce; - }else{ - if (isset($data['SiteAddress'])){ - $send="Line ".__LINE__." API Responce Error!!! Aborting...\n".$data['LocalID']." ".mb_trim($data['SiteAddress'])." from ".$sendto."\n\n".$responce; - echo $send."\n"; - var_dump($data); - echo "\007";sleep(1);echo "\007";sleep(1);echo "\007";sleep(1); - exit; + $sendto=BaseAPI."/collect_ts.php"; + if (isset($sendto) and isset($data2send) and isset($BOT)){ + $server=server($sendto,$data2send,$BOT['MyID']); + if (isset($server)){$responce=mb_trim($server);}else{echo "MPAQ Network error, abort.\n";exit;} + if (is_numeric($responce) and $responce>0){ + $send="API Server ".$sendto." responded with: ".$responce; + return $responce; + }else{ + if (isset($data['SiteAddress'])){ + $send="Line ".__LINE__." API Responce Error!!! Aborting...\n".$data['LocalID']." ".mb_trim($data['SiteAddress'])." from ".$sendto."\n\n".$responce; + echo $send."\n"; + if (DEBUG==0){send_group($caller,"system@lobby.mpaq.org",$locked,"mpaq.org");} + } } + }else{ + return -4; } } function server($sendto,$data,$botid){ @@ -347,7 +353,8 @@ function server($sendto,$data,$botid){ "Content-Type: application/json", "Authorization: Bearer your_token", ); - $config['useragent']='TidyBot (TidySearch http://search.mpaq.org)'; + $config['useragent']='TidyBot (TidySearch (beta) http://search.mpaq.org)'; + $config['cookie_file']=LOG.md5($sendto).'.txt'; $dir=dirname(__FILE__); $ch=curl_init($sendto); curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); @@ -362,7 +369,6 @@ function server($sendto,$data,$botid){ curl_setopt($ch, CURLOPT_POSTFIELDS, http_build_query($data)); $logFile=fopen(LOG."Curl.log", "a"); curl_setopt($ch, CURLOPT_STDERR, $logFile); - $config['cookie_file']=LOG.md5($sendto).'.txt'; curl_setopt($ch, CURLOPT_COOKIEFILE, $config['cookie_file']); curl_setopt($ch, CURLOPT_COOKIEJAR, $config['cookie_file']); curl_setopt($ch, CURLOPT_USERAGENT, $config['useragent']); @@ -370,11 +376,10 @@ function server($sendto,$data,$botid){ $response=curl_exec($ch); if (curl_errno($ch)){ $send=__LINE__." sendto: ".$sendto."\nresponse: ".$response."\ncurl_error: ".curl_error($ch); -// log_files($send,LOG.'API_Response.log'); - exit; + log_files($send,LOG.'API_Response.log'); }else{ - $send=__LINE__." sendto: ".$sendto."\nresponse: ".$response."\ncurl_error: ".curl_error($ch); -// log_files($send,LOG.'API_Response.log'); + $send=__LINE__." sendto: ".$sendto." ".$response; + log_files($send,LOG.'API_Response.log'); return trim($response); } } @@ -389,6 +394,16 @@ function pad_2($ThisNumber){if (isset($ThisNumber)){return str_pad($ThisNumber,2 function pad_3($ThisNumber){if (isset($ThisNumber)){return str_pad($ThisNumber,3,"0",STR_PAD_LEFT);}else{return "000";}} function pad_5($ThisNumber){if (isset($ThisNumber)){return str_pad($ThisNumber,5,"0",STR_PAD_LEFT);}else{return "00000";}} function pad_8($ThisNumber){if (isset($ThisNumber)){return str_pad($ThisNumber,8,"0",STR_PAD_LEFT);}else{return "00000000";}} +function pad_r($ThisNumber){ + if (isset($ThisNumber)){ + $ThisNumber=str_pad($ThisNumber,4,"0",STR_PAD_LEFT); + $ThisNumber=str_pad($ThisNumber,4,"0",STR_PAD_RIGHT); +//echo $ThisNumber."\n";exit; + return $ThisNumber; + }else{ + return "0000.0000"; + } +} function pad_space($This){if (strlen($This)>20){$result=substr($This,0,20);}else{$result=str_pad($This,20," ");}return $result;} function KWords($title,$desc,$site){ $INDEX_List=array( @@ -460,21 +475,98 @@ function KWords($title,$desc,$site){ }else{return NULL;} } function send_group($BName,$Group,$Send,$Server){ - if (isset($_SERVER['SERVER_NAME'])){if ($_SERVER['SERVER_NAME']=="localhost"){exit;}} - $ERROR=NULL; - $ERROR=shell_exec("echo '".$Send."' | sendxmpp --username netbot --password 4cUTyfWwN5wc7St --jserver ".$Server." --resource ".$BName." --message-type=headline --chatroom ".$Group.""); - return $ERROR; + if (isset($BName) and isset($Group) and isset($Send) and isset($Server)){ + $ERROR=NULL; + $ERROR=shell_exec("echo '".$Send."' | sendxmpp --username netbot --password 4cUTyfWwN5wc7St --jserver ".$Server." --resource ".$BName." --message-type=headline --chatroom ".$Group.""); + if (!is_null($ERROR)){echo $ERROR."\n";exit;}else{return NULL;} +// echo $ERROR."\n";exit; + } + return NULL; +} +///////////////////////////////////////////////////////// +/* +class GhostAPI{ + private $token; + private $instance_url; + public function __construct($token, $instance_url){ + $this->token = $token; + $this->instance_url = $instance_url; + } + public function postStatus($status){ + return $this->callAPI('/api/v1/post', 'POST', $status); + } + public function callAPI($endpoint, $method, $data){ + $headers = [ + 'Authorization: Bearer '.$this->token, + 'Content-Type: multipart/form-data', + ]; + $ch = curl_init(); + curl_setopt($ch, CURLOPT_URL, $this->instance_url.$endpoint); + curl_setopt($ch, CURLOPT_CUSTOMREQUEST, $method); + curl_setopt($ch, CURLOPT_POSTFIELDS, $data); + curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + curl_setopt($ch, CURLOPT_HTTPHEADER, $headers); + curl_setopt($ch, CURLOPT_USERAGENT,"MPAQ"); + $reply = curl_exec($ch); + if (!$reply) { + return json_encode(['ok'=>false, 'curl_error_code' => curl_errno($ch), 'curl_error' => curl_error($ch)]); + } + return json_decode($reply, true); + } } /* +class MastodonAPI{ +// $token=MTOKEN; +// $instance_url=M_URL; + private $token; + private $instance_url; + public function __construct($token, $instance_url){ + $this->token = $token; + $this->instance_url = $instance_url; + } + public function postStatus($status){return $this->callAPI('/api/v1/statuses', 'POST', $status);} + public function uploadMedia($media){return $this->callAPI('/api/v1/media', 'POST', $media);} -UPDATE tidysearch.sites SET url='',name='' WHERE sites.index=1538; + public function callAPI($endpoint, $method, $data){ + $headers = [ + 'Authorization: Bearer '.$this->token, + 'Content-Type: multipart/form-data', + ]; + $ch = curl_init(); + curl_setopt($ch, CURLOPT_URL, $this->instance_url.$endpoint); + curl_setopt($ch, CURLOPT_CUSTOMREQUEST, $method); + curl_setopt($ch, CURLOPT_POSTFIELDS, $data); + curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + curl_setopt($ch, CURLOPT_HTTPHEADER, $headers); + curl_setopt($ch, CURLOPT_USERAGENT, "curl"); + $reply = curl_exec($ch); + if (!$reply) { + log_files(curl_errno($ch)."\n\t".$instance_url."@".$instance_url,LOG."MastodonAPI.log"); + return json_encode(['ok'=>false, 'curl_error_code' => curl_errno($ch), 'curl_error' => curl_error($ch)]); + } + log_files(curl_errno($ch)."\n\t".$instance_url."@".$instance_url,LOG."MastodonAPI.log"); + return json_decode($reply, true); + } +} +/* +UPDATE tidysearch.pages SET _lang=null WHERE sites._lang=""; +UPDATE tidysearch.sites_rss SET url='https://ojrock.mpaq.org/feed' WHERE sites_rss.index=437; -UPDATE tidysearch.sites_rss SET botid='404' WHERE sites_rss.index=587; -UPDATE tidysearch.sites SET botid=0,hits=0,name='',url='' WHERE sites.index=xxx; -UPDATE tidysearch.sites_rss SET botid=0,hits=0,site_name='',url='' WHERE sites_rss.index=xxx; -UPDATE tidysearch.sites SET botid=0 WHERE sites.botid=9000; -UPDATE tidysearch.sites_rss SET botid=0 WHERE sites_rss.botid=9000; -UPDATE tidysearch.sites SET botid=0 WHERE sites.botid=9001; -UPDATE tidysearch.sites_rss SET botid=0 WHERE sites_rss.botid=9001; +DELETE FROM tidysearch.pages WHERE pages.title is null; +DELETE FROM tidysearch.pages WHERE pages.title LIKE ""; + +DELETE FROM tidysearch.pages WHERE pages.date = NULL; + +delete from tidysearch.pages where kwords is null; +SELECT * FROM tidysearch.pages where pages.description like ''; + +SELECT * FROM tidysearch.pages ORDER BY pages.date DESC LIMIT 10; + +https://news.mpaq.org/sitemap-posts.xml +https://news.mpaq.org/.ghost/activitypub/nodeinfo/2.1 + +OpenFire chatter HWrBVELMAh8YYdk + +https://bots.mpaq.org/.well-known/webfinger?resource=acct:@bob@mpaq.org */ ?>