['verify_peer' => false,'verify_peer_name' => false]]; $context=stream_context_create($options); $json=file_get_contents(SYNC,false,$context); $Site=json_decode($json); $websites=$next=0; while(isset($Site[$next])){$next++;}$last=$next=$next-1;$next=0; echo "\t\tUsing ".SYNC."\n"; for ($site=$next;$next<=$last;$next++){ $websites++;$data['G2G']=0; $data['RSSindex']=$Site[$next]->index; $data['SiteID']=$Site[$next]->sitesid; $data['RemoteID']=$Site[$next]->remoteid; $data['SiteName']=$Site[$next]->sitename; $data['datetime']=$Site[$next]->datetime; $data['hits']=$Site[$next]->hits; if ($Site[$next]->botid==0){ $data['botid']=$BOT['MyID']; }else{ $data['botid']=$Site[$next]->botid; } $data['feed']=$Site[$next]->url; $data['SiteNotes']=$Site[$next]->site_notes; $HTMLCode=test_channel($data['feed']); if ($HTMLCode>399){ $data['DataType']=2; $data['SITES']="UPDATE tidysearch.sites SET botid='".$HTMLCode."',datetime=NULL WHERE sites.index=".$data['SiteID'].";"; $data['G2G']=send_2_server($BOT,$data); $data['DataType']=3; $data['RSS']="UPDATE tidysearch.sites_rss SET botid='".$HTMLCode."' WHERE sites_rss.index=".$data['RSSindex'].";"; $data['G2G']=send_2_server($BOT,$data); echo "Disabled Feed SiteID: ".$data['SiteID']." RSS: ".$data['RSSindex']."\n"; } if ($data['RSSindex']>0){$RID=trim($RID." ".pad_5($data['RSSindex']));} $rss=new DOMDocument(); @$rss->load($data['feed']); $feed = array(); if ($rss->getElementsByTagName('item')){ foreach ($rss->getElementsByTagName('item') as $node){ $data['G2G']=$newLink=0; if (isset($node->getElementsByTagName('link')->item(0)->nodeValue)){ $data['link']=$node->getElementsByTagName('link')->item(0)->nodeValue; $data['Source_Code']=mb_trim($data['link'],"/"); if (strripos($data['Source_Code'],'?')){ $data['Source_Code']=substr($data['Source_Code'],0,strpos($data['Source_Code'],'?')); $data['Source_Code']=str_ireplace("?",'',$data['Source_Code']); } $data['Source_Code']=str_ireplace("'","%27",$data['Source_Code']); $data['Source_Code']=trim($data['Source_Code']); $data['Source_Code']=mb_trim($data['Source_Code'],"/"); $sc=BaseAPI."/api/v1/source_code?sc=".$data['Source_Code']; // Error keeps popping up HERE, GRR $sc=@file_get_contents($sc,false,$context); // Error keeps popping up HERE, GRR $sc=json_decode($sc); if (!isset($sc[0])){ if (isset($node->getElementsByTagName('title')->item(0)->nodeValue)){ $data['Title']=clean_string($node->getElementsByTagName('title')->item(0)->nodeValue); }else{$data['Title']=NULL;$data['G2G']=-1;} if (isset($node->getElementsByTagName('description')->item(0)->nodeValue)){ $RSSdata['desc']=clean_string($node->getElementsByTagName('description')->item(0)->nodeValue); if (isset($RSSdata['desc']) and $RSSdata['desc']!==""){ $data['Description']=strip_tags($RSSdata['desc'],""); $data['Description']=str_ireplace("\n","",$data['Description']); $data['Description']=str_ireplace("[…]","...",$data['Description']); $data['Description']=substr($data['Description'],0,1000); if (str_contains($data['Description'],'utm_source')){ $data['Description']=substr($data['Description'],0,strpos($data['Description'],'?')); $data['Description']=str_ireplace("?",'',$data['Description']); } for ($x=0;$x<=50;$x++){$data['Description']=str_ireplace(' ',' ',$data['Description']);} $data['Description']=trim($data['Description']); $data['Description']=mb_trim($data['Description']); }else{$data['Description']=NULL;$data['G2G']=-1;} }else{$data['Description']=NULL;$data['G2G']=-1;} if (isset($node->getElementsByTagName('pubDate')->item(0)->nodeValue)){ $data['date']=$node->getElementsByTagName('pubDate')->item(0)->nodeValue; }elseif (isset($node->getElementsByTagName('dc:date')->item(0)->nodeValue)){ $data['date']=$node->getElementsByTagName('dc:date')->item(0)->nodeValue; } if (isset($data['date']) and !is_null($data['date'])){ $tmp=date('Y-m-d H:m:i',strtotime($data['date'])); if (verifyDateTime($tmp)){ $data['IndexDate']=date('Y-m-d H:i:s',strtotime($tmp)); }elseif (verifyDate($tmp)){ $data['IndexDate']=date('Y-m-d H:i:s',strtotime($tmp)); } }else{$data['IndexDate']=NULL;$data['G2G']=-1;} if (is_null($data['link'])){ log_files("Missing Link SiteID: ".$data['SiteID']."\t@ ".__LINE__."\n",LOG."Missing_Data.log"); $flag++;$newLink=0; } if (!is_null($data['Title']) and !is_null($data['Description']) and !is_null($data['Source_Code'])){ $data['KWords']=KWords($data['Title'],$data['Description'],$data['SiteName']); $data['Description']=strip_tags($data['Description']); }else{$data['G2G']=-1;} if (isset($data['IndexDate']) and !is_null($data['IndexDate'])){ if (!isset($new_date)){ $new_date=date('Y-m-d H:m:i',strtotime($data['IndexDate'])); }else{ if (!is_null($new_date)){ if (strtotime($new_date)10){ $G2G=$data['G2G'];$Count++; }else{$G2G=$data['G2G'];} }else{ $data['G2G']=-3; } if (!is_null($data['KWords']) and $data['G2G']>10){ if (str_contains($data['KWords'],'homeless') or str_contains($data['KWords'],'houseless') or str_contains($data['KWords'],'homelessness')){ if ($data['G2G']>10){$data['PageID']=$data['G2G'];} $homeless++; if (DEBUG==0){$data['caller']=$caller;Fediverse ($data);} } } } if (!is_null($data['Title'])){ $display=pad_space(html_entity_decode($data['Title'],ENT_QUOTES|ENT_HTML5,'UTF-8')); }else{ $display=pad_space(" "); } if (!isset($data['G2G'])){$data['G2G']=-1;} echo pad_5($data['SiteID'])." ".pad_5($data['RSSindex'])." ".pad_3($BOT['MyID'])." ".pad_3($Count)." ".$data['IndexDate']." - ".$display."\t".memory_get_usage()."\t".pad_r(time_check($StartTime))."\tG2G: ".$data['G2G']."\n"; if (DEBUG==1){log_files("\t\t******",LOG.pad_8($data['SiteID']).'-TidySearch.log');} }else{ $display=pad_space("No New Data"); if (!isset($data['G2G'])){$data['G2G']=-10;} echo pad_5($data['SiteID'])." ".pad_5($data['RSSindex'])." ".pad_3($BOT['MyID'])." ".pad_3($Count)." - ".$display."\t".memory_get_usage()."\t".pad_r(time_check($StartTime))."\tG2G: ".$data['G2G']."\n"; } } } if ($Count>0){ $hits=$data['hits']+$Count; $data['DataType']=3; $data['RSS']="UPDATE tidysearch.sites_rss SET datetime='".$new_date."',hits=".$hits." WHERE sites_rss.index=".$data['RSSindex'].";"; if (DEBUG==0){$data['G2G']=send_2_server($BOT,$data);} $count=$count+$Count; }else{ $data['DataType']=3; $date=date('Y-m-d H:i:s'); $data['RSS']="UPDATE tidysearch.sites_rss SET datetime='".$date."' WHERE sites_rss.index=".$data['RSSindex'].";"; if (DEBUG==0){$data['G2G']=send_2_server($BOT,$data);} } } $Count=0; $RSSindex=$data['RSSindex']; unset($data); echo "\n"; } $data['DataType']=4; $now=date('Y-m-d'); $proc_time=time_check($StartTime); $data['Stats']=$now.":&:".$BOT['MyID'].":&:".$websites.":&:".$count.":&:".$homeless.":&:".$proc_time; $data['SiteID']=0; if (DEBUG==0){$data['G2G']=send_2_server($BOT,$data);} $display="\nSites: ".pad_3($websites)." Links: ".pad_3($count)." Home: ".pad_3($homeless)."\nRSS_IDs: ".$RID."\n".round($proc_time,2); if (isset($G2G) and $G2G>0){ $display=$caller." Data Sent:".$display; }else{ $display=$caller." No Data Sent:".$display; } if (DEBUG==0){send_group($caller,"tidybot@lobby.mpaq.org",$display,"mpaq.org");} echo $display."\t\tDone\007\n";sleep(1);echo "\007"; if (file_exists($lockFile)){unlink($lockFile);} //////////////////////////// The End //////////////////////////// //////////////////////////// The End //////////////////////////// //////////////////////////// The End //////////////////////////// function Fediverse ($data){ include_once('/media/10T/ADMIN/.BotSolo.php'); include_once('/var/www/00_00/class/Mastodon.php'); $caller=$data['caller']; $display=strip_tags($data['Description']); $display=substr($display,0,1000); $report=$data['Title']."\n\n".$display."\n"; unset($display); $report=str_ireplace('… Continue reading →','...',$report); $report=str_ireplace('’','\'',$report); $report=str_ireplace('‘','\'',$report); $report=str_replace('homelessness','#homelessness',$report); $report=str_replace('houseless','#houseless',$report); $report=str_replace('homeless','#homeless',$report); $report=str_replace('Homelessness','#Homelessness',$report); $report=str_replace('Houseless','#Houseless',$report); $report=str_replace('Homeless','#Homeless',$report); $report=str_ireplace('##','#',$report); $report=$report."\n".$data['Source_Code']."\n".date('M j,Y',strtotime($data['IndexDate']))."\n\n#TidySearch Bot".$data['botid']." (".$data['SiteID'].") For more visit http://search.mpaq.org"; $report=$report." #Homeless #Poverty\n"; for ($x=0;$x<=50;$x++){$report=str_ireplace(' ',' ',$report);} if (!is_null($report) and isset($MTOKEN) and isset($M_URL)){ $mastodon=new MastodonAPI($MTOKEN,$M_URL); $status_data=[ 'status'=> $report, 'visibility'=> 'public', 'language'=> 'en', ]; $status=$mastodon->postStatus($status_data); } if (isset($status['id']) and isset($data['PageID'])){ $send=$caller.": NewsFeed Posted ".$data['SiteID']."-".$data['PageID']." ".M_URL."/@newsbot/statuses/".$status['id']; }elseif (isset($data['PageID'])){ $send=$caller." ".$data['SiteID']."-".$data['PageID'].": NewsFeed \n".$report; }else{ $send=$caller." ".$data['SiteID'].": NewsFeed \n".$report; } $send=str_ireplace('’','',$send); $send=str_ireplace('\'','',$send); send_group($caller,"news@lobby.mpaq.org",$send,"mpaq.org"); log_files($send,LOG."Mastodon.log"); } function time_check($StartTime){ $tmp=microtime(true)-$StartTime; return round($tmp,3); } function log_files($notes,$name){ $entry=$notes."\n"; if (!file_exists($name)){touch ($name);} $data_file=$name; $handle=fopen($name, 'r'); $new_data_file=fopen($data_file, 'a+') or die('Cannot open file: '.$data_file); fwrite($new_data_file, $entry); fclose($new_data_file); fclose($handle); } function sort_words($input){ $words=explode(" ", $input); sort($words); $sorted_string=implode(" ", $words); return $sorted_string; } function verifyDateTime($date, $format = 'Y-m-d H:i:s'){ if (strtotime($date)1970){ $dd = DateTime::createFromFormat($format, $date); return $dd && $dd->format($format) === $date; // }else{ // return null; // } }else{ return null; } } function verifyDate($date, $format = 'Y-m-d'){ // $year=date('Y',strtotime($date)); // if ($year>1970){ $dd = DateTime::createFromFormat($format, $date); return $dd && $dd->format($format) === $date; // }else{ // return null; // } // }else{ return null; // } } function send_2_server($BOT,$data){ include_once('/media/10T/ADMIN/.BotSolo.php'); if (!isset($data['PAGES'])){$data['PAGES']=NULL;} if (!isset($data['SITES'])){$data['SITES']=NULL;} if (!isset($data['RSS'])){$data['RSS']=NULL;} if (!isset($data['Source_Code'])){$data['Source_Code']=NULL;} if (!isset($data['SiteID'])){$data['SiteID']=NULL;} if (!isset($data['RSSindex'])){$data['RSSindex']=NULL;} if (!isset($data['SiteName'])){$data['SiteName']=NULL;} if (!isset($data['Stats'])){$data['Stats']=NULL;} if (!isset($data['DataType'])){return NULL;} $data2send=array( 'BotPubKey' => $BOT['PubKey'], 'BotID' => encryptText($BOT['MyID'],$BOT['PVTKey']), 'SiteID' => encryptText($data['SiteID'],$BOT['PVTKey']), 'RSSindex' => encryptText($data['RSSindex'],$BOT['PVTKey']), 'SiteName' => encryptText($data['SiteName'],$BOT['PVTKey']), 'DataType' => encryptText($data['DataType'],$BOT['PVTKey']), 'DataPage' => encryptText($data['PAGES'],$BOT['PVTKey']), 'DataSites' => encryptText($data['SITES'],$BOT['PVTKey']), 'DataRSS' => encryptText($data['RSS'],$BOT['PVTKey']), 'Source_Code' => encryptText($data['Source_Code'],$BOT['PVTKey']), 'Stats' => encryptText($data['Stats'],$BOT['PVTKey']), ); $sendto=BaseAPI."/collect_ts.php"; if (isset($sendto) and isset($data2send) and isset($BOT)){ $server=server($sendto,$data2send,$BOT['MyID']); if (isset($server)){$responce=mb_trim($server);}else{echo "MPAQ Network error, abort.\n";exit;} if (is_numeric($responce) and $responce>0){ $send="API Server ".$sendto." responded with: ".$responce; return $responce; }else{ if (isset($data['SiteAddress'])){ $send="Line ".__LINE__." API Responce Error!!! Aborting...\n".$data['LocalID']." ".mb_trim($data['SiteAddress'])." from ".$sendto."\n\n".$responce; echo $send."\n"; if (DEBUG==0){send_group($caller,"system@lobby.mpaq.org",$locked,"mpaq.org");} } } }else{ return -4; } } function server($sendto,$data,$botid){ include_once('/media/10T/ADMIN/.BotSolo.php'); $headers=array( "Content-Type: application/json", "Authorization: Bearer your_token", ); $config['useragent']='TidyBot (TidySearch (beta) http://search.mpaq.org)'; $config['cookie_file']=LOG.md5($sendto).'.txt'; $dir=dirname(__FILE__); $ch=curl_init($sendto); curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); curl_setopt($ch, CURLOPT_POST, true); curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true); curl_setopt($ch, CURLOPT_TIMEOUT, 10); curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 10); curl_setopt($ch, CURLOPT_CERTINFO, true); curl_setopt($ch, CURLOPT_VERBOSE, true); curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false); curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false); curl_setopt($ch, CURLOPT_POSTFIELDS, http_build_query($data)); $logFile=fopen(LOG."Curl.log", "a"); curl_setopt($ch, CURLOPT_STDERR, $logFile); curl_setopt($ch, CURLOPT_COOKIEFILE, $config['cookie_file']); curl_setopt($ch, CURLOPT_COOKIEJAR, $config['cookie_file']); curl_setopt($ch, CURLOPT_USERAGENT, $config['useragent']); curl_setopt($ch, CURLOPT_REFERER, 'http://search.mpaq.org'); $response=curl_exec($ch); if (curl_errno($ch)){ $send=__LINE__." sendto: ".$sendto."\nresponse: ".$response."\ncurl_error: ".curl_error($ch); log_files($send,LOG.'API_Response.log'); }else{ $send=__LINE__." sendto: ".$sendto." ".$response; log_files($send,LOG.'API_Response.log'); return trim($response); } } function encryptText($text,$key){ if (!isset($text)){return NULL;} if (!isset($key)){return NULL;} $iv=openssl_random_pseudo_bytes(openssl_cipher_iv_length('aes-256-cbc')); $encrypted=openssl_encrypt($text, 'aes-256-cbc', $key, 0, $iv); return base64_encode($iv.$encrypted); } function pad_2($ThisNumber){if (isset($ThisNumber)){return str_pad($ThisNumber,2,"0",STR_PAD_LEFT);}else{return "00";}} function pad_3($ThisNumber){if (isset($ThisNumber)){return str_pad($ThisNumber,3,"0",STR_PAD_LEFT);}else{return "000";}} function pad_5($ThisNumber){if (isset($ThisNumber)){return str_pad($ThisNumber,5,"0",STR_PAD_LEFT);}else{return "00000";}} function pad_8($ThisNumber){if (isset($ThisNumber)){return str_pad($ThisNumber,8,"0",STR_PAD_LEFT);}else{return "00000000";}} function pad_r($ThisNumber){ if (isset($ThisNumber)){ $ThisNumber=str_pad($ThisNumber,4,"0",STR_PAD_LEFT); $ThisNumber=str_pad($ThisNumber,4,"0",STR_PAD_RIGHT); //echo $ThisNumber."\n";exit; return $ThisNumber; }else{ return "0000.0000"; } } function pad_space($This){if (strlen($This)>20){$result=substr($This,0,20);}else{$result=str_pad($This,20," ");}return $result;} function KWords($title,$desc,$site){ $INDEX_List=array( "'", ".", "\"", "(", ")", "~", ":", "-", "_", "’", "!", "/", "'", "‘", "”", "“", "—", "?", "[…]", "»", "«", "", "", "", "", "", "", ";", ); $ww=NULL; if (is_null($site) or $site==""){return NULL;} $site=html_entity_decode($site,ENT_QUOTES|ENT_HTML5,'UTF-8'); $site=str_replace($INDEX_List,' ',$site); $KWords=strip_tags($title)." ".strip_tags($desc); $KWords=str_replace($INDEX_List,' ',$KWords); $KWords=preg_replace('/[0-9]+/',' ',$KWords); $KWords=str_ireplace('\t',' ',$KWords); $KWords=str_ireplace(',',' ',$KWords); $KWords=str_ireplace('.',' ',$KWords); $KWords=strtolower($KWords); $KWords=sort_words($KWords); $KWords=explode(" ",$KWords); for ($x=0;$x<=50;$x++){$KWords=str_ireplace('\n',' ',$KWords);} for ($x=0;$x<=50;$x++){$KWords=str_ireplace('\r',' ',$KWords);} for ($x=0;$x<=50;$x++){$KWords=str_ireplace(' ',' ',$KWords);} for ($x=0;$x<=100;$x++){$KWords=str_ireplace(' ',' ',$KWords);} foreach($KWords as $ThisWord){ if (strlen($ThisWord)>3){ if (is_null($ww)){ $ww=$ThisWord." "; }else{ if (!stristr($ww,$ThisWord)){$ww=$ww.$ThisWord." ";} } } } if (isset($ww)){ for ($x=0;$x<=100;$x++){$ww=str_ireplace(' ',' ',$ww);} $data['KWords']=$site." ".trim($ww); } if (isset($data['KWords'])){ for ($x=0;$x<=100;$x++){ $data['KWords']=str_ireplace(' ',' ',$data['KWords']); } return $data['KWords']; }else{return NULL;} } function send_group($BName,$Group,$Send,$Server){ if (isset($BName) and isset($Group) and isset($Send) and isset($Server)){ $ERROR=NULL; $ERROR=shell_exec("echo '".$Send."' | sendxmpp --username netbot --password 4cUTyfWwN5wc7St --jserver ".$Server." --resource ".$BName." --message-type=headline --chatroom ".$Group.""); if (!is_null($ERROR)){echo $ERROR."\n";exit;}else{return NULL;} // echo $ERROR."\n";exit; } return NULL; } ///////////////////////////////////////////////////////// /* class GhostAPI{ private $token; private $instance_url; public function __construct($token, $instance_url){ $this->token = $token; $this->instance_url = $instance_url; } public function postStatus($status){ return $this->callAPI('/api/v1/post', 'POST', $status); } public function callAPI($endpoint, $method, $data){ $headers = [ 'Authorization: Bearer '.$this->token, 'Content-Type: multipart/form-data', ]; $ch = curl_init(); curl_setopt($ch, CURLOPT_URL, $this->instance_url.$endpoint); curl_setopt($ch, CURLOPT_CUSTOMREQUEST, $method); curl_setopt($ch, CURLOPT_POSTFIELDS, $data); curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); curl_setopt($ch, CURLOPT_HTTPHEADER, $headers); curl_setopt($ch, CURLOPT_USERAGENT,"MPAQ"); $reply = curl_exec($ch); if (!$reply) { return json_encode(['ok'=>false, 'curl_error_code' => curl_errno($ch), 'curl_error' => curl_error($ch)]); } return json_decode($reply, true); } } /* class MastodonAPI{ // $token=MTOKEN; // $instance_url=M_URL; private $token; private $instance_url; public function __construct($token, $instance_url){ $this->token = $token; $this->instance_url = $instance_url; } public function postStatus($status){return $this->callAPI('/api/v1/statuses', 'POST', $status);} public function uploadMedia($media){return $this->callAPI('/api/v1/media', 'POST', $media);} public function callAPI($endpoint, $method, $data){ $headers = [ 'Authorization: Bearer '.$this->token, 'Content-Type: multipart/form-data', ]; $ch = curl_init(); curl_setopt($ch, CURLOPT_URL, $this->instance_url.$endpoint); curl_setopt($ch, CURLOPT_CUSTOMREQUEST, $method); curl_setopt($ch, CURLOPT_POSTFIELDS, $data); curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); curl_setopt($ch, CURLOPT_HTTPHEADER, $headers); curl_setopt($ch, CURLOPT_USERAGENT, "curl"); $reply = curl_exec($ch); if (!$reply) { log_files(curl_errno($ch)."\n\t".$instance_url."@".$instance_url,LOG."MastodonAPI.log"); return json_encode(['ok'=>false, 'curl_error_code' => curl_errno($ch), 'curl_error' => curl_error($ch)]); } log_files(curl_errno($ch)."\n\t".$instance_url."@".$instance_url,LOG."MastodonAPI.log"); return json_decode($reply, true); } } /* UPDATE tidysearch.pages SET _lang=null WHERE sites._lang=""; UPDATE tidysearch.sites_rss SET url='https://ojrock.mpaq.org/feed' WHERE sites_rss.index=437; DELETE FROM tidysearch.pages WHERE pages.title is null; DELETE FROM tidysearch.pages WHERE pages.title LIKE ""; DELETE FROM tidysearch.pages WHERE pages.date = NULL; delete from tidysearch.pages where kwords is null; SELECT * FROM tidysearch.pages where pages.description like ''; SELECT * FROM tidysearch.pages ORDER BY pages.date DESC LIMIT 10; https://news.mpaq.org/sitemap-posts.xml https://news.mpaq.org/.ghost/activitypub/nodeinfo/2.1 OpenFire chatter HWrBVELMAh8YYdk https://bots.mpaq.org/.well-known/webfinger?resource=acct:@bob@mpaq.org */ ?>