481 lines
21 KiB
PHP
481 lines
21 KiB
PHP
|
<?PHP
|
|||
|
|
ini_set('display_errors',1);ini_set('display_startup_errors',1);error_reporting(E_ALL);
|
|||
|
|
date_default_timezone_set("America/Chicago");
|
|||
|
|
$StartTime=microtime(true);$homeless=$new_post=$Count=$count=0;
|
|||
|
|
$home_dir=dirname(__FILE__);
|
|||
|
|
include_once('/config/folder/.BotSolo.php');
|
|||
|
|
$lockFile="/tmp/RSSSolo.lock";
|
|||
|
|
if (file_exists($lockFile)){
|
|||
|
|
$locked=$caller." has been locked at ".date('Y:m:d H:i:s');
|
|||
|
|
if (DEBUG==0){send_group($caller,"tidybot@lobby.mpaq.org",$locked,"mpaq.org");}
|
|||
|
|
exit;
|
|||
|
|
}
|
|||
|
|
if (DEBUG==0){file_put_contents($lockFile,getmypid());}else{if (file_exists($lockFile)){unlink($lockFile);}}
|
|||
|
|
$ERROR=NULL;$caller="BotRSS".$BOT['MyID'];
|
|||
|
|
$sync=BaseAPI."synczero";
|
|||
|
|
$options=['ssl' => ['verify_peer' => false,'verify_peer_name' => false]];
|
|||
|
|
$context=stream_context_create($options);
|
|||
|
|
$json=file_get_contents($sync,false,$context);
|
|||
|
|
$Site=json_decode($json);
|
|||
|
|
$websites=$next=0;
|
|||
|
|
while(isset($Site[$next])){$next++;}$last=$next=$next-1;$next=0;
|
|||
|
|
for ($site=$next;$next<=$last;$next++){
|
|||
|
|
$websites++;$data['G2G']=0;
|
|||
|
|
$data['RSSindex']=$Site[$next]->index;
|
|||
|
|
$data['SiteID']=$Site[$next]->sitesid;
|
|||
|
|
$data['RemoteID']=$Site[$next]->remoteid;
|
|||
|
|
$data['SiteName']=$Site[$next]->sitename;
|
|||
|
|
$data['datetime']=$Site[$next]->datetime;
|
|||
|
|
$data['hits']=$Site[$next]->hits;
|
|||
|
|
if ($Site[$next]->botid==0){$data['botid']=$BOT['MyID'];}else{$data['botid']=$Site[$next]->botid;}
|
|||
|
|
$data['feed']=$Site[$next]->url;
|
|||
|
|
$data['SiteNotes']=$Site[$next]->site_notes;
|
|||
|
|
$HTMLCode=test_channel($data['feed']);
|
|||
|
|
if ($HTMLCode>399){
|
|||
|
|
$data['DataType']=2;
|
|||
|
|
$data['SITES']="UPDATE tidysearch.sites SET botid='".$HTMLCode."' WHERE sites.index=".$data['SiteID'].";";
|
|||
|
|
$data['G2G']=send_2_server($BOT,$data);
|
|||
|
|
$data['DataType']=3;
|
|||
|
|
$data['RSS']="UPDATE tidysearch.sites_rss SET botid='".$HTMLCode."' WHERE sites_rss.index=".$data['RSSindex'].";";
|
|||
|
|
$data['G2G']=send_2_server($BOT,$data);
|
|||
|
|
echo "Disabled Feed SiteID: ".$data['SiteID']." RSS: ".$data['RSSindex']."\n";
|
|||
|
|
}
|
|||
|
|
$rss=new DOMDocument();
|
|||
|
|
@$rss->load($data['feed']);
|
|||
|
|
$feed = array();
|
|||
|
|
if (!$rss->getElementsByTagName('item')){var_dump($data);exit;}
|
|||
|
|
foreach ($rss->getElementsByTagName('item') as $node){
|
|||
|
|
$data['G2G']=$newLink=0;
|
|||
|
|
if (isset($node->getElementsByTagName('link')->item(0)->nodeValue)){
|
|||
|
|
$data['link']=$node->getElementsByTagName('link')->item(0)->nodeValue;
|
|||
|
|
$data['Source_Code']=mb_trim($data['link'],"/");
|
|||
|
|
if (strripos($data['Source_Code'],'?')){
|
|||
|
|
$data['Source_Code']=substr($data['Source_Code'],0,strpos($data['Source_Code'],'?'));
|
|||
|
|
$data['Source_Code']=str_ireplace("?",'',$data['Source_Code']);
|
|||
|
|
}
|
|||
|
|
$data['Source_Code']=str_ireplace("'","%27",$data['Source_Code']);
|
|||
|
|
$data['Source_Code']=trim($data['Source_Code']);
|
|||
|
|
$data['Source_Code']=mb_trim($data['Source_Code'],"/");
|
|||
|
|
$sc=BaseAPI."source_code?sc=".$data['Source_Code'];
|
|||
|
|
$sc=file_get_contents($sc,false,$context);
|
|||
|
|
$sc=json_decode($sc);
|
|||
|
|
if (!isset($sc[0])){
|
|||
|
|
if (isset($node->getElementsByTagName('title')->item(0)->nodeValue)){
|
|||
|
|
$data['Title']=clean_string($node->getElementsByTagName('title')->item(0)->nodeValue);
|
|||
|
|
}else{$data['Title']=NULL;}
|
|||
|
|
if (isset($node->getElementsByTagName('description')->item(0)->nodeValue)){
|
|||
|
|
$RSSdata['desc']=clean_string($node->getElementsByTagName('description')->item(0)->nodeValue);
|
|||
|
|
if (isset($RSSdata['desc']) and $RSSdata['desc']!==""){
|
|||
|
|
$data['Description']=strip_tags($RSSdata['desc'],"<A>");
|
|||
|
|
$data['Description']=str_ireplace("\n","",$data['Description']);
|
|||
|
|
$data['Description']=str_ireplace("[…]","...",$data['Description']);
|
|||
|
|
$data['Description']=substr($data['Description'],0,1000);
|
|||
|
|
if (str_contains($data['Description'],'utm_source')){
|
|||
|
|
$data['Description']=substr($data['Description'],0,strpos($data['Description'],'?'));
|
|||
|
|
$data['Description']=str_ireplace("?",'',$data['Description']);
|
|||
|
|
/* Look in $data['Description'] for
|
|||
|
|
?utm_source=social&utm_medium=feed&utm_campaign=profeed
|
|||
|
|
rel="nofollow"
|
|||
|
|
*/
|
|||
|
|
}
|
|||
|
|
for ($x=0;$x<=50;$x++){$data['Description']=str_ireplace(' ',' ',$data['Description']);}
|
|||
|
|
$data['Description']=trim($data['Description']);
|
|||
|
|
$data['Description']=mb_trim($data['Description']);
|
|||
|
|
}else{$data['Description']=NULL;}
|
|||
|
|
}else{$data['Description']=NULL;}
|
|||
|
|
if (isset($node->getElementsByTagName('pubDate')->item(0)->nodeValue)){
|
|||
|
|
$data['date']=$node->getElementsByTagName('pubDate')->item(0)->nodeValue;
|
|||
|
|
}elseif (isset($node->getElementsByTagName('dc:date')->item(0)->nodeValue)){
|
|||
|
|
$data['date']=$node->getElementsByTagName('dc:date')->item(0)->nodeValue;
|
|||
|
|
}
|
|||
|
|
if (isset($data['date']) and !is_null($data['date'])){
|
|||
|
|
$tmp=date('Y-m-d H:m:i',strtotime($data['date']));
|
|||
|
|
if (verifyDateTime($tmp)){
|
|||
|
|
$data['IndexDate']=date('Y-m-d H:i:s',strtotime($tmp));
|
|||
|
|
}elseif (verifyDate($tmp)){
|
|||
|
|
$data['IndexDate']=date('Y-m-d H:i:s',strtotime($tmp));
|
|||
|
|
}
|
|||
|
|
}else{$data['IndexDate']=NULL;}
|
|||
|
|
if (is_null($data['link'])){
|
|||
|
|
log_files("Missing Link SiteID: ".$data['SiteID']."\t@ ".__LINE__."\n",LOG."Missing_Data.log");
|
|||
|
|
$flag++;$newLink=0;
|
|||
|
|
}
|
|||
|
|
if (!is_null($data['Title']) and !is_null($data['Description']) and !is_null($data['Source_Code'])){
|
|||
|
|
$data['KWords']=KWords($data['Title'],$data['Description'],$data['SiteName']);
|
|||
|
|
$data['Description']=strip_tags($data['Description']);
|
|||
|
|
}
|
|||
|
|
if (isset($data['IndexDate']) and !is_null($data['IndexDate'])){
|
|||
|
|
if (!isset($new_date)){
|
|||
|
|
$new_date=date('Y-m-d H:m:i',strtotime($data['IndexDate']));
|
|||
|
|
}else{
|
|||
|
|
if (!is_null($new_date)){
|
|||
|
|
if (strtotime($new_date)<strtotime($data['IndexDate'])){
|
|||
|
|
$new_date=date('Y-m-d H:m:i',strtotime($data['IndexDate']));
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}else{
|
|||
|
|
$data['IndexDate']=NULL;
|
|||
|
|
}
|
|||
|
|
if (DEBUG==1){
|
|||
|
|
if (is_null($data['datetime']) or $data['datetime']=="" or !isset($data['datetime'])){$data['datetime']="Never";}
|
|||
|
|
log_files("SiteID:\t\t".pad_8($data['SiteID']),LOG.pad_8($data['SiteID']).'-TidySearch.log');
|
|||
|
|
log_files("RemoteID:\t".$data['RemoteID'],LOG.pad_8($data['SiteID']).'-TidySearch.log');
|
|||
|
|
log_files("BotID:\t\t".pad_3($BOT['MyID']),LOG.pad_8($data['SiteID']).'-TidySearch.log');
|
|||
|
|
log_files("Site_Name:\t".$data['SiteName'],LOG.pad_8($data['SiteID']).'-TidySearch.log');
|
|||
|
|
if (isset($new_date)){log_files("Last Update:\t".$new_date,LOG.pad_8($data['SiteID']).'-TidySearch.log');}
|
|||
|
|
log_files("URL:\t\t".$data['feed'],LOG.pad_8($data['SiteID']).'-TidySearch.log');
|
|||
|
|
log_files("Source_Code:\t".$data['Source_Code'],LOG.pad_8($data['SiteID']).'-TidySearch.log');
|
|||
|
|
log_files("IndexDate:\t".$data['IndexDate'],LOG.pad_8($data['SiteID']).'-TidySearch.log');
|
|||
|
|
if (!is_null($data['Title']) and !is_null($data['Description']) and !is_null($data['KWords'])){
|
|||
|
|
log_files("Title:\t\t".$data['Title'],LOG.pad_8($data['SiteID']).'-TidySearch.log');
|
|||
|
|
log_files("Description:\t".$data['Description'],LOG.pad_8($data['SiteID']).'-TidySearch.log');
|
|||
|
|
log_files("KWords:\t\t".$data['KWords'],LOG.pad_8($data['SiteID']).'-TidySearch.log');
|
|||
|
|
}
|
|||
|
|
log_files("SiteNotes:\t".$data['SiteNotes'],LOG.pad_8($data['SiteID']).'-TidySearch.log');
|
|||
|
|
}
|
|||
|
|
if (!is_null($data['Title']) and !is_null($data['Description']) and !is_null($data['Source_Code']) and !is_null($data['IndexDate'])){
|
|||
|
|
$tmp1=htmlentities($data['Title'],ENT_QUOTES|ENT_HTML5,'UTF-8');
|
|||
|
|
$tmp1="'".str_ireplace('\'',''',$tmp1)."'";
|
|||
|
|
$data['Description']=strip_tags($data['Description']);
|
|||
|
|
$tmp2=htmlentities($data['Description'],ENT_QUOTES|ENT_HTML5,'UTF-8');
|
|||
|
|
$tmp2="'".str_ireplace('\'',''',$tmp2)."'";
|
|||
|
|
$data['DataType']=1;
|
|||
|
|
$data['PAGES']="INSERT INTO tidysearch.pages (site_id,date,title,description,kwords,source_code,botid,hostid,lan_code,rank)VALUES(".pad_5($data['SiteID']).",'".$data['IndexDate']."',".$tmp1.",".$tmp2.",'".$data['KWords']."','".$data['Source_Code']."',".$BOT['MyID'].",3,NULL,0);";
|
|||
|
|
if (DEBUG==1){log_files("PAGES:\t\t".$data['PAGES'],LOG.pad_8($data['SiteID']).'-TidySearch.log');}
|
|||
|
|
if (!is_null($data['Title']) and !is_null($data['Description']) and !is_null($data['KWords']) and DEBUG==0){
|
|||
|
|
$data['G2G']=send_2_server($BOT,$data);
|
|||
|
|
if ($data['G2G']>10){
|
|||
|
|
$G2G=$data['G2G'];$Count++;
|
|||
|
|
log_files($data['PAGES']." -- Page Idex: ".$data['G2G'],SQL.pad_8($data['SiteID']).'-TidySearch.sql');
|
|||
|
|
}
|
|||
|
|
}else{
|
|||
|
|
$data['G2G']=0;
|
|||
|
|
}
|
|||
|
|
if (!is_null($data['KWords'])){
|
|||
|
|
if (str_contains($data['KWords'],'homeless') or str_contains($data['KWords'],'houseless') or str_contains($data['KWords'],'homelessness')){
|
|||
|
|
$homeless++;
|
|||
|
|
if (DEBUG==0){Fediverse ($data);}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
if (!is_null($data['Title'])){
|
|||
|
|
$display=pad_space(html_entity_decode($data['Title'],ENT_QUOTES|ENT_HTML5,'UTF-8'));
|
|||
|
|
}else{
|
|||
|
|
$display=pad_space(" ");
|
|||
|
|
}
|
|||
|
|
echo pad_5($data['SiteID'])." ".pad_5($data['RSSindex'])." ".pad_3($BOT['MyID'])." ".pad_3($Count)." ".$data['IndexDate']." - ".$display."\t".memory_get_usage()."\t".time_check($StartTime)."\tG2G: ".$data['G2G']."\n";
|
|||
|
|
if (DEBUG==1){log_files("\t\t******",LOG.pad_8($data['SiteID']).'-TidySearch.log');}
|
|||
|
|
}else{
|
|||
|
|
$display=pad_space("No New Data");
|
|||
|
|
echo pad_5($data['SiteID'])." ".pad_5($data['RSSindex'])." ".pad_3($BOT['MyID'])." ".pad_3($Count)." - ".$display."\t".memory_get_usage()."\t".time_check($StartTime)."\tG2G: ".$data['G2G']."\n";
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
if ($Count>0){
|
|||
|
|
$hits=$data['hits']+$Count;
|
|||
|
|
$data['PAGES']=$data['SITES']=$data['RSS']=NULL;
|
|||
|
|
$data['DataType']=2;
|
|||
|
|
$data['SITES']="UPDATE tidysearch.sites SET datetime='".$new_date."' WHERE sites.index=".$data['SiteID'].";";
|
|||
|
|
if (DEBUG==0){$data['G2G']=send_2_server($BOT,$data);}
|
|||
|
|
$data['DataType']=3;
|
|||
|
|
$data['RSS']="UPDATE tidysearch.sites_rss SET datetime='".$new_date."',hits=".$hits." WHERE sites_rss.index=".$data['RSSindex'].";";
|
|||
|
|
if (DEBUG==0){$data['G2G']=send_2_server($BOT,$data);}
|
|||
|
|
log_files($data['SITES']."\n".$data['RSS'],SQL.pad_8($data['SiteID']).'-TidySearch.sql');
|
|||
|
|
$count=$count+$Count;
|
|||
|
|
}else{
|
|||
|
|
$data['DataType']=3;
|
|||
|
|
$date=date('Y-m-d H:i:s');
|
|||
|
|
$data['RSS']="UPDATE tidysearch.sites_rss SET datetime='".$date."' WHERE sites_rss.index=".$data['RSSindex'].";";
|
|||
|
|
if (DEBUG==0){$data['G2G']=send_2_server($BOT,$data);}
|
|||
|
|
}
|
|||
|
|
$Count=0;
|
|||
|
|
$RSSindex=$data['RSSindex'];
|
|||
|
|
unset($data);
|
|||
|
|
echo "\n";
|
|||
|
|
}
|
|||
|
|
$display="BotID: ".pad_3($BOT['MyID'])."\nSites: ".pad_3($websites)." Feeds: ".pad_3($count)."\nHomeless: ".pad_3($homeless)."\n".time_check($StartTime);
|
|||
|
|
if (DEBUG==0 and $count>0){send_group("NewsBot".$BOT['MyID'],"tidybot@lobby.mpaq.org",$display,"mpaq.org");}
|
|||
|
|
if (isset($G2G) and $G2G>0){
|
|||
|
|
echo "\nData Sent: ".$display."\t\tDone\007\n";sleep(1);echo "\007";
|
|||
|
|
}else{
|
|||
|
|
echo "\nNo Data Sent: ".$display."\t\tDone\007\n";sleep(1);echo "\007";
|
|||
|
|
}
|
|||
|
|
if (file_exists($lockFile)){unlink($lockFile);}
|
|||
|
|
//////////////////////////// The End ////////////////////////////
|
|||
|
|
//////////////////////////// The End ////////////////////////////
|
|||
|
|
//////////////////////////// The End ////////////////////////////
|
|||
|
|
function Fediverse ($data){
|
|||
|
|
include_once('/media/10T/ADMIN/.BotSolo.php');
|
|||
|
|
$display=strip_tags($data['Description']);
|
|||
|
|
$display=substr($display,0,1000);
|
|||
|
|
$report=$data['Title']."\n\n".$display."\n";
|
|||
|
|
unset($display);
|
|||
|
|
$report=str_ireplace('… Continue reading →','...',$report);
|
|||
|
|
$report=str_ireplace('’','\'',$report);
|
|||
|
|
$report=str_replace('homelessness','#homelessness',$report);
|
|||
|
|
$report=str_replace('houseless','#houseless',$report);
|
|||
|
|
$report=str_replace('homeless','#homeless',$report);
|
|||
|
|
$report=str_replace('Homelessness','#Homelessness',$report);
|
|||
|
|
$report=str_replace('Houseless','#Houseless',$report);
|
|||
|
|
$report=str_replace('Homeless','#Homeless',$report);
|
|||
|
|
$report=str_ireplace('##','#',$report);
|
|||
|
|
$report=str_ireplace(' ','',$report);
|
|||
|
|
$report=$report.date('M j,Y',strtotime($data['IndexDate']))." #Homeless #Poverty\n".$data['Source_Code']."\n\n#TidySearch Bot".$data['botid']." (".$data['SiteID'].") For more posts, follow @newsbot @ https://beamship.mpaq.org and Visit http://search.mpaq.org for more.";
|
|||
|
|
for ($x=0;$x<=50;$x++){$report=str_ireplace(' ',' ',$report);}
|
|||
|
|
/*
|
|||
|
|
if (!is_null($report)){
|
|||
|
|
$headers = [
|
|||
|
|
'Authorization: Bearer '.TOKEN,
|
|||
|
|
'Content-Type: multipart/form-data',
|
|||
|
|
];
|
|||
|
|
$ch = curl_init();
|
|||
|
|
curl_setopt($ch, CURLOPT_URL, Mast_URL."/api/v1/statuses");
|
|||
|
|
curl_setopt($ch, CURLOPT_CUSTOMREQUEST, "Post");
|
|||
|
|
curl_setopt($ch, CURLOPT_POSTFIELDS, $report);
|
|||
|
|
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
|
|||
|
|
curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
|
|||
|
|
curl_setopt($ch, CURLOPT_USERAGENT,"MPAQ");
|
|||
|
|
var_dump($ch);
|
|||
|
|
$reply = curl_exec($ch);
|
|||
|
|
echo $report."\n\n".$reply."\n";exit;
|
|||
|
|
if (!$reply) {return json_encode(['ok'=>false, 'curl_error_code' => curl_errno($ch), 'curl_error' => curl_error($ch)]);}
|
|||
|
|
return json_decode($reply, true);
|
|||
|
|
*/
|
|||
|
|
log_files($report,LOG.pad_3($data['SiteID'])."-Homeless.log");
|
|||
|
|
send_group("NewsBot".$data['botid'],"news@lobby.mpaq.org",$report,"mpaq.org");
|
|||
|
|
log_files("\t*****",LOG.pad_3($data['SiteID'])."-Homeless.log");
|
|||
|
|
// }
|
|||
|
|
}
|
|||
|
|
function time_check($StartTime){
|
|||
|
|
$tmp=microtime(true)-$StartTime;
|
|||
|
|
if ($tmp>60){
|
|||
|
|
$tmp=$tmp/60;$return=pad_2(round($tmp,2))."M";
|
|||
|
|
}else{
|
|||
|
|
$return=pad_2(round($tmp,2))."S";
|
|||
|
|
}
|
|||
|
|
return $return;
|
|||
|
|
}
|
|||
|
|
function log_files($notes,$name){
|
|||
|
|
$entry=$notes."\n";
|
|||
|
|
if (!file_exists($name)){touch ($name);}
|
|||
|
|
$data_file=$name;
|
|||
|
|
$handle=fopen($name, 'r');
|
|||
|
|
$new_data_file=fopen($data_file, 'a+') or die('Cannot open file: '.$data_file);
|
|||
|
|
fwrite($new_data_file, $entry);
|
|||
|
|
fclose($new_data_file);
|
|||
|
|
fclose($handle);
|
|||
|
|
}
|
|||
|
|
function sort_words($input){
|
|||
|
|
$words=explode(" ", $input);
|
|||
|
|
sort($words);
|
|||
|
|
$sorted_string=implode(" ", $words);
|
|||
|
|
return $sorted_string;
|
|||
|
|
}
|
|||
|
|
function verifyDateTime($date, $format = 'Y-m-d H:i:s'){
|
|||
|
|
if (strtotime($date)<strtotime(date('Y-m-d H:i:s'))){
|
|||
|
|
$dd = DateTime::createFromFormat($format, $date);
|
|||
|
|
return $dd && $dd->format($format) === $date;
|
|||
|
|
}else{
|
|||
|
|
return null;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
function verifyDate($date, $format = 'Y-m-d'){
|
|||
|
|
if (strtotime($date)<strtotime(date('Y-m-d'))){
|
|||
|
|
$dd = DateTime::createFromFormat($format, $date);
|
|||
|
|
return $dd && $dd->format($format) === $date;
|
|||
|
|
}else{
|
|||
|
|
return null;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
function test_channel($url){
|
|||
|
|
ini_set('display_errors',1);ini_set('display_startup_errors',1);error_reporting(E_ALL);
|
|||
|
|
include('/media/10T/ADMIN/.tidysearch.php');
|
|||
|
|
$curl_test = array(
|
|||
|
|
CURLOPT_HEADER => true, // we want headers
|
|||
|
|
CURLOPT_NOBODY => true, // we don't need body
|
|||
|
|
CURLOPT_RETURNTRANSFER => 1,
|
|||
|
|
CURLOPT_TIMEOUT => 2, // timeout on response
|
|||
|
|
CURLOPT_USERAGENT => "TidySearch Engine: visit http://search.mpaq.org for details.", // who am i
|
|||
|
|
CURLOPT_CONNECTTIMEOUT => 2, // timeout on connect
|
|||
|
|
CURLOPT_MAXREDIRS => 5, // stop after 10 redirects
|
|||
|
|
);
|
|||
|
|
$ch=curl_init($url);
|
|||
|
|
curl_setopt_array($ch,$curl_test);
|
|||
|
|
$output=curl_exec($ch);
|
|||
|
|
$httpcode=curl_getinfo($ch, CURLINFO_HTTP_CODE);
|
|||
|
|
return $httpcode;
|
|||
|
|
}
|
|||
|
|
function send_2_server($BOT,$data){
|
|||
|
|
include_once('/media/10T/ADMIN/.BotSolo.php');
|
|||
|
|
if (!isset($data['PAGES'])){$data['PAGES']=NULL;}
|
|||
|
|
if (!isset($data['SITES'])){$data['SITES']=NULL;}
|
|||
|
|
if (!isset($data['RSS'])){$data['RSS']=NULL;}
|
|||
|
|
if (!isset($data['Source_Code'])){$data['Source_Code']=NULL;}
|
|||
|
|
if (!isset($data['DataType'])){return NULL;}
|
|||
|
|
$data2send=array(
|
|||
|
|
'BotPubKey' => $BOT['PubKey'],
|
|||
|
|
'BotID' => encryptText($BOT['MyID'],$BOT['PVTKey']),
|
|||
|
|
'SiteID' => encryptText($data['SiteID'],$BOT['PVTKey']),
|
|||
|
|
'RSSindex' => encryptText($data['RSSindex'],$BOT['PVTKey']),
|
|||
|
|
'SiteName' => encryptText($data['SiteName'],$BOT['PVTKey']),
|
|||
|
|
'DataType' => encryptText($data['DataType'],$BOT['PVTKey']),
|
|||
|
|
'DataPage' => encryptText($data['PAGES'],$BOT['PVTKey']),
|
|||
|
|
'DataSites' => encryptText($data['SITES'],$BOT['PVTKey']),
|
|||
|
|
'DataRSS' => encryptText($data['RSS'],$BOT['PVTKey']),
|
|||
|
|
'Source_Code' => encryptText($data['Source_Code'],$BOT['PVTKey']),
|
|||
|
|
);
|
|||
|
|
// $sendto=BaseAPI."collect_ts";
|
|||
|
|
$sendto="https://212.102.59.80:250/collect_ts.php";
|
|||
|
|
$responce=mb_trim(server($sendto,$data2send,$BOT['MyID'],));
|
|||
|
|
if (is_numeric($responce) and $responce>0){
|
|||
|
|
$send="API Server ".$sendto." responded with: ".$responce;
|
|||
|
|
return $responce;
|
|||
|
|
}else{
|
|||
|
|
if (isset($data['SiteAddress'])){
|
|||
|
|
$send="Line ".__LINE__." API Responce Error!!! Aborting...\n".$data['LocalID']." ".mb_trim($data['SiteAddress'])." from ".$sendto."\n\n".$responce;
|
|||
|
|
echo $send."\n";
|
|||
|
|
var_dump($data);
|
|||
|
|
echo "\007";sleep(1);echo "\007";sleep(1);echo "\007";sleep(1);
|
|||
|
|
exit;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
function server($sendto,$data,$botid){
|
|||
|
|
include_once('/media/10T/ADMIN/.BotSolo.php');
|
|||
|
|
$headers=array(
|
|||
|
|
"Content-Type: application/json",
|
|||
|
|
"Authorization: Bearer your_token",
|
|||
|
|
);
|
|||
|
|
$config['useragent']='TidyBot (TidySearch http://search.mpaq.org)';
|
|||
|
|
$dir=dirname(__FILE__);
|
|||
|
|
$ch=curl_init($sendto);
|
|||
|
|
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
|
|||
|
|
curl_setopt($ch, CURLOPT_POST, true);
|
|||
|
|
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
|
|||
|
|
curl_setopt($ch, CURLOPT_TIMEOUT, 10);
|
|||
|
|
curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 10);
|
|||
|
|
curl_setopt($ch, CURLOPT_CERTINFO, true);
|
|||
|
|
curl_setopt($ch, CURLOPT_VERBOSE, true);
|
|||
|
|
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
|
|||
|
|
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false);
|
|||
|
|
curl_setopt($ch, CURLOPT_POSTFIELDS, http_build_query($data));
|
|||
|
|
$logFile=fopen(LOG."Curl.log", "a");
|
|||
|
|
curl_setopt($ch, CURLOPT_STDERR, $logFile);
|
|||
|
|
$config['cookie_file']=LOG.md5($sendto).'.txt';
|
|||
|
|
curl_setopt($ch, CURLOPT_COOKIEFILE, $config['cookie_file']);
|
|||
|
|
curl_setopt($ch, CURLOPT_COOKIEJAR, $config['cookie_file']);
|
|||
|
|
curl_setopt($ch, CURLOPT_USERAGENT, $config['useragent']);
|
|||
|
|
curl_setopt($ch, CURLOPT_REFERER, 'http://search.mpaq.org');
|
|||
|
|
$response=curl_exec($ch);
|
|||
|
|
if (curl_errno($ch)){
|
|||
|
|
$send=__LINE__." sendto: ".$sendto."\nresponse: ".$response."\ncurl_error: ".curl_error($ch);
|
|||
|
|
// log_files($send,LOG.'API_Response.log');
|
|||
|
|
exit;
|
|||
|
|
}else{
|
|||
|
|
$send=__LINE__." sendto: ".$sendto."\nresponse: ".$response."\ncurl_error: ".curl_error($ch);
|
|||
|
|
// log_files($send,LOG.'API_Response.log');
|
|||
|
|
return trim($response);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
function encryptText($text,$key){
|
|||
|
|
if (!isset($text)){return NULL;}
|
|||
|
|
if (!isset($key)){return NULL;}
|
|||
|
|
$iv=openssl_random_pseudo_bytes(openssl_cipher_iv_length('aes-256-cbc'));
|
|||
|
|
$encrypted=openssl_encrypt($text, 'aes-256-cbc', $key, 0, $iv);
|
|||
|
|
return base64_encode($iv.$encrypted);
|
|||
|
|
}
|
|||
|
|
function pad_2($ThisNumber){if (isset($ThisNumber)){return str_pad($ThisNumber,2,"0",STR_PAD_LEFT);}else{return "00";}}
|
|||
|
|
function pad_3($ThisNumber){if (isset($ThisNumber)){return str_pad($ThisNumber,3,"0",STR_PAD_LEFT);}else{return "000";}}
|
|||
|
|
function pad_5($ThisNumber){if (isset($ThisNumber)){return str_pad($ThisNumber,5,"0",STR_PAD_LEFT);}else{return "00000";}}
|
|||
|
|
function pad_8($ThisNumber){if (isset($ThisNumber)){return str_pad($ThisNumber,8,"0",STR_PAD_LEFT);}else{return "00000000";}}
|
|||
|
|
function pad_space($This){if (strlen($This)>20){$result=substr($This,0,20);}else{$result=str_pad($This,20," ");}return $result;}
|
|||
|
|
function KWords($title,$desc,$site){
|
|||
|
|
$INDEX_List=array(
|
|||
|
|
"'",
|
|||
|
|
".",
|
|||
|
|
"\"",
|
|||
|
|
"(",
|
|||
|
|
")",
|
|||
|
|
"~",
|
|||
|
|
":",
|
|||
|
|
"-",
|
|||
|
|
"_",
|
|||
|
|
"’",
|
|||
|
|
"!",
|
|||
|
|
"/",
|
|||
|
|
"'",
|
|||
|
|
"‘",
|
|||
|
|
"”",
|
|||
|
|
"“",
|
|||
|
|
"—",
|
|||
|
|
"?",
|
|||
|
|
"[…]",
|
|||
|
|
"»",
|
|||
|
|
"«",
|
|||
|
|
"",
|
|||
|
|
"",
|
|||
|
|
"",
|
|||
|
|
"",
|
|||
|
|
"",
|
|||
|
|
"",
|
|||
|
|
";",
|
|||
|
|
);
|
|||
|
|
$ww=NULL;
|
|||
|
|
if (is_null($site) or $site==""){return NULL;}
|
|||
|
|
$site=html_entity_decode($site,ENT_QUOTES|ENT_HTML5,'UTF-8');
|
|||
|
|
$site=str_replace($INDEX_List,' ',$site);
|
|||
|
|
$KWords=strip_tags($title)." ".strip_tags($desc);
|
|||
|
|
$KWords=str_replace($INDEX_List,' ',$KWords);
|
|||
|
|
$KWords=preg_replace('/[0-9]+/',' ',$KWords);
|
|||
|
|
$KWords=str_ireplace('\t',' ',$KWords);
|
|||
|
|
$KWords=str_ireplace(',',' ',$KWords);
|
|||
|
|
$KWords=str_ireplace('.',' ',$KWords);
|
|||
|
|
$KWords=strtolower($KWords);
|
|||
|
|
$KWords=sort_words($KWords);
|
|||
|
|
$KWords=explode(" ",$KWords);
|
|||
|
|
for ($x=0;$x<=50;$x++){$KWords=str_ireplace('\n',' ',$KWords);}
|
|||
|
|
for ($x=0;$x<=50;$x++){$KWords=str_ireplace('\r',' ',$KWords);}
|
|||
|
|
for ($x=0;$x<=50;$x++){$KWords=str_ireplace('
|
|||
|
|
',' ',$KWords);}
|
|||
|
|
for ($x=0;$x<=100;$x++){$KWords=str_ireplace(' ',' ',$KWords);}
|
|||
|
|
foreach($KWords as $ThisWord){
|
|||
|
|
if (strlen($ThisWord)>3){
|
|||
|
|
if (is_null($ww)){
|
|||
|
|
$ww=$ThisWord." ";
|
|||
|
|
}else{
|
|||
|
|
if (!stristr($ww,$ThisWord)){$ww=$ww.$ThisWord." ";}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
if (isset($ww)){
|
|||
|
|
for ($x=0;$x<=100;$x++){$ww=str_ireplace(' ',' ',$ww);}
|
|||
|
|
$data['KWords']=$site." ".trim($ww);
|
|||
|
|
}
|
|||
|
|
if (isset($data['KWords'])){
|
|||
|
|
for ($x=0;$x<=100;$x++){
|
|||
|
|
$data['KWords']=str_ireplace(' ',' ',$data['KWords']);
|
|||
|
|
}
|
|||
|
|
return $data['KWords'];
|
|||
|
|
}else{return NULL;}
|
|||
|
|
}
|
|||
|
|
function send_group($BName,$Group,$Send,$Server){
|
|||
|
|
if (isset($_SERVER['SERVER_NAME'])){if ($_SERVER['SERVER_NAME']=="localhost"){exit;}}
|
|||
|
|
$ERROR=NULL;
|
|||
|
|
$ERROR=shell_exec("echo '".$Send."' | sendxmpp --username netbot --password 4cUTyfWwN5wc7St --jserver ".$Server." --resource ".$BName." --message-type=headline --chatroom ".$Group."");
|
|||
|
|
return $ERROR;
|
|||
|
|
}
|
|||
|
|
/*
|
|||
|
|
|
|||
|
|
UPDATE tidysearch.sites SET url='',name='' WHERE sites.index=1538;
|
|||
|
|
|
|||
|
|
UPDATE tidysearch.sites_rss SET botid='404' WHERE sites_rss.index=587;
|
|||
|
|
UPDATE tidysearch.sites SET botid=0,hits=0,name='',url='' WHERE sites.index=xxx;
|
|||
|
|
UPDATE tidysearch.sites_rss SET botid=0,hits=0,site_name='',url='' WHERE sites_rss.index=xxx;
|
|||
|
|
UPDATE tidysearch.sites SET botid=0 WHERE sites.botid=9000;
|
|||
|
|
UPDATE tidysearch.sites_rss SET botid=0 WHERE sites_rss.botid=9000;
|
|||
|
|
UPDATE tidysearch.sites SET botid=0 WHERE sites.botid=9001;
|
|||
|
|
UPDATE tidysearch.sites_rss SET botid=0 WHERE sites_rss.botid=9001;
|
|||
|
|
*/
|
|||
|
|
?>
|