Irina Peshinskaya
11 years ago
parent
cdba3fd0ac
commit
f6a23d0602
@ -0,0 +1,286 @@
|
|||||||
|
#! /usr/bin/perl
|
||||||
|
|
||||||
|
use strict;
|
||||||
|
|
||||||
|
use Data::Dumper;
|
||||||
|
use WWW::Mechanize::Firefox;
|
||||||
|
use DBI;
|
||||||
|
use DateTime;
|
||||||
|
use DateTime::Format::ISO8601;
|
||||||
|
#use Storable qw/dclone/;
|
||||||
|
use Clone 'clone';
|
||||||
|
|
||||||
|
my $datetime = DateTime->from_epoch(
|
||||||
|
time_zone => DateTime::TimeZone->new(name => 'local'),
|
||||||
|
epoch => time(),
|
||||||
|
);
|
||||||
|
my $mech = WWW::Mechanize::Firefox->new();
|
||||||
|
my $dbh = DBI->connect('dbi:mysql:spider;host=localhost', 'root', '');
|
||||||
|
my $cfg = {
|
||||||
|
url_duplicate_template =>{
|
||||||
|
rule => 'deny',
|
||||||
|
rule_allow => '',
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
#$dbh->do('drop table urltemplate');
|
||||||
|
#$dbh->do('drop table url');
|
||||||
|
#$dbh->do('drop table urlcontent');
|
||||||
|
#$dbh->do('drop table urlvariant');
|
||||||
|
#$dbh->do('drop table control');
|
||||||
|
#$dbh->do('drop table url_control');
|
||||||
|
#$dbh->do('drop table url_template');
|
||||||
|
#$dbh->do('create table urltemplate(id integer(11) auto_increment primary key, template varchar(512))');
|
||||||
|
#$dbh->do('create table url(id integer(11) unsigned not null auto_increment primary key, url varchar(512), urltemplate_id integer(11), lastvisit timestamp default 0)');
|
||||||
|
#$dbh->do('create table urlvariant(id integer(11) unsigned not null, url varchar(512), container_url_id integer(11))');
|
||||||
|
#$dbh->do('create table urlcontent(id integer(11) unsigned not null, content text)');
|
||||||
|
#$dbh->do('create table control(id integer(11) unsigned not null auto_increment primary key, label text, goto_url_id integer(11) unsigned)');
|
||||||
|
#$dbh->do('create table url_control(url_id integer(11) unsigned, control_id integer(11) unsigned)');
|
||||||
|
#$dbh->do('create table url_template(url_id integer(11) unsigned, urltemplate_id integer(11) unsigned)');
|
||||||
|
|
||||||
|
# Xvfb :99 &
|
||||||
|
# DISPLAY=:99 firefox --display=:99 &
|
||||||
|
## DISPLAY=:99 xdotool search --onlyvisible --title firefox
|
||||||
|
## DISPLAY=:99 xdotool windowfocus 4194416
|
||||||
|
# DISPLAY=:99 ~/fillform.pl
|
||||||
|
|
||||||
|
|
||||||
|
our $host = 'https://192.168.56.7:1444';
|
||||||
|
|
||||||
|
|
||||||
|
get_data($mech,$host.'/dashboard');
|
||||||
|
#while (my $url = $dbh->selectrow_array('select url from url where unix_timestamp(lastvisit) < ? limit 1', undef, $datetime->epoch)){
|
||||||
|
# get_data($mech,$url);
|
||||||
|
#}
|
||||||
|
|
||||||
|
sub get_data{
|
||||||
|
my ($url_in,$mech,$container_url,$loaded) = {};
|
||||||
|
($mech,@{$url_in}{qw/url/},$container_url,$loaded) = @_;
|
||||||
|
print $url_in->{url}.";\n";
|
||||||
|
if(!check_url_toadd($url_in->{url})){
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
my($url) = get_url_db($url_in);
|
||||||
|
my($urltemplate) = get_urltemplate_db({
|
||||||
|
template => get_urltemplate($url->{url}),
|
||||||
|
});
|
||||||
|
|
||||||
|
my $url_toadd = 0;
|
||||||
|
if( $url && (!$url->{epoch}) || ( $url->{epoch} < $datetime->epoch) ){
|
||||||
|
$url_toadd = 1;
|
||||||
|
if($container_url){
|
||||||
|
$url_toadd = rule_url_template($container_url,$urltemplate);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if($container_url){#after checking
|
||||||
|
register_url_template($container_url,$urltemplate);
|
||||||
|
}
|
||||||
|
|
||||||
|
if($url_toadd){
|
||||||
|
if(!$loaded){
|
||||||
|
print pre_get_url($url->{url}).";\n";
|
||||||
|
$mech->get(pre_get_url($url->{url}));
|
||||||
|
}else{
|
||||||
|
#here add to db urls from clicks, if necessary
|
||||||
|
}
|
||||||
|
#$mech->eval("alert('QQ');");
|
||||||
|
#follow_link starts here. Specially for follow_link we don't need recursion, but for click we do.
|
||||||
|
$url->{content} = \$mech->content();
|
||||||
|
set_url_visited($url);
|
||||||
|
#foreach my $clickable_in ( $mech->clickables() ) {
|
||||||
|
# my $control = get_control_db({
|
||||||
|
## goto_url_id => $goto_url->{id},
|
||||||
|
# label => process_control_label($clickable_in->{innerHTML})
|
||||||
|
# }
|
||||||
|
# );
|
||||||
|
# register_control($url,$control);
|
||||||
|
# #print Dumper $control;
|
||||||
|
# #$mech->click($control);
|
||||||
|
#}
|
||||||
|
#foreach my $link ( $mech->links() ) {
|
||||||
|
#foreach my $link ( $mech->find_all_links_dom() ) {
|
||||||
|
#my $repl = $mech->repl();
|
||||||
|
my $contentDiv = $mech->xpath('//div[@id="content"]', single => 1);
|
||||||
|
my @links = $mech->find_all_links_dom( node => $contentDiv );
|
||||||
|
#my @links = $mech->find_all_links_dom( );
|
||||||
|
print $#links;
|
||||||
|
die();
|
||||||
|
foreach my $link ( @links ) {
|
||||||
|
if(!check_url_toadd($link->{href})){
|
||||||
|
next;
|
||||||
|
}
|
||||||
|
|
||||||
|
my $control_url = { url => process_url($link->{href}) };
|
||||||
|
my $goto_url = get_url_db( $control_url );
|
||||||
|
if(! ($goto_url)){
|
||||||
|
next;
|
||||||
|
}
|
||||||
|
my $control=get_control_db({
|
||||||
|
goto_url_id => $goto_url->{id},
|
||||||
|
label => process_control_label($link->{innerHTML} || $link->{href}) ,
|
||||||
|
#label => process_control_label($link->text)
|
||||||
|
});
|
||||||
|
register_control($url,$control);
|
||||||
|
|
||||||
|
#print Dumper $link;
|
||||||
|
print "================================\n";
|
||||||
|
print Dumper { map { $_ => $url->{$_}; } qw/id url urltemplate_id/};
|
||||||
|
print Dumper $control_url;
|
||||||
|
print Dumper $control;
|
||||||
|
print Dumper $link->{tagName};
|
||||||
|
if(( 'A' eq $link->{tagName}) && check_url_toadd($goto_url->{url})){
|
||||||
|
#get_data($goto_url->{url},$url,1);
|
||||||
|
#get_data($goto_url->{url},$url,0);
|
||||||
|
#my $mechRecursion = clone($mech);
|
||||||
|
$SIG{ALRM} = \&request_timed_out;
|
||||||
|
eval {
|
||||||
|
alarm (10);
|
||||||
|
#$mechRecursion->follow_link($link);
|
||||||
|
print "follow_link\n";
|
||||||
|
$mech->follow_link($link);
|
||||||
|
alarm(0); # Cancel the pending alarm if user responds.
|
||||||
|
};
|
||||||
|
if('link_timed_out' eq $@){
|
||||||
|
print "link hanged\n";
|
||||||
|
}else{
|
||||||
|
#get_data($mechRecursion,$goto_url->{url},$url,1);
|
||||||
|
print "get_data\n";
|
||||||
|
get_data($mech,$goto_url->{url},$url,0);
|
||||||
|
eval {
|
||||||
|
alarm (10);
|
||||||
|
$mech->back();
|
||||||
|
alarm(0); # Cancel the pending alarm if user responds.
|
||||||
|
};
|
||||||
|
}
|
||||||
|
#if ($@ =~ /LOCAL_LINK/) {
|
||||||
|
# print "Timed out. Proceeding with default\n";
|
||||||
|
#}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sub request_timed_out{
|
||||||
|
die("link_timed_out");
|
||||||
|
}
|
||||||
|
sub get_url_db{
|
||||||
|
my($url) = @_;
|
||||||
|
my $url_db;
|
||||||
|
if(!($url_db = $dbh->selectrow_hashref('select *,unix_timestamp(lastvisit) as epoch from url where url=?',undef,$url->{url}))){
|
||||||
|
$dbh->do('insert into url(url) values(?)',undef,$url->{url});
|
||||||
|
$url_db = $url;
|
||||||
|
$url_db->{id} = $dbh->last_insert_id(undef,'spider','url','id');
|
||||||
|
}
|
||||||
|
return $url_db;
|
||||||
|
}
|
||||||
|
sub set_url_visited{
|
||||||
|
my($url_db) = @_;
|
||||||
|
print Dumper [ "update url set lastvisit=from_unixtime(?) where id=?\n", $datetime->epoch, $url_db->{id} ];
|
||||||
|
$dbh->do('update url set lastvisit=from_unixtime(?) where id=?',undef,$datetime->epoch,$url_db->{id});
|
||||||
|
#die();
|
||||||
|
$dbh->do('update url_content set content=? where id=?',undef,${$url_db->{content}},$url_db->{id});
|
||||||
|
$dbh->do('delete from url_control where url_id=?',undef,$url_db->{id});
|
||||||
|
}
|
||||||
|
sub get_control_db{
|
||||||
|
my($control) = @_;
|
||||||
|
my $control_db;
|
||||||
|
if(!($control_db = $dbh->selectrow_hashref('select * from control where label=? and goto_url_id=?',undef,@$control{qw/label goto_url_id/}))){
|
||||||
|
$dbh->do('insert into control(label,goto_url_id) values(?,?)',undef,@$control{qw/label goto_url_id/});
|
||||||
|
$control_db = $control;
|
||||||
|
$control_db->{id} = $dbh->last_insert_id(undef,'spider','control','id');
|
||||||
|
#if(!(my $res=$dbh->selectrow_array('select id from url_variant where url=?'))){
|
||||||
|
#
|
||||||
|
#}
|
||||||
|
}
|
||||||
|
return $control_db;
|
||||||
|
}
|
||||||
|
sub register_control{
|
||||||
|
my($url,$control) = @_;
|
||||||
|
if(!(my $res = $dbh->selectrow_hashref('select * from url_control where url_id=? and control_id=?',undef,$url->{id},$control->{id}))){
|
||||||
|
$dbh->do('insert into url_control(url_id,control_id)values(?,?)',undef,$url->{id},$control->{id});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sub set_control_db{
|
||||||
|
my($control) = @_;
|
||||||
|
$dbh->do('update control set label=?,goto_url_id=? where id=?',undef,@$control{qw/label goto_url_id id/});
|
||||||
|
}
|
||||||
|
sub get_urltemplate{
|
||||||
|
my($url) = @_;
|
||||||
|
$url = ~s/\d+/\d+/;
|
||||||
|
return $url;
|
||||||
|
}
|
||||||
|
sub get_urltemplate_db{
|
||||||
|
my($template) = @_;
|
||||||
|
my $template_db;
|
||||||
|
if(!($template_db = $dbh->selectrow_hashref('select * from urltemplate where template=?',undef,$template->{template}))){
|
||||||
|
$dbh->do('insert into urltemplate(template) values(?)',undef,$template->{template});
|
||||||
|
$template_db = $template;
|
||||||
|
$template_db->{id} = $dbh->last_insert_id(undef,'spider','urltemplate','id');
|
||||||
|
}
|
||||||
|
return $template_db;
|
||||||
|
}
|
||||||
|
sub check_url_toadd{
|
||||||
|
my ($url) = @_;
|
||||||
|
my $res = 1;
|
||||||
|
$url=~s/\s+//g;
|
||||||
|
if(!$url){
|
||||||
|
$res = 0;
|
||||||
|
}
|
||||||
|
if($url =~/javascript:;?$/i){
|
||||||
|
$res = 0;
|
||||||
|
}
|
||||||
|
if($url =~/#$/i){
|
||||||
|
$res = 0;
|
||||||
|
}
|
||||||
|
if($url =~/lang=[a-z]{2}/i){
|
||||||
|
$res = 0;
|
||||||
|
}
|
||||||
|
return $res;
|
||||||
|
}
|
||||||
|
sub process_url{
|
||||||
|
my ($url) = @_;
|
||||||
|
$url=~s/\Q$host\E//;
|
||||||
|
$url=~s/[&]?back=[^&]+//;
|
||||||
|
$url=~s/\?$//;
|
||||||
|
$url=~s/^\/$//;
|
||||||
|
return $url;
|
||||||
|
}
|
||||||
|
sub pre_get_url{
|
||||||
|
my ($url) = @_;
|
||||||
|
$url=~s/\Q$host\E//;
|
||||||
|
if($url !~/^https?:/){
|
||||||
|
$url=$host.'/'.$url;
|
||||||
|
}
|
||||||
|
$url =~ s!/+!/!g;
|
||||||
|
$url =~ s!(https?:(?:\d+)?)/+!$1//!g;
|
||||||
|
return $url;
|
||||||
|
}
|
||||||
|
sub process_control_label{
|
||||||
|
my ($str) = @_;
|
||||||
|
$str=~s/^\s*|\s*$//gsm;
|
||||||
|
return $str;
|
||||||
|
}
|
||||||
|
##--------------------------------------
|
||||||
|
sub rule_url_template{
|
||||||
|
my($container_url,$urltemplate) = @_;
|
||||||
|
my $result = 1;
|
||||||
|
#here something - if config deny add duplicate template - don't add and parse duplicate url for the same template on the samepage
|
||||||
|
#ifconfig specify exceptions for deny - consider it
|
||||||
|
#if config specify allow duplicate - add duplicate, and consider defined deny config
|
||||||
|
if($cfg->{url_duplicate_template}->{rule} eq 'deny'){
|
||||||
|
if(get_registered_url_template($container_url,$urltemplate)){
|
||||||
|
$result = 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return $result;
|
||||||
|
}
|
||||||
|
sub register_url_template{
|
||||||
|
my($container_url,$urltemplate) = @_;
|
||||||
|
$dbh->do('insert into url_template(url_id,urltemplate_id)values(?,?)',undef,$container_url->{id},$urltemplate->{id});
|
||||||
|
}
|
||||||
|
sub get_registered_url_template{
|
||||||
|
my($container_url,$urltemplate) = @_;
|
||||||
|
return $dbh->selectrow_array('select urltemplate_id from url_template where url_id=? and urltemplate_id=?',undef,$container_url->{id},$urltemplate->{id});
|
||||||
|
}
|
||||||
|
1;
|
||||||
|
__END__
|
||||||
Loading…
Reference in new issue