Skip to content
Snippets Groups Projects
Commit ade0a55a authored by Robert Martin-Legene's avatar Robert Martin-Legene
Browse files

Working code

parent 3aabd8b8
Branches master
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
collector/collector.pl
+ 128
− 99
View file @ ade0a55a
...@@ -9,7 +9,7 @@ use Data::Dumper; ...@@ -9,7 +9,7 @@ use Data::Dumper;
use Math::BigInt; use Math::BigInt;
use LWP; use LWP;
use JSON; use JSON;
use Net::DNS; use Net::DNS;
use Net::DNS::Resolver; use Net::DNS::Resolver;
use lib '/home/bfa/collector'; use lib '/home/bfa/collector';
use lib 'collector'; use lib 'collector';
...@@ -17,10 +17,11 @@ use lib '.'; ...@@ -17,10 +17,11 @@ use lib '.';
use sql; use sql;
# 47525974938; # 47525974938;
my $netversion = Math::BigInt->new( "0xb10c4d39a" ); my $netversion = Math::BigInt->new( "0xb10c4d39a" );
# 200941592 # 200941592
my $chainid = Math::BigInt->new( "0xbfa2018" ); my $chainid = Math::BigInt->new( "0xbfa2018" );
my $idcounter = 0; my $idcounter = 0;
my $stop_request = 0;
my $ua; my $ua;
my @blockqueue; my @blockqueue;
my %sealers; my %sealers;
...@@ -32,90 +33,99 @@ sub info(@) ...@@ -32,90 +33,99 @@ sub info(@)
# if 1 param only # if 1 param only
unshift @_, '%s' unshift @_, '%s'
if $#_ == 0; if $#_ == 0;
my $format = shift; my $format = shift;
$format = "%s: " . $format; $format = "%s: " . $format;
printf $format, astime(time), @_; my $txt = sprintf $format, astime(time), @_;
$txt .= "\n" if $txt !~ /\n$/;
print $txt;
} }
sub astime sub astime
{ {
my @t = localtime( $_[0] ); my @t = localtime( $_[0] );
$t[5] += 1900; $t[5] += 1900;
$t[4] += 1; $t[4] += 1;
return sprintf '%04d%02d%02d-%02d%02d%02d', @t[5,4,3,2,1,0]; return sprintf '%04d%02d%02d-%02d%02d%02d', @t[5,4,3,2,1,0];
} }
sub shorthash($) sub shorthash($)
{ {
local $_ = $_[0]; local $_ = $_[0];
return unless defined $_;
s/^(0x.......).*(.......)$/$1..$2/; s/^(0x.......).*(.......)$/$1..$2/;
return $_; return $_;
} }
sub stop
{
$stop_request = 1;
print STDERR "\rStop request received ($_[0]).\n";
}
sub rpcreq sub rpcreq
{ {
my ( $opname, @params )= @_; my ( $opname, @params )= @_;
$ua = LWP::UserAgent->new( keep_alive => 10 ) $ua = LWP::UserAgent->new( keep_alive => 10 )
if not defined $ua; if not defined $ua;
# $ua->ssl_opts( 'verify_hostname' => 0 ); # $ua->ssl_opts( 'verify_hostname' => 0 );
my %args = ( my %args = (
jsonrpc => '2.0', jsonrpc => '2.0',
method => $opname, method => $opname,
id => $idcounter++, id => $idcounter++,
); );
my @args = map { my @args = map {
my $v = $args{$_}; my $v = $args{$_};
$v = qq/"${v}"/ if $v !~ /^\d+$/; $v = qq/"${v}"/ if $v !~ /^\d+$/;
sprintf qq/"%s":%s/, $_, $v; sprintf qq/"%s":%s/, $_, $v;
} keys %args; } keys %args;
if ( scalar @params ) if ( scalar @params )
{ {
my $p = '"params":['; my $p = '"params":[';
foreach my $param ( @params ) foreach my $param ( @params )
{ {
$param = '"' . $param . '"' $param = '"' . $param . '"'
if $param ne 'true' if $param ne 'true'
and $param ne 'false' and $param ne 'false'
and $param !~ /^\d+$/; and $param !~ /^\d+$/;
$p .= $param . ','; $p .= $param . ',';
} }
chop $p; chop $p;
$p .= ']'; $p .= ']';
push @args, $p; push @args, $p;
} }
my $args = '{' . join(',',@args) . '}'; my $args = '{' . join(',',@args) . '}';
die 'The environment variable $BFANODE must contain a URL for a ' . die 'The environment variable $BFANODE must contain a URL for a ' .
'server to connect to (e.g. https://10.20.30.40/), but it it ' . 'server to connect to (e.g. https://10.20.30.40/), but it it ' .
'not set. Stopped' 'not set. Stopped'
unless defined $ENV{'BFANODE'}; unless defined $ENV{'BFANODE'};
my $res = $ua->post( my $res = $ua->post(
$ENV{BFANODE}, $ENV{BFANODE},
'Content-Type' => 'application/json', 'Content-Type' => 'application/json',
'Content' => $args, 'Content' => $args,
); );
die $res->status_line die $res->status_line
unless $res->is_success; unless $res->is_success;
my $json = decode_json($res->content); my $json = decode_json($res->content);
return $json->{'result'}; return $json->{'result'};
} }
sub getblock sub getblock
{ {
my $block = lc shift; my $block = lc shift;
my $rpccmd; my $rpccmd;
if ( $block =~ /^0x[\da-f]{64}$/ ) if ( $block =~ /^0x[\da-f]{64}$/ )
{ {
$rpccmd = 'eth_getBlockByHash'; $rpccmd = 'eth_getBlockByHash';
} }
else else
{ {
$rpccmd = 'eth_getBlockByNumber'; $rpccmd = 'eth_getBlockByNumber';
if ( $block =~ /^\d+$/ ) if ( $block =~ /^\d+$/ )
{ {
$block = Math::BigInt->new( $block )->as_hex; $block = Math::BigInt->new( $block )->as_hex;
} }
} }
my $json = rpcreq( $rpccmd, $block, 'false' ); my $json = rpcreq( $rpccmd, $block, 'false' );
die "We should have received some kind of JSON from our RPC call, " . die "We should have received some kind of JSON from our RPC call, " .
"but apparently not. Stopped " "but apparently not. Stopped "
if not defined $json; if not defined $json;
...@@ -123,10 +133,10 @@ sub getblock ...@@ -123,10 +133,10 @@ sub getblock
foreach my $key (qw( foreach my $key (qw(
timestamp gasUsed gasLimit difficulty number totalDifficulty size timestamp gasUsed gasLimit difficulty number totalDifficulty size
)) { )) {
$json->{$key} = Math::BigInt->new( $json->{$key} )->bstr $json->{$key} = Math::BigInt->new( $json->{$key} )->bstr
if exists $json->{$key}; if exists $json->{$key};
} }
info "B %s %s %s %s %s %s/%s\n", info "B %s %s %s %s %s %s/%s",
shorthash $json->{'hash'}, shorthash $json->{'hash'},
astime($json->{'timestamp'}), astime($json->{'timestamp'}),
$json->{'number'}, $json->{'number'},
...@@ -151,25 +161,26 @@ sub getblock ...@@ -151,25 +161,26 @@ sub getblock
getTransByHash( $txhash ); getTransByHash( $txhash );
} }
} }
# Put the parent block in front in @blockqueue
unshift @blockqueue, lc $json->{'parentHash'} unshift @blockqueue, lc $json->{'parentHash'}
if $json->{'number'} != 0; if $json->{'number'} != 0;
} }
sub getTransByHash sub getTransByHash
{ {
my $hash = shift; my $hash = shift;
# Don't get it via RPC if we already have it in the database # Don't get it via RPC if we already have it in the database
return return
if $sql->selectTransactionByHash( $hash ); if $sql->selectTransactionByHash( $hash );
my $json = rpcreq( my $json = rpcreq(
'eth_getTransactionByHash', 'eth_getTransactionByHash',
$hash $hash
); );
die "Invalid object received when asking for $hash:\n".Dumper($json) if not exists $json->{'blockNumber'}; die "Invalid object received when asking for $hash:\n".Dumper($json) if not exists $json->{'blockNumber'};
my $max = 0; my $max = 0;
my $rcpt = undef; my $rcpt = undef;
do { do {
$rcpt = rpcreq( $rcpt = rpcreq(
'eth_getTransactionReceipt', 'eth_getTransactionReceipt',
$hash $hash
); );
...@@ -179,40 +190,44 @@ sub getTransByHash ...@@ -179,40 +190,44 @@ sub getTransByHash
warn "Couldn't get the transaction receipt for $hash\n"; warn "Couldn't get the transaction receipt for $hash\n";
} }
} while not $rcpt and $max++ < 10; } while not $rcpt and $max++ < 10;
die "Couldn't get the transaction receipt for $hash\n" if not $rcpt; die "Couldn't get the transaction receipt for $hash:\n" if not $rcpt;
$json->{'status'} = $rcpt->{'status'}; foreach my $key (qw( hash blockHash nonce gas gasPrice value from to input ))
$json->{'gasUsed'} = $rcpt->{'gasUsed'}; {
die "JSON missing key $key:".Dumper($json) unless exists $json->{$key};
}
$json->{'status'} = $rcpt->{'status'};
$json->{'gasUsed'} = $rcpt->{'gasUsed'};
$json->{'contractAddress'} $json->{'contractAddress'}
= $rcpt->{'contractAddress'}; = $rcpt->{'contractAddress'};
foreach my $key (qw( nonce value gas gasPrice gasUsed status)) foreach my $key (qw( nonce value gas gasPrice gasUsed status))
{ {
$json->{$key} = Math::BigInt->new( $json->{$key} )->bstr; $json->{$key} = Math::BigInt->new( $json->{$key} )->bstr;
} }
info "T %s %s %s %s -> %s\n", info "T %s %s %s %s -> %s",
shorthash $json->{'hash'}, shorthash $json->{'hash'},
$json->{'nonce'}, $json->{'nonce'},
$json->{'value'}, $json->{'value'},
shorthash $json->{'from'}, shorthash $json->{'from'},
shorthash $json->{'to'}; shorthash ($json->{'to'}||'<NULL>');
my $input = $json->{'input'}; my $input = $json->{'input'};
my $inputlen = 0; my $inputlen = 0;
if ( $input =~ /^0x/ ) if ( $input =~ /^0x/ )
{ {
$inputlen = (length($input)-2) / 2; $inputlen = (length($input)-2) / 2;
} }
my @args = ( my @args = (
lc $json->{'hash'}, lc $json->{'hash'},
lc $json->{'blockHash'}, lc $json->{'blockHash'},
$json->{'nonce'}, $json->{'nonce'},
$json->{'gas'}, $json->{'gas'},
$json->{'gasPrice'}, $json->{'gasPrice'},
$json->{'value'}, $json->{'value'},
lc $json->{'from'}, lc $json->{'from'},
lc $json->{'to'}, defined $json->{'to'} ? lc $json->{'to'} : undef,
$json->{'status'}, $json->{'status'},
$json->{'gasUsed'}, $json->{'gasUsed'},
$input, $input,
$inputlen, $inputlen
); );
if ( $json->{'contractAddress'} ) if ( $json->{'contractAddress'} )
{ {
...@@ -227,8 +242,8 @@ sub getTransByHash ...@@ -227,8 +242,8 @@ sub getTransByHash
sub versioncheck sub versioncheck
{ {
my $n; my $n;
my $ok = 1; my $ok = 1;
$n = Math::BigInt->new( rpcreq( "net_version" ) ); $n = Math::BigInt->new( rpcreq( "net_version" ) );
if ( $n->bcmp( $netversion ) != 0 ) if ( $n->bcmp( $netversion ) != 0 )
{ {
warn "Network says it has net.version " warn "Network says it has net.version "
...@@ -238,13 +253,13 @@ sub versioncheck ...@@ -238,13 +253,13 @@ sub versioncheck
. "). Expected $netversion (" . "). Expected $netversion ("
. $netversion->as_hex . $netversion->as_hex
. ").\n"; . ").\n";
$ok = 0; $ok = 0;
} }
$n = Math::BigInt->new( rpcreq( "eth_chainId" ) ); $n = Math::BigInt->new( rpcreq( "eth_chainId" ) );
if ( $n->bcmp( $chainid ) != 0 ) if ( $n->bcmp( $chainid ) != 0 )
{ {
warn sprintf("Network says it has eth.chainId %s (%s). Expected %s (%s).\n", $n->bstr, $n->as_hex, $chainid->bstr, $chainid->as_hex); warn sprintf("Network says it has eth.chainId %s (%s). Expected %s (%s).\n", $n->bstr, $n->as_hex, $chainid->bstr, $chainid->as_hex);
$ok = 0; $ok = 0;
} }
exit 1 if not $ok; exit 1 if not $ok;
return $ok; return $ok;
...@@ -252,80 +267,94 @@ sub versioncheck ...@@ -252,80 +267,94 @@ sub versioncheck
sub getsnap sub getsnap
{ {
my $blockhash = $_[0]; my $blockhash = $_[0];
my $json = rpcreq( my $json = rpcreq(
'clique_getSnapshotAtHash', 'clique_getSnapshotAtHash',
$blockhash $blockhash
); );
# Sanity check 1 - does the lookup return the block? # Sanity check 1 - does the lookup return the block?
return if not defined $json; return if not defined $json;
my $recents = $json->{'recents'}; my $recents = $json->{'recents'};
# Blockhash of the requested block - we must use this for the insert # Blockhash of the requested block - we must use this for the insert
$blockhash = $json->{'hash'}; $blockhash = $json->{'hash'};
my $blocknumber = Math::BigInt->new( $json->{'number'} ); my $blocknumber = Math::BigInt->new( $json->{'number'} );
my @infonumbers; my @infonumbers;
while ( not $blocknumber->is_zero ) while ( not $blocknumber->is_zero )
{ {
my $sealerhash = $recents->{$blocknumber->bstr}; my $sealerhash = $recents->{$blocknumber->bstr};
last if not defined $sealerhash; last if not defined $sealerhash;
push @infonumbers, $blocknumber->bstr; push @infonumbers, $blocknumber->bstr;
my $ref = $sql->selectBlockByHash( $blockhash ); my $ref = $sql->selectBlockByHash( $blockhash );
# Stop if we get to blocks which we still haven't stored in the db # Stop if we get to blocks which we still haven't stored in the db
return ( undef, $blocknumber ) if ( not defined $ref )
if not defined $ref; {
$blockhash = undef;
last;
}
# Stop if we got to a block with sealers. # Stop if we got to a block with sealers.
return ( undef, $blocknumber ) if ( exists $ref->{'sealerAccountId'} and defined $ref->{'sealerAccountId'} )
if exists $ref->{'sealerAccountId'} and defined $ref->{'sealerAccountId'}; {
$blockhash = undef;
last;
}
$sql->updateWhoSealed( $sealerhash, $blockhash ); $sql->updateWhoSealed( $sealerhash, $blockhash );
my $parentblockid = $ref->{'parentBlockhashId'}; my $parentblockid = $ref->{'parentBlockhashId'};
# For next block... # For next block...
$blockhash = $sql->selectBlockhashById( $parentblockid ); $blockhash = $sql->selectBlockhashById( $parentblockid );
$blocknumber->bsub(1); $blocknumber->bsub(1);
} }
info "S Snapshot at block # @infonumbers.\n"; info "S Snapshot at block # @infonumbers."
if @infonumbers;
return ($blockhash, $blocknumber); return ($blockhash, $blocknumber);
} }
# Find strands of blocks in the database which have their sealer set to NULL # Find strands of blocks in the database which have their sealer set to NULL
sub allsnaps sub allsnaps
{ {
my $blockhash = $sql->selectMaxUnknownSigned; my $blockhash = $sql->selectMaxUnknownSigned;
my $committer = 0;
while ( defined $blockhash ) while ( defined $blockhash )
{ {
my $blocknumber; my $blocknumber;
( $blockhash, $blocknumber ) ( $blockhash, $blocknumber )= getsnap( $blockhash );
= getsnap( $blockhash );
$sql->commit
if ( $committer++ % 75 ) == 0;
last if $blocknumber->is_zero; last if $blocknumber->is_zero;
} }
} }
sub main sub main
{ {
$| = 1;
versioncheck(); versioncheck();
# Give Postgres time to start up. # Give Postgres time to start up.
sleep(3); sleep( 3 );
$sql = sql->new(); $sql = sql->new();
info "Looking for orphaned blocks in the " my $lastorphancheck= 0;
. "database.\n"; $SIG{'HUP'} = \&stop;
push @blockqueue, $sql->listOrphans(); $SIG{'INT'} = \&stop;
$SIG{'TERM'} = \&stop;
while ( 1 ) $SIG{'TSTP'} = \&stop;
while ( not $stop_request )
{ {
unshift @blockqueue, 'latest'; my $now = time();
while ( @blockqueue ) if ( $lastorphancheck <= $now+600 )
{ {
my $maxinarow = 100; $lastorphancheck = $now;
# Orphans happen if some blocks have been deleted
# or if the initial synchronization hasn't finished.
# This check should be done when orphans can be
# suspected, but when is that? Maybe every 10 minutes.
push @blockqueue, $sql->listOrphans();
}
do
{
unshift @blockqueue, 'latest';
my $maxinarow = 100;
getblock( shift @blockqueue ) getblock( shift @blockqueue )
while @blockqueue and --$maxinarow; while @blockqueue and --$maxinarow and not $stop_request;
# Find out who signed all blocks # Find out who signed all blocks
allsnaps(); allsnaps() if not $stop_request;
info "Committing.";
$sql->commit; $sql->commit;
} } while @blockqueue and not $stop_request;
sleep 5; sleep 5 if not @blockqueue and not $stop_request;
} }
} }
......
collector/sql.pm
+ 231
− 103
View file @ ade0a55a
...@@ -12,8 +12,8 @@ use base qw( Class::Accessor ); ...@@ -12,8 +12,8 @@ use base qw( Class::Accessor );
__PACKAGE__->mk_accessors( 'dbh' ); __PACKAGE__->mk_accessors( 'dbh' );
sub new sub new
{ {
my ( $class ) = @_; my ( $class ) = @_;
my $self = bless {}, ref $class || $class; my $self = bless {}, ref $class || $class;
$self->dbh( DBI->connect( $self->dbh( DBI->connect(
'dbi:Pg:dbname=postgres;host=postgres','postgres',$ENV{'POSTGRES_PASSWORD'}, 'dbi:Pg:dbname=postgres;host=postgres','postgres',$ENV{'POSTGRES_PASSWORD'},
{AutoCommit=>0,RaiseError=>1} {AutoCommit=>0,RaiseError=>1}
...@@ -23,37 +23,80 @@ sub new ...@@ -23,37 +23,80 @@ sub new
sub commit sub commit
{ {
my ( $self ) = @_; my ( $self ) = @_;
$self->dbh->commit; $self->dbh->commit;
} }
sub listOrphans sub listOrphans
{ {
my ( $self ) = @_; my ( $self ) = @_;
my $sth = $self->dbh->prepare(q( my $max = $self->maxBlockNumber;
SELECT "parentBlockhashId", id # max will be NULL when there are no blocks in the table.
FROM block return unless defined $max;
WHERE "parentBlockhashId" NOT IN ( my $sth_orphan = $self->dbh->prepare(q(
SELECT id SELECT "parentBlockhashId", id
FROM block FROM block
) WHERE number >= ?
AND number <= ?
AND "parentBlockhashId" NOT IN (
SELECT id FROM block
WHERE number >= ?-1
AND number <= ?-1
)
ORDER BY number;
)); ));
my @results; my @results;
$sth->execute(); my $stepping = 50000;
while ( my $row = $sth->fetchrow_arrayref ) my $i = 0;
# Had to split this into sections or it would take too long
# (1.4M rows, 3 CPUs, stopped after 8 hours of execution)
# This takes less than 10 seconds with 1.4M rows.
while ( $i < $max )
{ {
# Block 0 has it's parent listed as 0x0{64} my $bottom = $i;
next if $row->[0] =~ /^0x0{64}$/; my $top = $i + $stepping;
push @results, $self->selectBlockhashById( $row->[0] ); $sth_orphan->execute($bottom, $top, $bottom, $top);
$i += $stepping;
while ( my $row = $sth_orphan->fetchrow_arrayref )
{
next
if $row->[0]=~ /^0x0{64}$/;
push @results, $self->selectBlockhashById( $row->[0] );
}
} }
$sth->finish; $sth_orphan->finish;
# Take the lowest number and put it in the back
my $first = shift @results;
push @results, $first
if defined $first;
return @results; return @results;
} }
__PACKAGE__->mk_accessors( 'sth_max' );
sub maxBlockNumber
{
my ( $self ) = @_;
if ( not defined $self->sth_max )
{
$self->sth_max(
$self->dbh->prepare(q(
SELECT MAX(number)
FROM block
))
);
}
$self->sth_max->execute;
my $ref = $self->sth_max->fetchrow_arrayref;
$self->sth_max->finish;
die "Unable to get the highest block number. Stopped"
if not defined $ref;
return $ref->[0];
}
__PACKAGE__->mk_accessors( 'sth_insertTransactionWithContractAddress' ); __PACKAGE__->mk_accessors( 'sth_insertTransactionWithContractAddress' );
sub insertTransactionWithContractAddress sub insertTransactionWithContractAddress
{ {
my $self = shift; my $self = shift;
if ( not defined $self->sth_insertTransactionWithContractAddress ) if ( not defined $self->sth_insertTransactionWithContractAddress )
{ {
$self->sth_insertTransactionWithContractAddress( $self->sth_insertTransactionWithContractAddress(
...@@ -67,17 +110,17 @@ sub insertTransactionWithContractAddress ...@@ -67,17 +110,17 @@ sub insertTransactionWithContractAddress
)) ))
); );
} }
$_[1] = $self->selectBlockhashByHash( $_[1] ); $_[1] = $self->selectBlockhashByHash( $_[1] );
$_[6] = $self->selectAccountIdByAddress( $_[6] ); $_[6] = $self->selectAccountIdByAddress( $_[6] );
$_[7] = $self->selectAccountIdByAddress( $_[7] ); $_[7] = $self->selectAccountIdByAddress( $_[7] ) if defined $_[7];
$_[12] = $self->selectAccountIdByAddress( $_[12] ); $_[12] = $self->selectAccountIdByAddress( $_[12] ) if defined $_[12];
$self->sth_insertTransactionWithContractAddress->execute( @_, 'explicit' ); $self->sth_insertTransactionWithContractAddress->execute( @_, 'explicit' );
} }
__PACKAGE__->mk_accessors( 'sth_insertTransaction' ); __PACKAGE__->mk_accessors( 'sth_insertTransaction' );
sub insertTransaction sub insertTransaction
{ {
my $self = shift; my $self = shift;
if ( not defined $self->sth_insertTransaction ) if ( not defined $self->sth_insertTransaction )
{ {
$self->sth_insertTransaction( $self->sth_insertTransaction(
...@@ -91,16 +134,82 @@ sub insertTransaction ...@@ -91,16 +134,82 @@ sub insertTransaction
)) ))
); );
} }
$_[1] = $self->selectBlockhashByHash( $_[1] ); $_[1] = $self->selectBlockhashByHash( $_[1] );
$_[6] = $self->selectAccountIdByAddress( $_[6] ); $_[6] = $self->selectAccountIdByAddress( $_[6] );
$_[7] = $self->selectAccountIdByAddress( $_[7] ); $_[7] = $self->selectAccountIdByAddress( $_[7] ) if defined $_[7];
$self->sth_insertTransaction->execute( @_, 'explicit' ); $self->sth_insertTransaction->execute( @_, 'explicit' );
} }
my @cacheBlockhash;
sub _cacheBlockhash
{
my $args = shift;
my $i = scalar @cacheBlockhash;
if ( exists $args->{'id'} and exists $args->{'hash'} )
{
# Don't put something in the cache, if it's already there
while ( $i-- > 0 )
{
if ( $cacheBlockhash[$i]->{'id'} eq $args->{'id'} )
{
return;
}
}
# Store in the cache
push @cacheBlockhash, { 'id' => $args->{'id'}, 'hash' => $args->{'hash'} };
# may 10 entries
splice @cacheBlockhash, 10;
return;
}
# If just one element is given, then it is a lookup
if ( defined $args->{'id'} )
{
while ( $i-- > 0 )
{
if ( $cacheBlockhash[$i]->{'id'} eq $args->{'id'} )
{
return $cacheBlockhash[$i]->{'hash'}
}
}
return;
}
if ( defined $args->{'hash'} )
{
while ( $i-- > 0 )
{
if ( $cacheBlockhash[$i]->{'hash'} eq $args->{'hash'} )
{
return $cacheBlockhash[$i]->{'id'};
}
}
return;
}
return;
}
__PACKAGE__->mk_accessors( 'sth_insertBlockhash' );
sub insertBlockhash
{
my ( $self, $hash ) = @_;
$hash = lc $hash;
if ( not defined $self->sth_insertBlockhash )
{
$self->sth_insertBlockhash(
$self->dbh->prepare(q(
INSERT INTO blockhash
( hash )
VALUES (?)
))
)
}
$self->sth_insertBlockhash->execute( $hash );
return $self->dbh->last_insert_id(undef,undef,undef,undef,{sequence=>'blockhash_id_seq'});
}
__PACKAGE__->mk_accessors( 'sth_selectBlockhashById' ); __PACKAGE__->mk_accessors( 'sth_selectBlockhashById' );
sub selectBlockhashById sub selectBlockhashById
{ {
my ( $self, $id ) = @_; my ( $self, $id ) = @_;
if ( not defined $self->sth_selectBlockhashById ) if ( not defined $self->sth_selectBlockhashById )
{ {
$self->sth_selectBlockhashById( $self->sth_selectBlockhashById(
...@@ -111,19 +220,22 @@ sub selectBlockhashById ...@@ -111,19 +220,22 @@ sub selectBlockhashById
)) ))
); );
} }
my $hash = _cacheBlockhash( {'id'=>$id} );
return $hash if defined $hash;
# Ask the database
$self->sth_selectBlockhashById->execute( $id ); $self->sth_selectBlockhashById->execute( $id );
my $ref = $self->sth_selectBlockhashById->fetchrow_arrayref; my $ref = $self->sth_selectBlockhashById->fetchrow_arrayref;
$self->sth_selectBlockhashById->finish; $self->sth_selectBlockhashById->finish;
return return if not defined $ref;
if not defined $ref; $hash = $ref->[0];
return $ref->[0]; _cacheBlockhash( {'hash'=>$hash, 'id'=>$id} );
return $hash;
} }
__PACKAGE__->mk_accessors( 'sth_selectBlockhashByHash' ); __PACKAGE__->mk_accessors( 'sth_selectBlockhashByHash' );
sub selectBlockhashByHash sub selectBlockhashByHash
{ {
my ( $self, $hash )= @_; my ( $self, $hash )= @_;
$hash = lc $hash;
if ( not defined $self->sth_selectBlockhashByHash ) if ( not defined $self->sth_selectBlockhashByHash )
{ {
$self->sth_selectBlockhashByHash( $self->sth_selectBlockhashByHash(
...@@ -134,47 +246,68 @@ sub selectBlockhashByHash ...@@ -134,47 +246,68 @@ sub selectBlockhashByHash
)) ))
); );
} }
my $inserts = 0; $hash = lc $hash;
while ( 1 ) my $id = _cacheBlockhash( {'hash'=>$hash} );
{ return $id if defined $id;
# Try to look up the hash $self->sth_selectBlockhashByHash->execute( $hash );
$self->sth_selectBlockhashByHash->execute( $hash ); my $ref = $self->sth_selectBlockhashByHash->fetchrow_arrayref;
my $ref = $self->sth_selectBlockhashByHash->fetchrow_arrayref; $self->sth_selectBlockhashByHash->finish;
$self->sth_selectBlockhashByHash->finish; $id = defined $ref
# If found, return it immediately ? $ref->[0]
return $ref->[0] if defined $ref; : $self->insertBlockhash( $hash );
# Do maximum 1 attempt to insert a hash die if not defined $id;
return if $inserts > 0; _cacheBlockhash( {'id'=>$id, 'hash'=>$hash} );
# Insert a hash, because it is not already in the database. return $id;
$self->insertBlockhash( $hash );
# Avoid loops.
$inserts++;
}
} }
__PACKAGE__->mk_accessors( 'sth_insertBlockhash' ); my @cacheAccount;
sub insertBlockhash sub _cacheAccount
{ {
my ( $self, $hash )= @_; my $args = shift;
$hash = lc $hash; my $i = scalar @cacheAccount;
if ( not defined $self->sth_insertBlockhash ) if ( exists $args->{'id'} and exists $args->{'account'} )
{ {
$self->sth_insertBlockhash( # Don't put something in the cache, if it's already there
$self->dbh->prepare(q( while ( $i-- > 0 )
INSERT INTO blockhash {
( hash ) if ( $cacheAccount[$i]->{'id'} eq $args->{'id'} )
VALUES (?) {
)) return;
) }
}
# Store in the cache
push @cacheAccount, { 'id' => $args->{'id'}, 'account' => $args->{'account'} };
# may 10 entries
splice @cacheAccount, 10;
return;
} }
$self->sth_insertBlockhash->execute( $hash ); # If just one element is given, then it is a lookup
if ( defined $args->{'id'} )
{
while ( $i-- > 0 )
{
return $cacheAccount[$i]->{'account'}
if $cacheAccount[$i]->{'id'} eq $args->{'id'};
}
return;
}
if ( defined $args->{'account'} )
{
while ( $i-- > 0 )
{
return $cacheAccount[$i]->{'id'}
if $cacheAccount[$i]->{'account'} eq $args->{'account'};
}
return;
}
return;
} }
__PACKAGE__->mk_accessors( 'sth_selectAccountIdByAddress' ); __PACKAGE__->mk_accessors( 'sth_selectAccountIdByAddress' );
my %account; my %account;
sub selectAccountIdByAddress sub selectAccountIdByAddress
{ {
my ( $self, $address )= @_; my ( $self, $address ) = @_;
$address = lc $address; $address = lc $address;
if ( not defined $self->sth_selectAccountIdByAddress ) if ( not defined $self->sth_selectAccountIdByAddress )
{ {
...@@ -186,23 +319,18 @@ sub selectAccountIdByAddress ...@@ -186,23 +319,18 @@ sub selectAccountIdByAddress
)) ))
); );
} }
return $account{$address} if exists $account{$address}; my $id = _cacheAccount( {'address'=>$address} );
while ( not exists $account{$address} ) return $id if defined $id;
{ # Try to look up the hash
# Try to look up the hash $self->sth_selectAccountIdByAddress->execute( $address );
$self->sth_selectAccountIdByAddress->execute( $address ); my $ref = $self->sth_selectAccountIdByAddress->fetchrow_arrayref;
my $ref = $self->sth_selectAccountIdByAddress->fetchrow_arrayref; $self->sth_selectAccountIdByAddress->finish;
$self->sth_selectAccountIdByAddress->finish; $id = defined $ref
# If found, return it immediately ? $ref->[0]
if ( defined $ref ) : $self->insertAccount( $address );
{ die if not defined $id;
$account{$address} = $ref->[0]; _cacheAccount( {'id'=>$id, 'address'=>$address} );
last; return $id;
}
# Insert a hash, because it is not already in the database.
$self->insertAccount( $address );
}
return $account{$address};
} }
__PACKAGE__->mk_accessors( 'sth_insertAccount' ); __PACKAGE__->mk_accessors( 'sth_insertAccount' );
...@@ -221,13 +349,14 @@ sub insertAccount ...@@ -221,13 +349,14 @@ sub insertAccount
); );
} }
$self->sth_insertAccount->execute( $address ); $self->sth_insertAccount->execute( $address );
return $self->dbh->last_insert_id(undef,undef,undef,undef,{sequence=>'account_id_seq'});
} }
__PACKAGE__->mk_accessors( 'sth_insertBlock' ); __PACKAGE__->mk_accessors( 'sth_insertBlock' );
sub insertBlock sub insertBlock
{ {
my ( $self, $hash, $parentHash, $number, $timestamp, $difficulty, $gasUsed, $gasLimit, $size ) my ( $self, $hash, $parentHash, $number, $timestamp, $difficulty, $gasUsed, $gasLimit, $size )
= @_; = @_;
if ( not defined $self->sth_insertBlock ) if ( not defined $self->sth_insertBlock )
{ {
$self->sth_insertBlock( $self->sth_insertBlock(
...@@ -239,16 +368,16 @@ sub insertBlock ...@@ -239,16 +368,16 @@ sub insertBlock
)) ))
); );
} }
my $id = $self->selectBlockhashByHash( $hash ); my $id = $self->selectBlockhashByHash( $hash );
my $parentid = $self->selectBlockhashByHash( $parentHash ); my $parentid = $self->selectBlockhashByHash( $parentHash );
$self->sth_insertBlock->execute( $id, $parentid, $number, $timestamp, $difficulty, $gasUsed, $gasLimit, $size ); $self->sth_insertBlock->execute( $id, $parentid, $number, $timestamp, $difficulty, $gasUsed, $gasLimit, $size );
} }
__PACKAGE__->mk_accessors( 'sth_selectBlockByHash' ); __PACKAGE__->mk_accessors( 'sth_selectBlockByHash' );
sub selectBlockByHash sub selectBlockByHash
{ {
my ( $self, $hash) = @_; my ( $self, $hash) = @_;
$hash = lc $hash; $hash = lc $hash;
if ( not defined $self->sth_selectBlockByHash ) if ( not defined $self->sth_selectBlockByHash )
{ {
$self->sth_selectBlockByHash( $self->sth_selectBlockByHash(
...@@ -259,20 +388,19 @@ sub selectBlockByHash ...@@ -259,20 +388,19 @@ sub selectBlockByHash
)) ))
); );
} }
my $blockid = $self->selectBlockhashByHash( $hash ); my $blockid = $self->selectBlockhashByHash( $hash );
$self->sth_selectBlockByHash->execute( $blockid ); $self->sth_selectBlockByHash->execute( $blockid );
my $ref = $self->sth_selectBlockByHash->fetchrow_hashref; my $ref = $self->sth_selectBlockByHash->fetchrow_hashref;
$self->sth_selectBlockByHash->finish; $self->sth_selectBlockByHash->finish;
return return $ref if defined $ref;
if not defined $ref; return;
return $ref;
} }
__PACKAGE__->mk_accessors( 'sth_selectTransactionByHash' ); __PACKAGE__->mk_accessors( 'sth_selectTransactionByHash' );
sub selectTransactionByHash sub selectTransactionByHash
{ {
my ( $self, $hash) = @_; my ( $self, $hash) = @_;
$hash = lc $hash; $hash = lc $hash;
if ( not defined $self->sth_selectTransactionByHash ) if ( not defined $self->sth_selectTransactionByHash )
{ {
$self->sth_selectTransactionByHash( $self->sth_selectTransactionByHash(
...@@ -284,17 +412,17 @@ sub selectTransactionByHash ...@@ -284,17 +412,17 @@ sub selectTransactionByHash
); );
} }
$self->sth_selectTransactionByHash->execute( $hash ); $self->sth_selectTransactionByHash->execute( $hash );
my $ref = $self->sth_selectTransactionByHash->fetchrow_hashref; my $ref = $self->sth_selectTransactionByHash->fetchrow_hashref;
$self->sth_selectTransactionByHash->finish; $self->sth_selectTransactionByHash->finish;
return $ref return $ref if defined $ref;
if defined $ref;
return; return;
} }
__PACKAGE__->mk_accessors( 'sth_updateWhoSealed' ); __PACKAGE__->mk_accessors( 'sth_updateWhoSealed' );
sub updateWhoSealed sub updateWhoSealed
{ {
my ( $self, $sealerhash, $blockhash ) = @_; my ( $self, $sealerhash, $blockhash )
= @_;
if ( not defined $self->sth_updateWhoSealed ) if ( not defined $self->sth_updateWhoSealed )
{ {
$self->sth_updateWhoSealed( $self->sth_updateWhoSealed(
...@@ -306,15 +434,15 @@ sub updateWhoSealed ...@@ -306,15 +434,15 @@ sub updateWhoSealed
)) ))
); );
} }
my $blockid = $self->selectBlockhashByHash( $blockhash ); my $blockid = $self->selectBlockhashByHash( $blockhash );
my $sealerid = $self->selectAccountIdByAddress( $sealerhash ); my $sealerid = $self->selectAccountIdByAddress( $sealerhash );
$self->sth_updateWhoSealed->execute( $sealerid, $blockid ); $self->sth_updateWhoSealed->execute( $sealerid, $blockid );
} }
__PACKAGE__->mk_accessors( 'sth_selectmaxunknownsigned' ); __PACKAGE__->mk_accessors( 'sth_selectmaxunknownsigned' );
sub selectMaxUnknownSigned sub selectMaxUnknownSigned
{ {
my ( $self ) = @_; my ( $self ) = @_;
if ( not defined $self->sth_selectmaxunknownsigned ) if ( not defined $self->sth_selectmaxunknownsigned )
{ {
$self->sth_selectmaxunknownsigned( $self->sth_selectmaxunknownsigned(
...@@ -330,12 +458,12 @@ sub selectMaxUnknownSigned ...@@ -330,12 +458,12 @@ sub selectMaxUnknownSigned
); );
} }
$self->sth_selectmaxunknownsigned->execute(); $self->sth_selectmaxunknownsigned->execute();
my $ref = $self->sth_selectmaxunknownsigned->fetchrow_arrayref; my $ref = $self->sth_selectmaxunknownsigned->fetchrow_arrayref;
$self->sth_selectmaxunknownsigned->finish(); $self->sth_selectmaxunknownsigned->finish();
return if not defined $ref; return if not defined $ref;
my $blockid = $ref->[0]; my $blockid = $ref->[0];
my $blockhash = $self->selectBlockhashById( $blockid ); my $blockhash = $self->selectBlockhashById( $blockid );
return $blockhash; return $blockhash;
} }
1; 1;
docker-compose.yml
+ 4
− 2
View file @ ade0a55a
...@@ -12,7 +12,7 @@ services: ...@@ -12,7 +12,7 @@ services:
build: postgres build: postgres
restart: always restart: always
environment: environment:
POSTGRES_PASSWORD: onlythelonely POSTGRES_PASSWORD: theswampthing
volumes: volumes:
- pgdata:/var/lib/postgresql/data - pgdata:/var/lib/postgresql/data
ports: ports:
...@@ -22,8 +22,10 @@ services: ...@@ -22,8 +22,10 @@ services:
build: collector build: collector
restart: always restart: always
environment: environment:
POSTGRES_PASSWORD: onlythelonely POSTGRES_PASSWORD: theswampthing
BFANODE: http://200.108.146.200:8545/ BFANODE: http://200.108.146.200:8545/
volumes:
- ./collector:/home/bfa/collector
# adminer: # adminer:
# image: adminer # image: adminer
......
postgres/10-postgres.sql
+ 9
− 9
View file @ ade0a55a
...@@ -50,8 +50,8 @@ CREATE TABLE "public"."block" ( ...@@ -50,8 +50,8 @@ CREATE TABLE "public"."block" (
"sealerAccountId" bigint, "sealerAccountId" bigint,
"timestamp" bigint NOT NULL, "timestamp" bigint NOT NULL,
"difficulty" smallint NOT NULL, "difficulty" smallint NOT NULL,
"gasUsed" bigint NOT NULL, "gasUsed" numeric(79) NOT NULL,
"gasLimit" bigint NOT NULL, "gasLimit" numeric(79) NOT NULL,
"size" bigint NOT NULL, "size" bigint NOT NULL,
CONSTRAINT "block_id" CONSTRAINT "block_id"
PRIMARY KEY ("id"), PRIMARY KEY ("id"),
...@@ -90,23 +90,22 @@ CREATE TABLE "public"."transaction" ( ...@@ -90,23 +90,22 @@ CREATE TABLE "public"."transaction" (
NOT NULL, NOT NULL,
"nonce" bigint "nonce" bigint
NOT NULL, NOT NULL,
"gas" bigint "gas" numeric(79)
NOT NULL, NOT NULL,
"gasPrice" bigint "gasPrice" numeric(79)
NOT NULL, NOT NULL,
"value" bigint "value" numeric(79)
NOT NULL, NOT NULL,
"fromAccountId" bigint "fromAccountId" bigint
NOT NULL, NOT NULL,
"toAccountId" bigint "toAccountId" bigint,
NOT NULL,
"contractaddressAccountId" "contractaddressAccountId"
bigint, bigint,
"status" smallint "status" smallint
NOT NULL, NOT NULL,
"gasUsed" bigint "gasUsed" numeric(79)
NOT NULL, NOT NULL,
"inputlen" smallint "inputlen" integer
NOT NULL, NOT NULL,
"input" text "input" text
NOT NULL, NOT NULL,
...@@ -162,6 +161,7 @@ CREATE VIEW t ...@@ -162,6 +161,7 @@ CREATE VIEW t
AS AS
SELECT transaction.hash "transaction hash", SELECT transaction.hash "transaction hash",
block.number "block number", block.number "block number",
block.timestamp "block timestamp",
"type", "type",
nonce, nonce,
gas, gas,
......
postgres/90-add-db-user.sh
+ 4
− 4
View file @ ade0a55a
#!/bin/sh #!/bin/sh
psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
CREATE USER leer WITH PASSWORD 'bfa'; CREATE USER dgsi WITH PASSWORD 'bfa';
GRANT CONNECT ON DATABASE postgres TO leer; GRANT CONNECT ON DATABASE postgres TO dgsi;
GRANT SELECT ON ALL TABLES IN SCHEMA public TO leer; GRANT SELECT ON ALL TABLES IN SCHEMA public TO dgsi;
GRANT UPDATE (shortname,name) ON TABLE public.account TO leer; GRANT UPDATE (shortname,name) ON TABLE public.account TO dgsi;
EOSQL EOSQL
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment