diff --git a/collector/collector.pl b/collector/collector.pl
index 77966456eb62a25a3bad2f007fa060d1badc9a6d..297cba143cdda2fefba510b09387db60f9f88f21 100755
--- a/collector/collector.pl
+++ b/collector/collector.pl
@@ -9,7 +9,7 @@ use Data::Dumper;
use Math::BigInt;
use LWP;
use JSON;
-use Net::DNS;
+use Net::DNS;
use Net::DNS::Resolver;
use lib '/home/bfa/collector';
use lib 'collector';
@@ -17,10 +17,11 @@ use lib '.';
use sql;
# 47525974938;
-my $netversion = Math::BigInt->new( "0xb10c4d39a" );
+my $netversion = Math::BigInt->new( "0xb10c4d39a" );
# 200941592
-my $chainid = Math::BigInt->new( "0xbfa2018" );
-my $idcounter = 0;
+my $chainid = Math::BigInt->new( "0xbfa2018" );
+my $idcounter = 0;
+my $stop_request = 0;
my $ua;
my @blockqueue;
my %sealers;
@@ -32,90 +33,99 @@ sub info(@)
# if 1 param only
unshift @_, '%s'
if $#_ == 0;
- my $format = shift;
- $format = "%s: " . $format;
- printf $format, astime(time), @_;
+ my $format = shift;
+ $format = "%s: " . $format;
+ my $txt = sprintf $format, astime(time), @_;
+ $txt .= "\n" if $txt !~ /\n$/;
+ print $txt;
}
sub astime
{
- my @t = localtime( $_[0] );
- $t[5] += 1900;
- $t[4] += 1;
+ my @t = localtime( $_[0] );
+ $t[5] += 1900;
+ $t[4] += 1;
return sprintf '%04d%02d%02d-%02d%02d%02d', @t[5,4,3,2,1,0];
}
sub shorthash($)
{
- local $_ = $_[0];
+ local $_ = $_[0];
+ return unless defined $_;
s/^(0x.......).*(.......)$/$1..$2/;
return $_;
}
+sub stop
+{
+ $stop_request = 1;
+ print STDERR "\rStop request received ($_[0]).\n";
+}
+
sub rpcreq
{
my ( $opname, @params )= @_;
- $ua = LWP::UserAgent->new( keep_alive => 10 )
+ $ua = LWP::UserAgent->new( keep_alive => 10 )
if not defined $ua;
# $ua->ssl_opts( 'verify_hostname' => 0 );
- my %args = (
- jsonrpc => '2.0',
- method => $opname,
- id => $idcounter++,
+ my %args = (
+ jsonrpc => '2.0',
+ method => $opname,
+ id => $idcounter++,
);
- my @args = map {
- my $v = $args{$_};
- $v = qq/"${v}"/ if $v !~ /^\d+$/;
+ my @args = map {
+ my $v = $args{$_};
+ $v = qq/"${v}"/ if $v !~ /^\d+$/;
sprintf qq/"%s":%s/, $_, $v;
} keys %args;
if ( scalar @params )
{
- my $p = '"params":[';
+ my $p = '"params":[';
foreach my $param ( @params )
{
$param = '"' . $param . '"'
if $param ne 'true'
and $param ne 'false'
and $param !~ /^\d+$/;
- $p .= $param . ',';
+ $p .= $param . ',';
}
chop $p;
- $p .= ']';
+ $p .= ']';
push @args, $p;
}
- my $args = '{' . join(',',@args) . '}';
+ my $args = '{' . join(',',@args) . '}';
die 'The environment variable $BFANODE must contain a URL for a ' .
'server to connect to (e.g. https://10.20.30.40/), but it it ' .
'not set. Stopped'
unless defined $ENV{'BFANODE'};
- my $res = $ua->post(
+ my $res = $ua->post(
$ENV{BFANODE},
- 'Content-Type' => 'application/json',
- 'Content' => $args,
+ 'Content-Type' => 'application/json',
+ 'Content' => $args,
);
die $res->status_line
unless $res->is_success;
- my $json = decode_json($res->content);
+ my $json = decode_json($res->content);
return $json->{'result'};
}
sub getblock
{
- my $block = lc shift;
+ my $block = lc shift;
my $rpccmd;
if ( $block =~ /^0x[\da-f]{64}$/ )
{
- $rpccmd = 'eth_getBlockByHash';
+ $rpccmd = 'eth_getBlockByHash';
}
else
{
- $rpccmd = 'eth_getBlockByNumber';
+ $rpccmd = 'eth_getBlockByNumber';
if ( $block =~ /^\d+$/ )
{
- $block = Math::BigInt->new( $block )->as_hex;
+ $block = Math::BigInt->new( $block )->as_hex;
}
}
- my $json = rpcreq( $rpccmd, $block, 'false' );
+ my $json = rpcreq( $rpccmd, $block, 'false' );
die "We should have received some kind of JSON from our RPC call, " .
"but apparently not. Stopped "
if not defined $json;
@@ -123,10 +133,10 @@ sub getblock
foreach my $key (qw(
timestamp gasUsed gasLimit difficulty number totalDifficulty size
)) {
- $json->{$key} = Math::BigInt->new( $json->{$key} )->bstr
+ $json->{$key} = Math::BigInt->new( $json->{$key} )->bstr
if exists $json->{$key};
}
- info "B %s %s %s %s %s %s/%s\n",
+ info "B %s %s %s %s %s %s/%s",
shorthash $json->{'hash'},
astime($json->{'timestamp'}),
$json->{'number'},
@@ -151,25 +161,26 @@ sub getblock
getTransByHash( $txhash );
}
}
+ # Put the parent block in front in @blockqueue
unshift @blockqueue, lc $json->{'parentHash'}
if $json->{'number'} != 0;
}
sub getTransByHash
{
- my $hash = shift;
+ my $hash = shift;
# Don't get it via RPC if we already have it in the database
return
if $sql->selectTransactionByHash( $hash );
- my $json = rpcreq(
+ my $json = rpcreq(
'eth_getTransactionByHash',
$hash
);
die "Invalid object received when asking for $hash:\n".Dumper($json) if not exists $json->{'blockNumber'};
- my $max = 0;
- my $rcpt = undef;
+ my $max = 0;
+ my $rcpt = undef;
do {
- $rcpt = rpcreq(
+ $rcpt = rpcreq(
'eth_getTransactionReceipt',
$hash
);
@@ -179,40 +190,44 @@ sub getTransByHash
warn "Couldn't get the transaction receipt for $hash\n";
}
} while not $rcpt and $max++ < 10;
- die "Couldn't get the transaction receipt for $hash\n" if not $rcpt;
- $json->{'status'} = $rcpt->{'status'};
- $json->{'gasUsed'} = $rcpt->{'gasUsed'};
+ die "Couldn't get the transaction receipt for $hash:\n" if not $rcpt;
+ foreach my $key (qw( hash blockHash nonce gas gasPrice value from to input ))
+ {
+ die "JSON missing key $key:".Dumper($json) unless exists $json->{$key};
+ }
+ $json->{'status'} = $rcpt->{'status'};
+ $json->{'gasUsed'} = $rcpt->{'gasUsed'};
$json->{'contractAddress'}
- = $rcpt->{'contractAddress'};
+ = $rcpt->{'contractAddress'};
foreach my $key (qw( nonce value gas gasPrice gasUsed status))
{
- $json->{$key} = Math::BigInt->new( $json->{$key} )->bstr;
+ $json->{$key} = Math::BigInt->new( $json->{$key} )->bstr;
}
- info "T %s %s %s %s -> %s\n",
+ info "T %s %s %s %s -> %s",
shorthash $json->{'hash'},
$json->{'nonce'},
$json->{'value'},
shorthash $json->{'from'},
- shorthash $json->{'to'};
- my $input = $json->{'input'};
- my $inputlen = 0;
+ shorthash ($json->{'to'}||'<NULL>');
+ my $input = $json->{'input'};
+ my $inputlen = 0;
if ( $input =~ /^0x/ )
{
- $inputlen = (length($input)-2) / 2;
+ $inputlen = (length($input)-2) / 2;
}
- my @args = (
+ my @args = (
lc $json->{'hash'},
lc $json->{'blockHash'},
- $json->{'nonce'},
- $json->{'gas'},
- $json->{'gasPrice'},
- $json->{'value'},
+ $json->{'nonce'},
+ $json->{'gas'},
+ $json->{'gasPrice'},
+ $json->{'value'},
lc $json->{'from'},
- lc $json->{'to'},
- $json->{'status'},
- $json->{'gasUsed'},
+ defined $json->{'to'} ? lc $json->{'to'} : undef,
+ $json->{'status'},
+ $json->{'gasUsed'},
$input,
- $inputlen,
+ $inputlen
);
if ( $json->{'contractAddress'} )
{
@@ -227,8 +242,8 @@ sub getTransByHash
sub versioncheck
{
my $n;
- my $ok = 1;
- $n = Math::BigInt->new( rpcreq( "net_version" ) );
+ my $ok = 1;
+ $n = Math::BigInt->new( rpcreq( "net_version" ) );
if ( $n->bcmp( $netversion ) != 0 )
{
warn "Network says it has net.version "
@@ -238,13 +253,13 @@ sub versioncheck
. "). Expected $netversion ("
. $netversion->as_hex
. ").\n";
- $ok = 0;
+ $ok = 0;
}
- $n = Math::BigInt->new( rpcreq( "eth_chainId" ) );
+ $n = Math::BigInt->new( rpcreq( "eth_chainId" ) );
if ( $n->bcmp( $chainid ) != 0 )
{
warn sprintf("Network says it has eth.chainId %s (%s). Expected %s (%s).\n", $n->bstr, $n->as_hex, $chainid->bstr, $chainid->as_hex);
- $ok = 0;
+ $ok = 0;
}
exit 1 if not $ok;
return $ok;
@@ -252,80 +267,94 @@ sub versioncheck
sub getsnap
{
- my $blockhash = $_[0];
- my $json = rpcreq(
+ my $blockhash = $_[0];
+ my $json = rpcreq(
'clique_getSnapshotAtHash',
$blockhash
);
# Sanity check 1 - does the lookup return the block?
return if not defined $json;
- my $recents = $json->{'recents'};
+ my $recents = $json->{'recents'};
# Blockhash of the requested block - we must use this for the insert
- $blockhash = $json->{'hash'};
- my $blocknumber = Math::BigInt->new( $json->{'number'} );
+ $blockhash = $json->{'hash'};
+ my $blocknumber = Math::BigInt->new( $json->{'number'} );
my @infonumbers;
while ( not $blocknumber->is_zero )
{
- my $sealerhash = $recents->{$blocknumber->bstr};
- last if not defined $sealerhash;
+ my $sealerhash = $recents->{$blocknumber->bstr};
+ last if not defined $sealerhash;
push @infonumbers, $blocknumber->bstr;
- my $ref = $sql->selectBlockByHash( $blockhash );
+ my $ref = $sql->selectBlockByHash( $blockhash );
# Stop if we get to blocks which we still haven't stored in the db
- return ( undef, $blocknumber )
- if not defined $ref;
+ if ( not defined $ref )
+ {
+ $blockhash = undef;
+ last;
+ }
# Stop if we got to a block with sealers.
- return ( undef, $blocknumber )
- if exists $ref->{'sealerAccountId'} and defined $ref->{'sealerAccountId'};
+ if ( exists $ref->{'sealerAccountId'} and defined $ref->{'sealerAccountId'} )
+ {
+ $blockhash = undef;
+ last;
+ }
$sql->updateWhoSealed( $sealerhash, $blockhash );
- my $parentblockid = $ref->{'parentBlockhashId'};
+ my $parentblockid = $ref->{'parentBlockhashId'};
# For next block...
- $blockhash = $sql->selectBlockhashById( $parentblockid );
+ $blockhash = $sql->selectBlockhashById( $parentblockid );
$blocknumber->bsub(1);
}
- info "S Snapshot at block # @infonumbers.\n";
+ info "S Snapshot at block # @infonumbers."
+ if @infonumbers;
return ($blockhash, $blocknumber);
}
# Find strands of blocks in the database which have their sealer set to NULL
sub allsnaps
{
- my $blockhash = $sql->selectMaxUnknownSigned;
- my $committer = 0;
+ my $blockhash = $sql->selectMaxUnknownSigned;
while ( defined $blockhash )
{
my $blocknumber;
- ( $blockhash, $blocknumber )
- = getsnap( $blockhash );
- $sql->commit
- if ( $committer++ % 75 ) == 0;
+ ( $blockhash, $blocknumber )= getsnap( $blockhash );
last if $blocknumber->is_zero;
}
}
sub main
{
- $| = 1;
versioncheck();
# Give Postgres time to start up.
- sleep(3);
- $sql = sql->new();
- info "Looking for orphaned blocks in the "
- . "database.\n";
- push @blockqueue, $sql->listOrphans();
-
- while ( 1 )
+ sleep( 3 );
+ $sql = sql->new();
+ my $lastorphancheck= 0;
+ $SIG{'HUP'} = \&stop;
+ $SIG{'INT'} = \&stop;
+ $SIG{'TERM'} = \&stop;
+ $SIG{'TSTP'} = \&stop;
+ while ( not $stop_request )
{
- unshift @blockqueue, 'latest';
- while ( @blockqueue )
+ my $now = time();
+ if ( $lastorphancheck <= $now+600 )
{
- my $maxinarow = 100;
+ $lastorphancheck = $now;
+ # Orphans happen if some blocks have been deleted
+ # or if the initial synchronization hasn't finished.
+ # This check should be done when orphans can be
+ # suspected, but when is that? Maybe every 10 minutes.
+ push @blockqueue, $sql->listOrphans();
+ }
+ do
+ {
+ unshift @blockqueue, 'latest';
+ my $maxinarow = 100;
getblock( shift @blockqueue )
- while @blockqueue and --$maxinarow;
+ while @blockqueue and --$maxinarow and not $stop_request;
# Find out who signed all blocks
- allsnaps();
+ allsnaps() if not $stop_request;
+ info "Committing.";
$sql->commit;
- }
- sleep 5;
+ } while @blockqueue and not $stop_request;
+ sleep 5 if not @blockqueue and not $stop_request;
}
}
diff --git a/collector/sql.pm b/collector/sql.pm
index 28cec895035f46093c3f11897bda3d3539b68ed2..204baa39e26a0bb0ae00d632d4f59dbc3fa5cf31 100644
--- a/collector/sql.pm
+++ b/collector/sql.pm
@@ -12,8 +12,8 @@ use base qw( Class::Accessor );
__PACKAGE__->mk_accessors( 'dbh' );
sub new
{
- my ( $class ) = @_;
- my $self = bless {}, ref $class || $class;
+ my ( $class ) = @_;
+ my $self = bless {}, ref $class || $class;
$self->dbh( DBI->connect(
'dbi:Pg:dbname=postgres;host=postgres','postgres',$ENV{'POSTGRES_PASSWORD'},
{AutoCommit=>0,RaiseError=>1}
@@ -23,37 +23,80 @@ sub new
sub commit
{
- my ( $self ) = @_;
+ my ( $self ) = @_;
$self->dbh->commit;
}
sub listOrphans
{
- my ( $self ) = @_;
- my $sth = $self->dbh->prepare(q(
- SELECT "parentBlockhashId", id
- FROM block
- WHERE "parentBlockhashId" NOT IN (
- SELECT id
- FROM block
- )
+ my ( $self ) = @_;
+ my $max = $self->maxBlockNumber;
+ # max will be NULL when there are no blocks in the table.
+ return unless defined $max;
+ my $sth_orphan = $self->dbh->prepare(q(
+ SELECT "parentBlockhashId", id
+ FROM block
+ WHERE number >= ?
+ AND number <= ?
+ AND "parentBlockhashId" NOT IN (
+ SELECT id FROM block
+ WHERE number >= ?-1
+ AND number <= ?-1
+ )
+ ORDER BY number;
));
my @results;
- $sth->execute();
- while ( my $row = $sth->fetchrow_arrayref )
+ my $stepping = 50000;
+ my $i = 0;
+ # Had to split this into sections or it would take too long
+ # (1.4M rows, 3 CPUs, stopped after 8 hours of execution)
+ # This takes less than 10 seconds with 1.4M rows.
+ while ( $i < $max )
{
- # Block 0 has it's parent listed as 0x0{64}
- next if $row->[0] =~ /^0x0{64}$/;
- push @results, $self->selectBlockhashById( $row->[0] );
+ my $bottom = $i;
+ my $top = $i + $stepping;
+ $sth_orphan->execute($bottom, $top, $bottom, $top);
+ $i += $stepping;
+ while ( my $row = $sth_orphan->fetchrow_arrayref )
+ {
+ next
+ if $row->[0]=~ /^0x0{64}$/;
+ push @results, $self->selectBlockhashById( $row->[0] );
+ }
}
- $sth->finish;
+ $sth_orphan->finish;
+ # Take the lowest number and put it in the back
+ my $first = shift @results;
+ push @results, $first
+ if defined $first;
return @results;
}
+__PACKAGE__->mk_accessors( 'sth_max' );
+sub maxBlockNumber
+{
+ my ( $self ) = @_;
+ if ( not defined $self->sth_max )
+ {
+ $self->sth_max(
+ $self->dbh->prepare(q(
+ SELECT MAX(number)
+ FROM block
+ ))
+ );
+ }
+ $self->sth_max->execute;
+ my $ref = $self->sth_max->fetchrow_arrayref;
+ $self->sth_max->finish;
+ die "Unable to get the highest block number. Stopped"
+ if not defined $ref;
+ return $ref->[0];
+}
+
__PACKAGE__->mk_accessors( 'sth_insertTransactionWithContractAddress' );
sub insertTransactionWithContractAddress
{
- my $self = shift;
+ my $self = shift;
if ( not defined $self->sth_insertTransactionWithContractAddress )
{
$self->sth_insertTransactionWithContractAddress(
@@ -67,17 +110,17 @@ sub insertTransactionWithContractAddress
))
);
}
- $_[1] = $self->selectBlockhashByHash( $_[1] );
- $_[6] = $self->selectAccountIdByAddress( $_[6] );
- $_[7] = $self->selectAccountIdByAddress( $_[7] );
- $_[12] = $self->selectAccountIdByAddress( $_[12] );
+ $_[1] = $self->selectBlockhashByHash( $_[1] );
+ $_[6] = $self->selectAccountIdByAddress( $_[6] );
+ $_[7] = $self->selectAccountIdByAddress( $_[7] ) if defined $_[7];
+ $_[12] = $self->selectAccountIdByAddress( $_[12] ) if defined $_[12];
$self->sth_insertTransactionWithContractAddress->execute( @_, 'explicit' );
}
__PACKAGE__->mk_accessors( 'sth_insertTransaction' );
sub insertTransaction
{
- my $self = shift;
+ my $self = shift;
if ( not defined $self->sth_insertTransaction )
{
$self->sth_insertTransaction(
@@ -91,16 +134,82 @@ sub insertTransaction
))
);
}
- $_[1] = $self->selectBlockhashByHash( $_[1] );
- $_[6] = $self->selectAccountIdByAddress( $_[6] );
- $_[7] = $self->selectAccountIdByAddress( $_[7] );
+ $_[1] = $self->selectBlockhashByHash( $_[1] );
+ $_[6] = $self->selectAccountIdByAddress( $_[6] );
+ $_[7] = $self->selectAccountIdByAddress( $_[7] ) if defined $_[7];
$self->sth_insertTransaction->execute( @_, 'explicit' );
}
+my @cacheBlockhash;
+sub _cacheBlockhash
+{
+ my $args = shift;
+ my $i = scalar @cacheBlockhash;
+ if ( exists $args->{'id'} and exists $args->{'hash'} )
+ {
+ # Don't put something in the cache, if it's already there
+ while ( $i-- > 0 )
+ {
+ if ( $cacheBlockhash[$i]->{'id'} eq $args->{'id'} )
+ {
+ return;
+ }
+ }
+ # Store in the cache
+ push @cacheBlockhash, { 'id' => $args->{'id'}, 'hash' => $args->{'hash'} };
+ # may 10 entries
+ splice @cacheBlockhash, 10;
+ return;
+ }
+ # If just one element is given, then it is a lookup
+ if ( defined $args->{'id'} )
+ {
+ while ( $i-- > 0 )
+ {
+ if ( $cacheBlockhash[$i]->{'id'} eq $args->{'id'} )
+ {
+ return $cacheBlockhash[$i]->{'hash'}
+ }
+ }
+ return;
+ }
+ if ( defined $args->{'hash'} )
+ {
+ while ( $i-- > 0 )
+ {
+ if ( $cacheBlockhash[$i]->{'hash'} eq $args->{'hash'} )
+ {
+ return $cacheBlockhash[$i]->{'id'};
+ }
+ }
+ return;
+ }
+ return;
+}
+
+__PACKAGE__->mk_accessors( 'sth_insertBlockhash' );
+sub insertBlockhash
+{
+ my ( $self, $hash ) = @_;
+ $hash = lc $hash;
+ if ( not defined $self->sth_insertBlockhash )
+ {
+ $self->sth_insertBlockhash(
+ $self->dbh->prepare(q(
+ INSERT INTO blockhash
+ ( hash )
+ VALUES (?)
+ ))
+ )
+ }
+ $self->sth_insertBlockhash->execute( $hash );
+ return $self->dbh->last_insert_id(undef,undef,undef,undef,{sequence=>'blockhash_id_seq'});
+}
+
__PACKAGE__->mk_accessors( 'sth_selectBlockhashById' );
sub selectBlockhashById
{
- my ( $self, $id ) = @_;
+ my ( $self, $id ) = @_;
if ( not defined $self->sth_selectBlockhashById )
{
$self->sth_selectBlockhashById(
@@ -111,19 +220,22 @@ sub selectBlockhashById
))
);
}
+ my $hash = _cacheBlockhash( {'id'=>$id} );
+ return $hash if defined $hash;
+ # Ask the database
$self->sth_selectBlockhashById->execute( $id );
- my $ref = $self->sth_selectBlockhashById->fetchrow_arrayref;
+ my $ref = $self->sth_selectBlockhashById->fetchrow_arrayref;
$self->sth_selectBlockhashById->finish;
- return
- if not defined $ref;
- return $ref->[0];
+ return if not defined $ref;
+ $hash = $ref->[0];
+ _cacheBlockhash( {'hash'=>$hash, 'id'=>$id} );
+ return $hash;
}
__PACKAGE__->mk_accessors( 'sth_selectBlockhashByHash' );
sub selectBlockhashByHash
{
my ( $self, $hash )= @_;
- $hash = lc $hash;
if ( not defined $self->sth_selectBlockhashByHash )
{
$self->sth_selectBlockhashByHash(
@@ -134,47 +246,68 @@ sub selectBlockhashByHash
))
);
}
- my $inserts = 0;
- while ( 1 )
- {
- # Try to look up the hash
- $self->sth_selectBlockhashByHash->execute( $hash );
- my $ref = $self->sth_selectBlockhashByHash->fetchrow_arrayref;
- $self->sth_selectBlockhashByHash->finish;
- # If found, return it immediately
- return $ref->[0] if defined $ref;
- # Do maximum 1 attempt to insert a hash
- return if $inserts > 0;
- # Insert a hash, because it is not already in the database.
- $self->insertBlockhash( $hash );
- # Avoid loops.
- $inserts++;
- }
+ $hash = lc $hash;
+ my $id = _cacheBlockhash( {'hash'=>$hash} );
+ return $id if defined $id;
+ $self->sth_selectBlockhashByHash->execute( $hash );
+ my $ref = $self->sth_selectBlockhashByHash->fetchrow_arrayref;
+ $self->sth_selectBlockhashByHash->finish;
+ $id = defined $ref
+ ? $ref->[0]
+ : $self->insertBlockhash( $hash );
+ die if not defined $id;
+ _cacheBlockhash( {'id'=>$id, 'hash'=>$hash} );
+ return $id;
}
-__PACKAGE__->mk_accessors( 'sth_insertBlockhash' );
-sub insertBlockhash
+my @cacheAccount;
+sub _cacheAccount
{
- my ( $self, $hash )= @_;
- $hash = lc $hash;
- if ( not defined $self->sth_insertBlockhash )
+ my $args = shift;
+ my $i = scalar @cacheAccount;
+ if ( exists $args->{'id'} and exists $args->{'account'} )
{
- $self->sth_insertBlockhash(
- $self->dbh->prepare(q(
- INSERT INTO blockhash
- ( hash )
- VALUES (?)
- ))
- )
+ # Don't put something in the cache, if it's already there
+ while ( $i-- > 0 )
+ {
+ if ( $cacheAccount[$i]->{'id'} eq $args->{'id'} )
+ {
+ return;
+ }
+ }
+ # Store in the cache
+ push @cacheAccount, { 'id' => $args->{'id'}, 'account' => $args->{'account'} };
+ # may 10 entries
+ splice @cacheAccount, 10;
+ return;
}
- $self->sth_insertBlockhash->execute( $hash );
+ # If just one element is given, then it is a lookup
+ if ( defined $args->{'id'} )
+ {
+ while ( $i-- > 0 )
+ {
+ return $cacheAccount[$i]->{'account'}
+ if $cacheAccount[$i]->{'id'} eq $args->{'id'};
+ }
+ return;
+ }
+ if ( defined $args->{'account'} )
+ {
+ while ( $i-- > 0 )
+ {
+ return $cacheAccount[$i]->{'id'}
+ if $cacheAccount[$i]->{'account'} eq $args->{'account'};
+ }
+ return;
+ }
+ return;
}
__PACKAGE__->mk_accessors( 'sth_selectAccountIdByAddress' );
my %account;
sub selectAccountIdByAddress
{
- my ( $self, $address )= @_;
+ my ( $self, $address ) = @_;
$address = lc $address;
if ( not defined $self->sth_selectAccountIdByAddress )
{
@@ -186,23 +319,18 @@ sub selectAccountIdByAddress
))
);
}
- return $account{$address} if exists $account{$address};
- while ( not exists $account{$address} )
- {
- # Try to look up the hash
- $self->sth_selectAccountIdByAddress->execute( $address );
- my $ref = $self->sth_selectAccountIdByAddress->fetchrow_arrayref;
- $self->sth_selectAccountIdByAddress->finish;
- # If found, return it immediately
- if ( defined $ref )
- {
- $account{$address} = $ref->[0];
- last;
- }
- # Insert a hash, because it is not already in the database.
- $self->insertAccount( $address );
- }
- return $account{$address};
+ my $id = _cacheAccount( {'address'=>$address} );
+ return $id if defined $id;
+ # Try to look up the hash
+ $self->sth_selectAccountIdByAddress->execute( $address );
+ my $ref = $self->sth_selectAccountIdByAddress->fetchrow_arrayref;
+ $self->sth_selectAccountIdByAddress->finish;
+ $id = defined $ref
+ ? $ref->[0]
+ : $self->insertAccount( $address );
+ die if not defined $id;
+ _cacheAccount( {'id'=>$id, 'address'=>$address} );
+ return $id;
}
__PACKAGE__->mk_accessors( 'sth_insertAccount' );
@@ -221,13 +349,14 @@ sub insertAccount
);
}
$self->sth_insertAccount->execute( $address );
+ return $self->dbh->last_insert_id(undef,undef,undef,undef,{sequence=>'account_id_seq'});
}
__PACKAGE__->mk_accessors( 'sth_insertBlock' );
sub insertBlock
{
my ( $self, $hash, $parentHash, $number, $timestamp, $difficulty, $gasUsed, $gasLimit, $size )
- = @_;
+ = @_;
if ( not defined $self->sth_insertBlock )
{
$self->sth_insertBlock(
@@ -239,16 +368,16 @@ sub insertBlock
))
);
}
- my $id = $self->selectBlockhashByHash( $hash );
- my $parentid = $self->selectBlockhashByHash( $parentHash );
+ my $id = $self->selectBlockhashByHash( $hash );
+ my $parentid = $self->selectBlockhashByHash( $parentHash );
$self->sth_insertBlock->execute( $id, $parentid, $number, $timestamp, $difficulty, $gasUsed, $gasLimit, $size );
}
__PACKAGE__->mk_accessors( 'sth_selectBlockByHash' );
sub selectBlockByHash
{
- my ( $self, $hash) = @_;
- $hash = lc $hash;
+ my ( $self, $hash) = @_;
+ $hash = lc $hash;
if ( not defined $self->sth_selectBlockByHash )
{
$self->sth_selectBlockByHash(
@@ -259,20 +388,19 @@ sub selectBlockByHash
))
);
}
- my $blockid = $self->selectBlockhashByHash( $hash );
+ my $blockid = $self->selectBlockhashByHash( $hash );
$self->sth_selectBlockByHash->execute( $blockid );
- my $ref = $self->sth_selectBlockByHash->fetchrow_hashref;
+ my $ref = $self->sth_selectBlockByHash->fetchrow_hashref;
$self->sth_selectBlockByHash->finish;
- return
- if not defined $ref;
- return $ref;
+ return $ref if defined $ref;
+ return;
}
__PACKAGE__->mk_accessors( 'sth_selectTransactionByHash' );
sub selectTransactionByHash
{
- my ( $self, $hash) = @_;
- $hash = lc $hash;
+ my ( $self, $hash) = @_;
+ $hash = lc $hash;
if ( not defined $self->sth_selectTransactionByHash )
{
$self->sth_selectTransactionByHash(
@@ -284,17 +412,17 @@ sub selectTransactionByHash
);
}
$self->sth_selectTransactionByHash->execute( $hash );
- my $ref = $self->sth_selectTransactionByHash->fetchrow_hashref;
+ my $ref = $self->sth_selectTransactionByHash->fetchrow_hashref;
$self->sth_selectTransactionByHash->finish;
- return $ref
- if defined $ref;
+ return $ref if defined $ref;
return;
}
__PACKAGE__->mk_accessors( 'sth_updateWhoSealed' );
sub updateWhoSealed
{
- my ( $self, $sealerhash, $blockhash ) = @_;
+ my ( $self, $sealerhash, $blockhash )
+ = @_;
if ( not defined $self->sth_updateWhoSealed )
{
$self->sth_updateWhoSealed(
@@ -306,15 +434,15 @@ sub updateWhoSealed
))
);
}
- my $blockid = $self->selectBlockhashByHash( $blockhash );
- my $sealerid = $self->selectAccountIdByAddress( $sealerhash );
+ my $blockid = $self->selectBlockhashByHash( $blockhash );
+ my $sealerid = $self->selectAccountIdByAddress( $sealerhash );
$self->sth_updateWhoSealed->execute( $sealerid, $blockid );
}
__PACKAGE__->mk_accessors( 'sth_selectmaxunknownsigned' );
sub selectMaxUnknownSigned
{
- my ( $self ) = @_;
+ my ( $self ) = @_;
if ( not defined $self->sth_selectmaxunknownsigned )
{
$self->sth_selectmaxunknownsigned(
@@ -330,12 +458,12 @@ sub selectMaxUnknownSigned
);
}
$self->sth_selectmaxunknownsigned->execute();
- my $ref = $self->sth_selectmaxunknownsigned->fetchrow_arrayref;
+ my $ref = $self->sth_selectmaxunknownsigned->fetchrow_arrayref;
$self->sth_selectmaxunknownsigned->finish();
- return if not defined $ref;
- my $blockid = $ref->[0];
- my $blockhash = $self->selectBlockhashById( $blockid );
- return $blockhash;
+ return if not defined $ref;
+ my $blockid = $ref->[0];
+ my $blockhash = $self->selectBlockhashById( $blockid );
+ return $blockhash;
}
1;
diff --git a/docker-compose.yml b/docker-compose.yml
index ec74c3e876ec06e4f231d5e39eeaf132e378612b..b46eaec5ca0da61c9e4d0be0c6474e1f63bbfec4 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -12,7 +12,7 @@ services:
build: postgres
restart: always
environment:
- POSTGRES_PASSWORD: onlythelonely
+ POSTGRES_PASSWORD: theswampthing
volumes:
- pgdata:/var/lib/postgresql/data
ports:
@@ -22,8 +22,10 @@ services:
build: collector
restart: always
environment:
- POSTGRES_PASSWORD: onlythelonely
+ POSTGRES_PASSWORD: theswampthing
BFANODE: http://200.108.146.200:8545/
+ volumes:
+ - ./collector:/home/bfa/collector
# adminer:
# image: adminer
diff --git a/postgres/10-postgres.sql b/postgres/10-postgres.sql
index 20fcab06cb209ef6d9f294a4cdd6ff68d95ef5c6..b58fe7ccbe6334eb6f4453564def524486c798ca 100644
--- a/postgres/10-postgres.sql
+++ b/postgres/10-postgres.sql
@@ -50,8 +50,8 @@ CREATE TABLE "public"."block" (
"sealerAccountId" bigint,
"timestamp" bigint NOT NULL,
"difficulty" smallint NOT NULL,
- "gasUsed" bigint NOT NULL,
- "gasLimit" bigint NOT NULL,
+ "gasUsed" numeric(79) NOT NULL,
+ "gasLimit" numeric(79) NOT NULL,
"size" bigint NOT NULL,
CONSTRAINT "block_id"
PRIMARY KEY ("id"),
@@ -90,23 +90,22 @@ CREATE TABLE "public"."transaction" (
NOT NULL,
"nonce" bigint
NOT NULL,
- "gas" bigint
+ "gas" numeric(79)
NOT NULL,
- "gasPrice" bigint
+ "gasPrice" numeric(79)
NOT NULL,
- "value" bigint
+ "value" numeric(79)
NOT NULL,
"fromAccountId" bigint
NOT NULL,
- "toAccountId" bigint
- NOT NULL,
+ "toAccountId" bigint,
"contractaddressAccountId"
bigint,
"status" smallint
NOT NULL,
- "gasUsed" bigint
+ "gasUsed" numeric(79)
NOT NULL,
- "inputlen" smallint
+ "inputlen" integer
NOT NULL,
"input" text
NOT NULL,
@@ -162,6 +161,7 @@ CREATE VIEW t
AS
SELECT transaction.hash "transaction hash",
block.number "block number",
+ block.timestamp "block timestamp",
"type",
nonce,
gas,
diff --git a/postgres/90-add-db-user.sh b/postgres/90-add-db-user.sh
index f734ed5ebf72951ab0d1b1ed88118f1d66cadab8..87657edc7fc72f5dfe6f2797db240b667e59ecef 100755
--- a/postgres/90-add-db-user.sh
+++ b/postgres/90-add-db-user.sh
@@ -1,8 +1,8 @@
#!/bin/sh
psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
- CREATE USER leer WITH PASSWORD 'bfa';
- GRANT CONNECT ON DATABASE postgres TO leer;
- GRANT SELECT ON ALL TABLES IN SCHEMA public TO leer;
- GRANT UPDATE (shortname,name) ON TABLE public.account TO leer;
+ CREATE USER dgsi WITH PASSWORD 'bfa';
+ GRANT CONNECT ON DATABASE postgres TO dgsi;
+ GRANT SELECT ON ALL TABLES IN SCHEMA public TO dgsi;
+ GRANT UPDATE (shortname,name) ON TABLE public.account TO dgsi;
EOSQL