diff --git a/collector/Dockerfile b/collector/Dockerfile
index a0b09ad3ce65026120dedb5a9853246e58331930..de9bd1e7cc4004418705f0b10d9ae3a2a07fea97 100644
--- a/collector/Dockerfile
+++ b/collector/Dockerfile
@@ -1,11 +1,8 @@
FROM debian
-RUN useradd --create-home --shell /bin/bash bfa && mkdir /home/bfa/collector
+RUN useradd --create-home --shell /bin/bash bfa && install --owner=bfa --directory /home/bfa/collector
RUN apt-get update && apt-get upgrade -y
-RUN apt-get install -y libjson-perl libnet-dns-perl libdbd-pg-perl libwww-perl libclass-accessor-perl
+RUN apt-get install -y libjson-perl libdbd-pg-perl libwww-perl libclass-accessor-perl less
WORKDIR /home/bfa
-COPY collector.pl sql.pm /home/bfa/collector/
-RUN chown -R bfa /home/bfa
USER bfa
-CMD /home/bfa/collector/collector.pl
-#CMD sleep 14d
-ENV BFANODE http://public.47525974938.bfa.martin-legene.dk:8545/
+COPY sql.pm collectorcommon.pm collector.pl txfetcher.pl /home/bfa/collector/
+CMD sleep 14d
diff --git a/collector/collector.pl b/collector/collector.pl
index 297cba143cdda2fefba510b09387db60f9f88f21..00609504d4ebd5d978df9c37829bae8e91cb9552 100755
--- a/collector/collector.pl
+++ b/collector/collector.pl
@@ -9,8 +9,6 @@ use Data::Dumper;
use Math::BigInt;
use LWP;
use JSON;
-use Net::DNS;
-use Net::DNS::Resolver;
use lib '/home/bfa/collector';
use lib 'collector';
use lib '.';
@@ -20,29 +18,32 @@ use sql;
my $netversion = Math::BigInt->new( "0xb10c4d39a" );
# 200941592
my $chainid = Math::BigInt->new( "0xbfa2018" );
-my $idcounter = 0;
-my $stop_request = 0;
-my $ua;
-my @blockqueue;
+my $maxinarow = 5000;
my %sealers;
+my $stop_request;
my $sql;
+sub stop
+{
+ $stop_request = 1;
+ $SIG{$_[0]} = 'DEFAULT';
+ print STDERR "\rStop request received ($_[0]).\n";
+}
+
sub info(@)
{
- return if not -t STDOUT;
+ #return if not -t STDOUT;
# if 1 param only
- unshift @_, '%s'
- if $#_ == 0;
+ unshift @_, '%s' if $#_ == 0;
my $format = shift;
- $format = "%s: " . $format;
- my $txt = sprintf $format, astime(time), @_;
- $txt .= "\n" if $txt !~ /\n$/;
- print $txt;
+ local $_ = sprintf $format, @_;
+ print;
+ print "\n" if not /\n$/;
}
sub astime
{
- my @t = localtime( $_[0] );
+ my @t = localtime( $_[0] );
$t[5] += 1900;
$t[4] += 1;
return sprintf '%04d%02d%02d-%02d%02d%02d', @t[5,4,3,2,1,0];
@@ -56,15 +57,11 @@ sub shorthash($)
return $_;
}
-sub stop
-{
- $stop_request = 1;
- print STDERR "\rStop request received ($_[0]).\n";
-}
-
+my $ua;
+my $idcounter = 0;
sub rpcreq
{
- my ( $opname, @params )= @_;
+ my ( $opname, @params ) = @_;
$ua = LWP::UserAgent->new( keep_alive => 10 )
if not defined $ua;
# $ua->ssl_opts( 'verify_hostname' => 0 );
@@ -73,11 +70,11 @@ sub rpcreq
method => $opname,
id => $idcounter++,
);
- my @args = map {
- my $v = $args{$_};
- $v = qq/"${v}"/ if $v !~ /^\d+$/;
- sprintf qq/"%s":%s/, $_, $v;
- } keys %args;
+ my @args = map {
+ my $v = $args{$_};
+ $v = qq/"${v}"/ if $v !~ /^\d+$/;
+ sprintf qq/"%s":%s/, $_, $v;
+ } keys %args;
if ( scalar @params )
{
my $p = '"params":[';
@@ -93,50 +90,69 @@ sub rpcreq
$p .= ']';
push @args, $p;
}
- my $args = '{' . join(',',@args) . '}';
+ my $args = '{' . join(',',@args) . '}';
die 'The environment variable $BFANODE must contain a URL for a ' .
- 'server to connect to (e.g. https://10.20.30.40/), but it it ' .
+ 'server to connect to (e.g. http://10.20.30.40/), but it it ' .
'not set. Stopped'
unless defined $ENV{'BFANODE'};
- my $res = $ua->post(
+ my $res = $ua->post(
$ENV{BFANODE},
'Content-Type' => 'application/json',
'Content' => $args,
);
die $res->status_line
unless $res->is_success;
- my $json = decode_json($res->content);
+ my $json = decode_json($res->content);
return $json->{'result'};
}
-sub getblock
+sub versioncheck
{
- my $block = lc shift;
- my $rpccmd;
- if ( $block =~ /^0x[\da-f]{64}$/ )
+ my $n;
+ my $ok = 1;
+ $n = Math::BigInt->new( rpcreq( "net_version" ) );
+ if ( $n->bcmp( $netversion ) != 0 )
{
- $rpccmd = 'eth_getBlockByHash';
+ warn "Network says it has net.version "
+ . $n->bstr
+ . " ("
+ . $n->as_hex
+ . "). Expected $netversion ("
+ . $netversion->as_hex
+ . ").\n";
+ $ok = 0;
}
- else
+ $n = Math::BigInt->new( rpcreq( "eth_chainId" ) );
+ if ( $n->bcmp( $chainid ) != 0 )
{
- $rpccmd = 'eth_getBlockByNumber';
- if ( $block =~ /^\d+$/ )
- {
- $block = Math::BigInt->new( $block )->as_hex;
- }
+ warn sprintf("Network says it has eth.chainId %s (%s). Expected %s (%s).\n", $n->bstr, $n->as_hex, $chainid->bstr, $chainid->as_hex);
+ $ok = 0;
}
- my $json = rpcreq( $rpccmd, $block, 'false' );
+ exit 1 if not $ok;
+}
+
+sub getblock
+{
+ my $block = lc shift;
+ return ( undef, undef ) if $block =~ /^0x0{64}$/;
+ my $rpccmd = $block =~ /^0x[\da-f]{64}$/
+ ? 'eth_getBlockByHash'
+ : 'eth_getBlockByNumber';
+ $block = Math::BigInt->new( $block )->as_hex
+ if $block =~ /^\d+$/;
+ my $json = rpcreq( $rpccmd, $block, 'true' );
die "We should have received some kind of JSON from our RPC call, " .
- "but apparently not. Stopped "
+ "but apparently we did not.\n" . Dumper( $rpccmd, $block ) . "Stopped "
if not defined $json;
- return if $sql->selectBlockByHash( $json->{'hash'} );
+ # Skip it if we already have it.
+ return ( undef, undef ) if $sql->selectBlockByHash( $json->{'hash'} );
foreach my $key (qw(
timestamp gasUsed gasLimit difficulty number totalDifficulty size
)) {
$json->{$key} = Math::BigInt->new( $json->{$key} )->bstr
if exists $json->{$key};
}
- info "B %s %s %s %s %s %s/%s",
+ info "B %s %s %s %s %s %s/%s",
shorthash $json->{'hash'},
astime($json->{'timestamp'}),
$json->{'number'},
@@ -154,207 +170,268 @@ sub getblock
$json->{'gasLimit'},
$json->{'size'}
);
- if ( scalar @{$json->{'transactions'}} )
+ my $txcount = 0;
+ foreach ( @{ $json->{'transactions'} } )
{
- foreach my $txhash ( @{ $json->{'transactions'} } )
- {
- getTransByHash( $txhash );
- }
+ getTransByHash( $_ );
+ $txcount++;
}
- # Put the parent block in front in @blockqueue
- unshift @blockqueue, lc $json->{'parentHash'}
+ my $nextblock = lc $json->{'parentHash'}
if $json->{'number'} != 0;
+ return ( $nextblock, $txcount );
}
sub getTransByHash
{
- my $hash = shift;
+ my $hash = shift;
# Don't get it via RPC if we already have it in the database
- return
- if $sql->selectTransactionByHash( $hash );
- my $json = rpcreq(
- 'eth_getTransactionByHash',
- $hash
- );
- die "Invalid object received when asking for $hash:\n".Dumper($json) if not exists $json->{'blockNumber'};
- my $max = 0;
- my $rcpt = undef;
- do {
- $rcpt = rpcreq(
- 'eth_getTransactionReceipt',
- $hash
- );
- if ( not $rcpt )
- {
- sleep 1;
- warn "Couldn't get the transaction receipt for $hash\n";
- }
- } while not $rcpt and $max++ < 10;
- die "Couldn't get the transaction receipt for $hash:\n" if not $rcpt;
+ my $json;
+ if ( ref $hash eq '' )
+ {
+ # Don't fetch it, if we already have it in the database.
+ return if $sql->selectTransactionByHash( $hash );
+ $json = rpcreq( 'eth_getTransactionByHash', $hash );
+ }
+ elsif ( ref $hash eq 'HASH' )
+ {
+ $json = $hash;
+ }
+ else
+ {
+ die "Don't know how to fetch transactions of type "
+ . ref($hash)
+ . ". Stopped";
+ }
+ die "Invalid object received when asking for $hash:\n"
+ . Dumper('hash'=>$hash,'json'=>$json)
+ if not exists $json->{'blockNumber'};
foreach my $key (qw( hash blockHash nonce gas gasPrice value from to input ))
{
die "JSON missing key $key:".Dumper($json) unless exists $json->{$key};
}
- $json->{'status'} = $rcpt->{'status'};
- $json->{'gasUsed'} = $rcpt->{'gasUsed'};
- $json->{'contractAddress'}
- = $rcpt->{'contractAddress'};
- foreach my $key (qw( nonce value gas gasPrice gasUsed status))
+ foreach my $key (qw( nonce value gas gasPrice ))
{
$json->{$key} = Math::BigInt->new( $json->{$key} )->bstr;
}
- info "T %s %s %s %s -> %s",
+ info "T %s %s %s %s -> %s",
shorthash $json->{'hash'},
$json->{'nonce'},
$json->{'value'},
shorthash $json->{'from'},
- shorthash ($json->{'to'}||'<NULL>');
+ shorthash $json->{'to'}||'<NULL>';
my $input = $json->{'input'};
- my $inputlen = 0;
- if ( $input =~ /^0x/ )
- {
- $inputlen = (length($input)-2) / 2;
- }
- my @args = (
+ my $inputlen = $input =~ /^0x/
+ ? (length($input)-2) / 2
+ : 0;
+ my $to = defined $json->{'to'}
+ ? lc $json->{'to'}
+ : undef;
+ $sql->insertTransaction(
lc $json->{'hash'},
lc $json->{'blockHash'},
- $json->{'nonce'},
- $json->{'gas'},
- $json->{'gasPrice'},
- $json->{'value'},
+ $json->{'nonce'},
+ $json->{'gas'},
+ $json->{'gasPrice'},
+ $json->{'value'},
lc $json->{'from'},
- defined $json->{'to'} ? lc $json->{'to'} : undef,
- $json->{'status'},
- $json->{'gasUsed'},
+ $to,
$input,
- $inputlen
+ $inputlen,
+ 'explicit'
);
- if ( $json->{'contractAddress'} )
- {
- $sql->insertTransactionWithContractAddress( @args, lc $json->{'contractAddress'} );
- }
- else
- {
- $sql->insertTransaction( @args );
- }
-}
-
-sub versioncheck
-{
- my $n;
- my $ok = 1;
- $n = Math::BigInt->new( rpcreq( "net_version" ) );
- if ( $n->bcmp( $netversion ) != 0 )
- {
- warn "Network says it has net.version "
- . $n->bstr
- . " ("
- . $n->as_hex
- . "). Expected $netversion ("
- . $netversion->as_hex
- . ").\n";
- $ok = 0;
- }
- $n = Math::BigInt->new( rpcreq( "eth_chainId" ) );
- if ( $n->bcmp( $chainid ) != 0 )
- {
- warn sprintf("Network says it has eth.chainId %s (%s). Expected %s (%s).\n", $n->bstr, $n->as_hex, $chainid->bstr, $chainid->as_hex);
- $ok = 0;
- }
- exit 1 if not $ok;
- return $ok;
+ getTransactionReceipt( $json->{'hash'} );
}
sub getsnap
{
my $blockhash = $_[0];
- my $json = rpcreq(
- 'clique_getSnapshotAtHash',
- $blockhash
- );
+ my $json = rpcreq( 'clique_getSnapshotAtHash', $blockhash );
# Sanity check 1 - does the lookup return the block?
- return if not defined $json;
+ die "Did not get a response from the node, when asking about \"$blockhash\". Stopped"
+ if not defined $json;
+ # 'recents' has a list of blocknumbers and the signer for that block number
my $recents = $json->{'recents'};
- # Blockhash of the requested block - we must use this for the insert
+ # Blockhash of the requested block - we must use this for the insert, and
+ # is the highest blocknumbered block in the snapshot.
$blockhash = $json->{'hash'};
my $blocknumber = Math::BigInt->new( $json->{'number'} );
+ my $updatecounter = 0;
my @infonumbers;
while ( not $blocknumber->is_zero )
{
- my $sealerhash = $recents->{$blocknumber->bstr};
+ # Now we fetch the address of the sealer in the recents list,
+ # index by the block number
+ my $sealerhash = $recents->{$blocknumber->bstr};
last if not defined $sealerhash;
- push @infonumbers, $blocknumber->bstr;
- my $ref = $sql->selectBlockByHash( $blockhash );
+ my $ref = $sql->selectBlockByHash( $blockhash );
# Stop if we get to blocks which we still haven't stored in the db
if ( not defined $ref )
{
- $blockhash = undef;
- last;
+ $blockhash = undef;
+ last;
}
# Stop if we got to a block with sealers.
if ( exists $ref->{'sealerAccountId'} and defined $ref->{'sealerAccountId'} )
{
- $blockhash = undef;
- last;
+ $blockhash = undef;
+ last;
}
+ push @infonumbers, $blocknumber->bstr;
+ $updatecounter++;
$sql->updateWhoSealed( $sealerhash, $blockhash );
my $parentblockid = $ref->{'parentBlockhashId'};
# For next block...
$blockhash = $sql->selectBlockhashById( $parentblockid );
$blocknumber->bsub(1);
}
- info "S Snapshot at block # @infonumbers."
+ info "S Snapshot at block # @infonumbers."
if @infonumbers;
- return ($blockhash, $blocknumber);
+ return ($blockhash, $blocknumber, $updatecounter);
}
# Find strands of blocks in the database which have their sealer set to NULL
sub allsnaps
{
- my $blockhash = $sql->selectMaxUnknownSigned;
- while ( defined $blockhash )
+ my $startblock = shift;
+ my $counter = 0;
+ my $extra = 0;
+ my $blockhash = (defined $startblock and $startblock =~ /^0x[\da-f]{64}$/)
+ ? $startblock
+ : $sql->selectMaxUnknownSigned;
+ while ( defined $blockhash and not $stop_request and $counter < 10000 )
{
my $blocknumber;
- ( $blockhash, $blocknumber )= getsnap( $blockhash );
- last if $blocknumber->is_zero;
+ ( $blockhash, $blocknumber,
+ $extra ) = getsnap( $blockhash );
+ last if not defined $blockhash;
+ $counter += $extra;
+ last if $blocknumber->is_zero;
}
+ return $counter;
+}
+
+sub getTransactionReceipt
+{
+ my $hash = lc shift;
+ my $json = rpcreq( 'eth_getTransactionReceipt', $hash );
+ return if not defined $json;
+ return if not exists $json->{'transactionHash'};
+ return if not exists $json->{'status'};
+ return if not exists $json->{'gasUsed'};
+ my $status = Math::BigInt->new( $json->{'status'} )->bstr;
+ my $gasUsed = Math::BigInt->new( $json->{'gasUsed'} )->bstr;
+ my $contractAddress = $json->{'contractAddress'};
+ $sql->updateRcptInfo( $hash, $status, $gasUsed, $contractAddress );
}
sub main
{
+ $| = 1;
versioncheck();
- # Give Postgres time to start up.
- sleep( 3 );
+ # Give postgres time to start up
+ # if we are run from a tty, expect that postgres is already running
+ sleep 3 if not -t STDIN;
$sql = sql->new();
- my $lastorphancheck= 0;
$SIG{'HUP'} = \&stop;
$SIG{'INT'} = \&stop;
$SIG{'TERM'} = \&stop;
- $SIG{'TSTP'} = \&stop;
+ my $lastmaintanance = 0;
while ( not $stop_request )
{
- my $now = time();
- if ( $lastorphancheck <= $now+600 )
+ my @blockqueue;
+ my $deletecounter = 0;
+ my $snapcounter = 0;
+ my $extrarcptcounter = 0;
+ my $now = time();
+ if ( $lastmaintanance+300 <= $now )
{
- $lastorphancheck = $now;
+ # This block contains some rather expensive queries
+ # for things which are not urgent to get fixed,
+ # So we only run them every few minutes.
+ $lastmaintanance = $now;
+ # We don't only check for orphans.. first we see if
+ # we can create some by deleting duplicate blockheights.
+ # If any duplicates are found, we will be deleting the
+ # uncle and the canonical block. The canonical will
+ # the be listed as parent to an orphans and recreated.
+ my @blocks = $sql->selectduplicateblocknumbers;
+ if ( @blocks )
+ {
+ info( "D Deleting duplicate blocks @blocks." );
+ # Delete the receipts for these blocks, so we can
+ # remove the blocks.
+ $sql->deletetransactionsinblocknumbers( @blocks );
+ # Now we can remove the duplicate blocks.
+ $sql->deleteblocknumbers( @blocks );
+ $deletecounter += scalar @blocks;
+ }
+ #
# Orphans happen if some blocks have been deleted
# or if the initial synchronization hasn't finished.
# This check should be done when orphans can be
# suspected, but when is that? Maybe every 10 minutes.
push @blockqueue, $sql->listOrphans();
+ # Look for transactions which hasn't had their receipts
+ # processed (not common that this should happen).
+ my @rcptqueue = $sql->selecttxwhichmissreceipts( $maxinarow );
+ while ( @rcptqueue and not $stop_request )
+ {
+ print "R Extra receipts: "
+ if $extrarcptcounter == 0;
+ $extrarcptcounter++;
+ print "r";
+ getTransactionReceipt( shift @rcptqueue );
+ }
+ print "\n" if $extrarcptcounter > 0;
+ # Find out who signed all blocks
+ my $n = 1;
+ while ( not $stop_request and $n > 0 )
+ {
+ $n = allsnaps();
+ $snapcounter += $n;
+ }
}
- do
- {
+ my $didsomething = 0;
+ do {
unshift @blockqueue, 'latest';
- my $maxinarow = 100;
- getblock( shift @blockqueue )
- while @blockqueue and --$maxinarow and not $stop_request;
- # Find out who signed all blocks
- allsnaps() if not $stop_request;
- info "Committing.";
- $sql->commit;
- } while @blockqueue and not $stop_request;
- sleep 5 if not @blockqueue and not $stop_request;
+ my $blockcounter = 0;
+ my $newtx = 0;;
+ while ( @blockqueue and $blockcounter <= 2500 and not $stop_request )
+ {
+ my $block = shift @blockqueue;
+ my ( $nextblock, $n )
+ = getblock( $block );
+ # getblock returns two undefs if the block wasn't processed
+ next if not defined $n;
+ $blockcounter++;
+ $newtx += $n;
+ $snapcounter += allsnaps( $block );
+ # Put the parent block in front in @blockqueue
+ unshift @blockqueue, $nextblock
+ if defined $nextblock;
+ }
+ $didsomething = $blockcounter + $snapcounter + $extrarcptcounter;
+ if ( $didsomething > 0 )
+ {
+ info
+ "C Committing %d Delete%s, %d Block%s, %d Transaction%s, %d Snapshot%s, %d extra Receipt%s.",
+ $deletecounter, $deletecounter==1?'':'s',
+ $blockcounter, $blockcounter==1?'':'s',
+ $newtx, $newtx==1?'':'s',
+ $snapcounter, $snapcounter==1?'':'s',
+ $extrarcptcounter, $extrarcptcounter==1?'':'s';
+ $sql->commit;
+ }
+ $extrarcptcounter = 0;
+ $snapcounter = 0;
+ $deletecounter = 0;
+ } while $didsomething > 0 and not $stop_request;
+ my $sleep = 3;
+ while ( not $stop_request and $sleep-- > 0 )
+ {
+ printf " %d \r", $sleep if -t STDIN;
+ sleep 1;
+ }
}
}
diff --git a/collector/sql.pm b/collector/sql.pm
index 204baa39e26a0bb0ae00d632d4f59dbc3fa5cf31..238cd8d2e81e8985fd8516a44d042c45153c167e 100644
--- a/collector/sql.pm
+++ b/collector/sql.pm
@@ -16,7 +16,7 @@ sub new
my $self = bless {}, ref $class || $class;
$self->dbh( DBI->connect(
'dbi:Pg:dbname=postgres;host=postgres','postgres',$ENV{'POSTGRES_PASSWORD'},
- {AutoCommit=>0,RaiseError=>1}
+ { AutoCommit=>0, RaiseError=>1 }
));
return $self;
}
@@ -93,28 +93,24 @@ sub maxBlockNumber
return $ref->[0];
}
-__PACKAGE__->mk_accessors( 'sth_insertTransactionWithContractAddress' );
-sub insertTransactionWithContractAddress
+__PACKAGE__->mk_accessors( 'sth_updateRcptInfo' );
+sub updateRcptInfo
{
my $self = shift;
- if ( not defined $self->sth_insertTransactionWithContractAddress )
+ if ( not defined $self->sth_updateRcptInfo )
{
- $self->sth_insertTransactionWithContractAddress(
+ $self->sth_updateRcptInfo(
$self->dbh->prepare(q(
- INSERT INTO transaction
- ( hash, "blockId", nonce, gas,
- "gasPrice", value, "fromAccountId","toAccountId",
- status, "gasUsed", input, inputlen,
- "contractaddressAccountId", "type" )
- VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)
+ UPDATE transaction
+ SET status=?, "gasUsed"=?, "contractaddressAccountId"=?
+ WHERE hash=?
))
);
}
- $_[1] = $self->selectBlockhashByHash( $_[1] );
- $_[6] = $self->selectAccountIdByAddress( $_[6] );
- $_[7] = $self->selectAccountIdByAddress( $_[7] ) if defined $_[7];
- $_[12] = $self->selectAccountIdByAddress( $_[12] ) if defined $_[12];
- $self->sth_insertTransactionWithContractAddress->execute( @_, 'explicit' );
+ $_[3] = $self->selectAccountIdByAddress( $_[3] ) if defined $_[3];
+ # hash is in the front but goes in the back in SQL
+ push @_, shift;
+ $self->sth_updateRcptInfo->execute( @_ );
}
__PACKAGE__->mk_accessors( 'sth_insertTransaction' );
@@ -128,16 +124,15 @@ sub insertTransaction
INSERT INTO transaction
( hash, "blockId", nonce, gas,
"gasPrice", value, "fromAccountId","toAccountId",
- status, "gasUsed", input, inputlen,
- "type" )
- VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)
+ input, inputlen, "type" )
+ VALUES (?,?,?,?,?,?,?,?,?,?,?)
))
);
}
- $_[1] = $self->selectBlockhashByHash( $_[1] );
+ $_[1] = $self->selectBlockhashByHash( $_[1] );
$_[6] = $self->selectAccountIdByAddress( $_[6] );
$_[7] = $self->selectAccountIdByAddress( $_[7] ) if defined $_[7];
- $self->sth_insertTransaction->execute( @_, 'explicit' );
+ $self->sth_insertTransaction->execute( @_ );
}
my @cacheBlockhash;
@@ -405,7 +400,7 @@ sub selectTransactionByHash
{
$self->sth_selectTransactionByHash(
$self->dbh->prepare(q(
- SELECT 1
+ SELECT "blockId", "fromAccountId", nonce
FROM transaction
WHERE hash=?
))
@@ -466,4 +461,99 @@ sub selectMaxUnknownSigned
return $blockhash;
}
+__PACKAGE__->mk_accessors( 'sth_selecttxwhichmissreceipts' );
+sub selecttxwhichmissreceipts
+{
+ my ( $self, $maxinarow ) = @_;
+ if ( not defined $self->sth_selecttxwhichmissreceipts )
+ {
+ $self->sth_selecttxwhichmissreceipts(
+ $self->dbh->prepare(q(
+ SELECT hash, "type"
+ FROM transaction
+ WHERE "gasUsed" IS NULL
+ ORDER BY "toAccountId" desc,
+ nonce
+ ))
+ );
+ }
+ my @txhashes;
+ $self->sth_selecttxwhichmissreceipts->execute();
+ while ( my $ref = $self->sth_selecttxwhichmissreceipts->fetchrow_arrayref and $maxinarow-- != 0 )
+ {
+ next if $ref->[1] eq 'external';
+ push @txhashes, $ref->[0];
+ }
+ $self->sth_selecttxwhichmissreceipts->finish();
+ return @txhashes;
+}
+
+__PACKAGE__->mk_accessors( 'sth_selectduplicateblocknumbers' );
+sub selectduplicateblocknumbers
+{
+ my ( $self ) = shift;
+ if ( not defined $self->sth_selectduplicateblocknumbers )
+ {
+ $self->sth_selectduplicateblocknumbers(
+ $self->dbh->prepare(q(
+ SELECT number
+ FROM block
+ WHERE number < (
+ SELECT MAX(number)
+ FROM block)-30
+ GROUP BY number
+ HAVING COUNT(number) > 1
+ ))
+ );
+ }
+ my @blocknumbers;
+ $self->sth_selectduplicateblocknumbers->execute();
+ while ( my $ref = $self->sth_selectduplicateblocknumbers->fetchrow_arrayref )
+ {
+ push @blocknumbers, $ref->[0];
+ }
+ $self->sth_selectduplicateblocknumbers->finish();
+ return @blocknumbers;
+}
+
+__PACKAGE__->mk_accessors( 'sth_deletetransactionsinblocknumbers' );
+sub deletetransactionsinblocknumbers
+{
+ my ( $self ) = shift;
+ if ( not defined $self->sth_deletetransactionsinblocknumbers )
+ {
+ $self->sth_deletetransactionsinblocknumbers(
+ $self->dbh->prepare(q(
+ DELETE FROM transaction
+ WHERE "blockId" IN (
+ SELECT id
+ FROM block
+ WHERE number = ?
+ )
+ ))
+ );
+ }
+ $self->sth_deletetransactionsinblocknumbers->execute($_)
+ foreach @_;
+ $self->sth_deletetransactionsinblocknumbers->finish();
+}
+
+__PACKAGE__->mk_accessors( 'sth_deleteblocknumber' );
+sub deleteblocknumbers
+{
+ my ( $self ) = shift;
+ if ( not defined $self->sth_deleteblocknumber )
+ {
+ $self->sth_deleteblocknumber(
+ $self->dbh->prepare(q(
+ DELETE FROM block
+ WHERE number = ?
+ ))
+ );
+ }
+ $self->sth_deleteblocknumber->execute($_)
+ foreach @_;
+ $self->sth_deleteblocknumber->finish();
+}
+
1;
diff --git a/docker-compose.yml b/docker-compose.yml
index b46eaec5ca0da61c9e4d0be0c6474e1f63bbfec4..ae3259f50dc057e47e74a03a82c9e28f4e53a527 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,5 +1,4 @@
-
## POSTGRES SUPERUSER user name is postgres
## POSTGRES SUPERUSER password is onlythelonely
## POSTGRES USER leer has password bfa defined in postgres/90-add-db-user.sh
@@ -20,10 +19,13 @@ services:
collector:
build: collector
+ #command: "collector/colletor.pl"
restart: always
+ depends_on:
+ - postgres
environment:
POSTGRES_PASSWORD: theswampthing
- BFANODE: http://200.108.146.200:8545/
+ BFANODE: http://public.47525974938.bfa.martin-legene.dk:8545/
volumes:
- ./collector:/home/bfa/collector
diff --git a/postgres/10-postgres.sql b/postgres/10-postgres.sql
index b58fe7ccbe6334eb6f4453564def524486c798ca..49142ed9e8fa410131b06df3b7fc6e04b5f762bd 100644
--- a/postgres/10-postgres.sql
+++ b/postgres/10-postgres.sql
@@ -101,10 +101,8 @@ CREATE TABLE "public"."transaction" (
"toAccountId" bigint,
"contractaddressAccountId"
bigint,
- "status" smallint
- NOT NULL,
- "gasUsed" numeric(79)
- NOT NULL,
+ "status" smallint,
+ "gasUsed" numeric(79),
"inputlen" integer
NOT NULL,
"input" text
@@ -126,6 +124,9 @@ CREATE TABLE "public"."transaction" (
REFERENCES block(id)
NOT DEFERRABLE
) WITH (oids = false);
+CREATE INDEX "transaction_hash"
+ ON "public"."transaction"
+ USING btree ("hash");
CREATE INDEX "transaction_contractaddressAccountId"
ON "public"."transaction"
USING btree ("contractaddressAccountId");
@@ -138,6 +139,9 @@ CREATE INDEX "transaction_toAccountId"
CREATE INDEX "transaction_blockId"
ON "public"."transaction"
USING btree ("blockId");
+CREATE INDEX "transaction_gasUsed"
+ ON "public"."transaction"
+ USING btree ("gasUsed");
CREATE VIEW b
AS
@@ -167,9 +171,9 @@ CREATE VIEW t
gas,
"gasPrice" "gas price",
value,
- a1.address "from",
- a2.address "to",
- a3.address "contractaddress",
+ coalesce(a1.shortname, a1.name, a1.address) "from",
+ coalesce(a2.shortname, a2.name, a2.address) "to",
+ coalesce(a3.shortname, a3.name, a3.address) "contractaddress",
status,
transaction."gasUsed" "gas used",
inputlen
@@ -184,33 +188,46 @@ CREATE VIEW t
AND a2.id = "toAccountId";
INSERT INTO "account" ("id", "address", "shortname", "name") VALUES
-(1, '0x377ab0cd00744dbb07b369cd5c0872dcd362c8f0', 'UNER', 'Universidad Nacional de Entre Rios'),
-(2, '0x2feb6a8876bd9e2116b47834b977506a08ea77bd', 'PNA', 'Prefectura Nacional Argentina'),
+(1, '0x377ab0cd00744dbb07b369cd5c0872dcd362c8f0', 'UNER', 'Universidad Nacional de Entre Rios'),
+(2, '0x2feb6a8876bd9e2116b47834b977506a08ea77bd', 'PNA', 'Prefectura Nacional Argentina'),
(3, '0x354779914a94ad428d2b53ae96cce3010bb0ce1e', 'RedLink', 'RedLink SA'),
-(4, '0x998c2651db6f76ca568c0071667d265bcc1b1e98', NULL, 'ASI'),
-(5, '0xd1f17aa41354d58940c300ffd79a200944dda2df', NULL, 'Marandu'),
+(4, '0x998c2651db6f76ca568c0071667d265bcc1b1e98', NULL, 'ASI'),
+(5, '0xd1f17aa41354d58940c300ffd79a200944dda2df', NULL, 'Marandu'),
(6, '0x39170a1ce03729d141dfaf8077c08b72c9cfdd0c', 'IXPBB', 'IXP Bahia Blanca'),
(7, '0x02665f10cb7b93b4491ac9594d188ef2973c310a', 'CABASE-MZA', 'CABASE Mendoza'),
-(8, '0x19fe7b9b3a1bebde77c5374c8e13c623e3d1b5b2', 'ARIU', 'Asociación Redes de Interconexión Universitaria'),
+(8, '0x19fe7b9b3a1bebde77c5374c8e13c623e3d1b5b2', 'ARIU', 'Asociación Redes de Interconexión Universitaria'),
(9, '0xe70fbc9d6be2fe509e4de7317637d8ee83d4f13c', 'CABASE-PMY', 'CABASE Puerto Madryn'),
-(10, '0xe191ac3108cb2c5d70d0e978876c048d4ba41b03', 'ANSV', 'Agencia Nacional de Seguridad Vial'),
-(11, '0xf36475eb25ba0c825455f150b26e24ab9449a443', 'SRT', 'Superintendencia de Riesgos del Trabajo'),
+(10, '0xe191ac3108cb2c5d70d0e978876c048d4ba41b03', 'ANSV', 'Agencia Nacional de Seguridad Vial'),
+(11, '0xf36475eb25ba0c825455f150b26e24ab9449a443', 'SRT', 'Superintendencia de Riesgos del Trabajo'),
(12, '0xd1420aa9dd092f50f68913e9e53b378a68e76ed7', 'SMGP/OPTIC', 'SecretarÃa de Modernización de la Gestión Pública / Oficina Provincial de TecnologÃas de la Información y la Comunicación - Gobierno de la Provincia del Neuquén'),
-(13, '0x2388d2cdb2cd6e7722b4af39c3bb406dd31f560e', 'UNR', 'Universidad Nacional de Rosario'),
+(13, '0x2388d2cdb2cd6e7722b4af39c3bb406dd31f560e', 'UNR', 'Universidad Nacional de Rosario'),
(14, '0x342e1d075d820ed3f9d9a05967ec4055ab23fa1e', 'CABASE', 'CABASE CABA'),
(15, '0xb3d1209aefbe00c78b2247656e2ddfa9e3897526', 'Colescriba', 'Colegio de Escribanos de la Provincia de Buenos Aires'),
-(16, '0xa14152753515674ae47453bea5e155a20c4ebabc', 'UP', 'Universidad de Palermo'),
-(17, '0x97a47d718eab9d660b10de08ef42bd7fd915b783', 'UNLP', 'Universidad Nacional de La Plata'),
-(18, '0x850b30dc584b39275a7ddcaf74a5c0e211523a30', 'UM', 'Ultima Milla'),
-(19, '0x609043ebde4a06bd28a1de238848e8f82cca9c23', 'UNSJ', 'Universidad Nacional de San Juan'),
-(20, '0xb43b53af0db2c3fac788195f4b4dcf2b3d72aa44', NULL, 'IPlan'),
-(21, '0x46991ada2a2544468eb3673524641bf293f23ccc', 'UNC', 'Universidad Nacional de Cordoba'),
-(22, '0x401d7a8432caa1025d5f093276cc6ec957b87c00', 'ONTI', 'Oficina Nacional de Tecnologias de Informacion'),
-(23, '0x91c055c6478bd0ad6d19bcb58f5e7ca7b04e67f1', 'DGSI', 'Dirección General de Sistemas Informáticos'),
+(16, '0xa14152753515674ae47453bea5e155a20c4ebabc', 'UP', 'Universidad de Palermo'),
+(17, '0x97a47d718eab9d660b10de08ef42bd7fd915b783', 'UNLP', 'Universidad Nacional de La Plata'),
+(18, '0x850b30dc584b39275a7ddcaf74a5c0e211523a30', 'UM', 'Ultima Milla'),
+(19, '0x609043ebde4a06bd28a1de238848e8f82cca9c23', 'UNSJ', 'Universidad Nacional de San Juan'),
+(20, '0xb43b53af0db2c3fac788195f4b4dcf2b3d72aa44', NULL, 'IPlan'),
+(21, '0x46991ada2a2544468eb3673524641bf293f23ccc', 'UNC', 'Universidad Nacional de Cordoba'),
+(22, '0x401d7a8432caa1025d5f093276cc6ec957b87c00', 'ONTI', 'Oficina Nacional de Tecnologias de Informacion'),
+(23, '0x91c055c6478bd0ad6d19bcb58f5e7ca7b04e67f1', 'DGSI', 'Dirección General de Sistemas Informáticos'),
(24, '0x52f8a89484947cd29903b6f52ec6beda69965e38', 'CABASE-PSS', 'CABASE Posadas'),
(25, '0x9b3ac6719b02ec7bb4820ae178d31c0bbda3a4e0', 'Everis', 'Everis'),
-(26, '0x99d6c9fca2a61d4ecdeb403515eb8508dc560c6b', NULL, NULL),
-(27, '0xc0310a7b3b25f49b11b901a667208a3eda8d7ceb', '', 'SyT'),
-(28, '0xabeff859aa6b0fb206d840dbf19de970065d4437', NULL, 'Belatrix');
+(26, '0x99d6c9fca2a61d4ecdeb403515eb8508dc560c6b', NULL, 'Ultima Milla 0x99 anterior'),
+(27, '0xc0310a7b3b25f49b11b901a667208a3eda8d7ceb', NULL, 'SyT'),
+(28, '0xabeff859aa6b0fb206d840dbf19de970065d4437', NULL, 'Belatrix');
+(401, '0xbfa02a9639318f5ed1291e2cee387aa9f9d68f98', NULL, 'BFA vault 01'),
+(402, '0xbfa1c42c7381a4ad32e13ce3263b639a0e0488f2', NULL, 'BFA vault 02'),
+(403, '0xbfa2c97c3f59cc929e8feb1aee2aca0b38235d18', NULL, 'BFA vault 03'),
+(404, '0xbfa3eb31f6526a5b29ae2302508bb6b1400d1fd1', NULL, 'BFA vault 04'),
+(405, '0xbfa846ddd1fb18af693f24d316c12d8e88f4b79f', NULL, 'BFA vault 05'),
+(406, '0xbfa8b1acfb51da0975274c6ebd46704c6c670a07', NULL, 'BFA vault 06'),
+(407, '0xbfab583022c5c18fec70965d63985b86f96c5657', NULL, 'BFA vault 07'),
+(408, '0xbfae9512c1cf4d9ce549333725c02ee8b3e5f049', NULL, 'BFA vault 08'),
+(500, '0xd15dd6dbbe722b451156a013c01b29d91e23c3d6', 'dist1-mgr', 'admin de destileria1'),
+(501, '0xecb6aff6e38dc58c4d9aae2f7927a282bcb77ac2', 'SC dist1', 'SC destileria1'),
+(502, '0xbf1ce9cca7dedea3fdb22d169b49643664602ee1', 'DGSI-dist2', 'Destileria de DGSI'),
+(503, '0xc3cf96af51d3c41dfbe428de75a8e5597d4d7a7b', 'SC-dist2', 'SC destileria de DGSI'),
+
-- 2020-07-13 05:48:47.242334+00