diff --git a/collector/Dockerfile b/collector/Dockerfile
index de9bd1e7cc4004418705f0b10d9ae3a2a07fea97..a391627828d9f40feeb3c9576141cbbd98f15d34 100644
--- a/collector/Dockerfile
+++ b/collector/Dockerfile
@@ -1,8 +1,8 @@
FROM debian
RUN useradd --create-home --shell /bin/bash bfa && install --owner=bfa --directory /home/bfa/collector
-RUN apt-get update && apt-get upgrade -y
-RUN apt-get install -y libjson-perl libdbd-pg-perl libwww-perl libclass-accessor-perl less
+RUN apt-get update && apt-get upgrade -y
+RUN apt-get install -y libjson-perl libdbd-pg-perl libwww-perl libclass-accessor-perl less
WORKDIR /home/bfa
USER bfa
-COPY sql.pm collectorcommon.pm collector.pl txfetcher.pl /home/bfa/collector/
-CMD sleep 14d
+COPY sql.pm collector.pl /home/bfa/collector/
+CMD collector/collector.pl
diff --git a/collector/collector.pl b/collector/collector.pl
index 00609504d4ebd5d978df9c37829bae8e91cb9552..a1cc48cc8738f4d71ffb1847704178716b22e51b 100755
--- a/collector/collector.pl
+++ b/collector/collector.pl
@@ -49,6 +49,13 @@ sub astime
return sprintf '%04d%02d%02d-%02d%02d%02d', @t[5,4,3,2,1,0];
}
+sub plural($)
+{
+ local $_ = $_[0];
+ return '' if $_[0] =~ /^\d+$/ and $_[0] == 1;
+ return 's';
+}
+
sub shorthash($)
{
local $_ = $_[0];
@@ -257,12 +264,13 @@ sub getsnap
my $blocknumber = Math::BigInt->new( $json->{'number'} );
my $updatecounter = 0;
my @infonumbers;
- while ( not $blocknumber->is_zero )
+ while ( not $blocknumber->is_zero and exists $recents->{$blocknumber->bstr} )
{
# Now we fetch the address of the sealer in the recents list,
# index by the block number
my $sealerhash = $recents->{$blocknumber->bstr};
last if not defined $sealerhash;
+ # See if we have that blocks sealer in the database
my $ref = $sql->selectBlockByHash( $blockhash );
# Stop if we get to blocks which we still haven't stored in the db
if ( not defined $ref )
@@ -270,7 +278,7 @@ sub getsnap
$blockhash = undef;
last;
}
- # Stop if we got to a block with sealers.
+ # Stop if we got to a block where we already know who sealed it.
if ( exists $ref->{'sealerAccountId'} and defined $ref->{'sealerAccountId'} )
{
$blockhash = undef;
@@ -298,19 +306,19 @@ sub allsnaps
my $blockhash = (defined $startblock and $startblock =~ /^0x[\da-f]{64}$/)
? $startblock
: $sql->selectMaxUnknownSigned;
- while ( defined $blockhash and not $stop_request and $counter < 10000 )
+ while ( defined $blockhash and not $stop_request and $counter < 50000 )
{
my $blocknumber;
( $blockhash, $blocknumber,
$extra ) = getsnap( $blockhash );
- last if not defined $blockhash;
$counter += $extra;
+ last if not defined $blockhash;
last if $blocknumber->is_zero;
}
return $counter;
}
-sub getTransactionReceipt
+sub getTransactionReceipt
{
my $hash = lc shift;
my $json = rpcreq( 'eth_getTransactionReceipt', $hash );
@@ -324,6 +332,41 @@ sub getTransactionReceipt
$sql->updateRcptInfo( $hash, $status, $gasUsed, $contractAddress );
}
+sub deleteDuplicates()
+{
+ # We don't only check for orphans.. first we see if
+ # we can create some by deleting duplicate blockheights.
+ # If any duplicates are found, we will be deleting the
+ # uncle and the canonical block. The canonical will
+ # the be listed as parent to an orphans and recreated.
+ info "Asking for duplicates to delete.";
+ my @blocks = $sql->selectduplicateblocknumbers;
+ my $counter = scalar @blocks;
+ return 0 if $counter == 0;
+ info "D Deleting duplicate blocks @blocks.";
+ # Delete the receipts for these blocks, so we can
+ # remove the blocks.
+ $sql->deletetransactionsinblocknumbers( @blocks );
+ # Now we can remove the duplicate blocks.
+ $sql->deleteblocknumbers( @blocks );
+ return $counter;
+}
+
+sub missingReceipts()
+{
+ my @rcpts = $sql->selecttxwhichmissreceipts( $maxinarow );
+ my $counter = scalar @rcpts;
+ return 0 if $counter == 0;
+ print "R Extra receipts: ";
+ while ( @rcpts and not $stop_request )
+ {
+ print "r";
+ getTransactionReceipt( shift @rcpts );
+ }
+ print "\n";
+ return $counter;
+}
+
sub main
{
$| = 1;
@@ -354,17 +397,7 @@ sub main
# If any duplicates are found, we will be deleting the
# uncle and the canonical block. The canonical will
# the be listed as parent to an orphans and recreated.
- my @blocks = $sql->selectduplicateblocknumbers;
- if ( @blocks )
- {
- info( "D Deleting duplicate blocks @blocks." );
- # Delete the receipts for these blocks, so we can
- # remove the blocks.
- $sql->deletetransactionsinblocknumbers( @blocks );
- # Now we can remove the duplicate blocks.
- $sql->deleteblocknumbers( @blocks );
- $deletecounter += scalar @blocks;
- }
+ $deletecounter = deleteDuplicates();
#
# Orphans happen if some blocks have been deleted
# or if the initial synchronization hasn't finished.
@@ -373,30 +406,14 @@ sub main
push @blockqueue, $sql->listOrphans();
# Look for transactions which hasn't had their receipts
# processed (not common that this should happen).
- my @rcptqueue = $sql->selecttxwhichmissreceipts( $maxinarow );
- while ( @rcptqueue and not $stop_request )
- {
- print "R Extra receipts: "
- if $extrarcptcounter == 0;
- $extrarcptcounter++;
- print "r";
- getTransactionReceipt( shift @rcptqueue );
- }
- print "\n" if $extrarcptcounter > 0;
- # Find out who signed all blocks
- my $n = 1;
- while ( not $stop_request and $n > 0 )
- {
- $n = allsnaps();
- $snapcounter += $n;
- }
+ $extrarcptcounter = missingReceipts();
}
my $didsomething = 0;
do {
unshift @blockqueue, 'latest';
my $blockcounter = 0;
- my $newtx = 0;;
- while ( @blockqueue and $blockcounter <= 2500 and not $stop_request )
+ my $newtx = 0;
+ while ( @blockqueue and $blockcounter < 2500 and not $stop_request )
{
my $block = shift @blockqueue;
my ( $nextblock, $n )
@@ -405,21 +422,27 @@ sub main
next if not defined $n;
$blockcounter++;
$newtx += $n;
- $snapcounter += allsnaps( $block );
# Put the parent block in front in @blockqueue
unshift @blockqueue, $nextblock
if defined $nextblock;
}
+ my $n = 1;
+ # Find out who signed all blocks
+ while ( not $stop_request and $n > 0 and $snapcounter < 5000 )
+ {
+ $n = allsnaps();
+ $snapcounter += $n;
+ }
$didsomething = $blockcounter + $snapcounter + $extrarcptcounter;
if ( $didsomething > 0 )
{
info
"C Committing %d Delete%s, %d Block%s, %d Transaction%s, %d Snapshot%s, %d extra Receipt%s.",
- $deletecounter, $deletecounter==1?'':'s',
- $blockcounter, $blockcounter==1?'':'s',
- $newtx, $newtx==1?'':'s',
- $snapcounter, $snapcounter==1?'':'s',
- $extrarcptcounter, $extrarcptcounter==1?'':'s';
+ $deletecounter, plural( $deletecounter ),
+ $blockcounter, plural( $blockcounter ),
+ $newtx, plural( $newtx ),
+ $snapcounter, plural( $snapcounter ),
+ $extrarcptcounter, plural( $extrarcptcounter );
$sql->commit;
}
$extrarcptcounter = 0;
diff --git a/docker-compose.yml b/docker-compose.yml
index ae3259f50dc057e47e74a03a82c9e28f4e53a527..e250889f97f2d6c4eb65eb2ba7cac58bc49e6983 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -16,18 +16,44 @@ services:
- pgdata:/var/lib/postgresql/data
ports:
- 5432:5432
+ logging:
+ driver: "json-file"
+ options:
+ max-file: "5"
+ max-size: "10m"
collector:
build: collector
- #command: "collector/colletor.pl"
restart: always
depends_on:
- postgres
+ - geth
environment:
POSTGRES_PASSWORD: theswampthing
- BFANODE: http://public.47525974938.bfa.martin-legene.dk:8545/
+ BFANODE: http://geth:8545/
volumes:
- ./collector:/home/bfa/collector
+ logging:
+ driver: "json-file"
+ options:
+ max-file: "5"
+ max-size: "10m"
+
+ geth:
+ #image: bfaar/nodo
+ build: geth
+ restart: always
+ ports:
+ - 8545:8545
+ - 8546:8546
+ volumes:
+ - geth:/home/bfa/bfa
+ logging:
+ driver: "json-file"
+ options:
+ max-file: "5"
+ max-size: "10m"
+ stop_grace_period: 300s
# adminer:
# image: adminer
@@ -37,3 +63,4 @@ services:
volumes:
pgdata:
+ geth:
diff --git a/postgres/10-postgres.sql b/postgres/10-postgres.sql
index 49142ed9e8fa410131b06df3b7fc6e04b5f762bd..a29279ea57d69824c93a0152842371025db86f1b 100644
--- a/postgres/10-postgres.sql
+++ b/postgres/10-postgres.sql
@@ -215,7 +215,7 @@ INSERT INTO "account" ("id", "address", "shortname", "name") VALUES
(25, '0x9b3ac6719b02ec7bb4820ae178d31c0bbda3a4e0', 'Everis', 'Everis'),
(26, '0x99d6c9fca2a61d4ecdeb403515eb8508dc560c6b', NULL, 'Ultima Milla 0x99 anterior'),
(27, '0xc0310a7b3b25f49b11b901a667208a3eda8d7ceb', NULL, 'SyT'),
-(28, '0xabeff859aa6b0fb206d840dbf19de970065d4437', NULL, 'Belatrix');
+(28, '0xabeff859aa6b0fb206d840dbf19de970065d4437', NULL, 'Belatrix'),
(401, '0xbfa02a9639318f5ed1291e2cee387aa9f9d68f98', NULL, 'BFA vault 01'),
(402, '0xbfa1c42c7381a4ad32e13ce3263b639a0e0488f2', NULL, 'BFA vault 02'),
(403, '0xbfa2c97c3f59cc929e8feb1aee2aca0b38235d18', NULL, 'BFA vault 03'),
@@ -227,7 +227,7 @@ INSERT INTO "account" ("id", "address", "shortname", "name") VALUES
(500, '0xd15dd6dbbe722b451156a013c01b29d91e23c3d6', 'dist1-mgr', 'admin de destileria1'),
(501, '0xecb6aff6e38dc58c4d9aae2f7927a282bcb77ac2', 'SC dist1', 'SC destileria1'),
(502, '0xbf1ce9cca7dedea3fdb22d169b49643664602ee1', 'DGSI-dist2', 'Destileria de DGSI'),
-(503, '0xc3cf96af51d3c41dfbe428de75a8e5597d4d7a7b', 'SC-dist2', 'SC destileria de DGSI'),
+(503, '0xc3cf96af51d3c41dfbe428de75a8e5597d4d7a7b', 'SC-dist2', 'SC destileria de DGSI');
-- 2020-07-13 05:48:47.242334+00