Skip to content
Snippets Groups Projects
Commit 6d97db99 authored by Robert Martin-Legene's avatar Robert Martin-Legene
Browse files

Optimizaciones y mejoras

parent 11635abe
No related branches found
No related tags found
No related merge requests found
FROM debian FROM debian
RUN useradd --create-home --shell /bin/bash bfa && install --owner=bfa --directory /home/bfa/collector RUN useradd --create-home --shell /bin/bash bfa && install --owner=bfa --directory /home/bfa/collector
RUN apt-get update && apt-get upgrade -y RUN apt-get update && apt-get upgrade -y
RUN apt-get install -y libjson-perl libdbd-pg-perl libwww-perl libclass-accessor-perl less RUN apt-get install -y libjson-perl libdbd-pg-perl libwww-perl libclass-accessor-perl less
WORKDIR /home/bfa WORKDIR /home/bfa
USER bfa USER bfa
COPY sql.pm collectorcommon.pm collector.pl txfetcher.pl /home/bfa/collector/ COPY sql.pm collector.pl /home/bfa/collector/
CMD sleep 14d CMD collector/collector.pl
...@@ -49,6 +49,13 @@ sub astime ...@@ -49,6 +49,13 @@ sub astime
return sprintf '%04d%02d%02d-%02d%02d%02d', @t[5,4,3,2,1,0]; return sprintf '%04d%02d%02d-%02d%02d%02d', @t[5,4,3,2,1,0];
} }
sub plural($)
{
local $_ = $_[0];
return '' if $_[0] =~ /^\d+$/ and $_[0] == 1;
return 's';
}
sub shorthash($) sub shorthash($)
{ {
local $_ = $_[0]; local $_ = $_[0];
...@@ -257,12 +264,13 @@ sub getsnap ...@@ -257,12 +264,13 @@ sub getsnap
my $blocknumber = Math::BigInt->new( $json->{'number'} ); my $blocknumber = Math::BigInt->new( $json->{'number'} );
my $updatecounter = 0; my $updatecounter = 0;
my @infonumbers; my @infonumbers;
while ( not $blocknumber->is_zero ) while ( not $blocknumber->is_zero and exists $recents->{$blocknumber->bstr} )
{ {
# Now we fetch the address of the sealer in the recents list, # Now we fetch the address of the sealer in the recents list,
# index by the block number # index by the block number
my $sealerhash = $recents->{$blocknumber->bstr}; my $sealerhash = $recents->{$blocknumber->bstr};
last if not defined $sealerhash; last if not defined $sealerhash;
# See if we have that blocks sealer in the database
my $ref = $sql->selectBlockByHash( $blockhash ); my $ref = $sql->selectBlockByHash( $blockhash );
# Stop if we get to blocks which we still haven't stored in the db # Stop if we get to blocks which we still haven't stored in the db
if ( not defined $ref ) if ( not defined $ref )
...@@ -270,7 +278,7 @@ sub getsnap ...@@ -270,7 +278,7 @@ sub getsnap
$blockhash = undef; $blockhash = undef;
last; last;
} }
# Stop if we got to a block with sealers. # Stop if we got to a block where we already know who sealed it.
if ( exists $ref->{'sealerAccountId'} and defined $ref->{'sealerAccountId'} ) if ( exists $ref->{'sealerAccountId'} and defined $ref->{'sealerAccountId'} )
{ {
$blockhash = undef; $blockhash = undef;
...@@ -298,19 +306,19 @@ sub allsnaps ...@@ -298,19 +306,19 @@ sub allsnaps
my $blockhash = (defined $startblock and $startblock =~ /^0x[\da-f]{64}$/) my $blockhash = (defined $startblock and $startblock =~ /^0x[\da-f]{64}$/)
? $startblock ? $startblock
: $sql->selectMaxUnknownSigned; : $sql->selectMaxUnknownSigned;
while ( defined $blockhash and not $stop_request and $counter < 10000 ) while ( defined $blockhash and not $stop_request and $counter < 50000 )
{ {
my $blocknumber; my $blocknumber;
( $blockhash, $blocknumber, ( $blockhash, $blocknumber,
$extra ) = getsnap( $blockhash ); $extra ) = getsnap( $blockhash );
last if not defined $blockhash;
$counter += $extra; $counter += $extra;
last if not defined $blockhash;
last if $blocknumber->is_zero; last if $blocknumber->is_zero;
} }
return $counter; return $counter;
} }
sub getTransactionReceipt sub getTransactionReceipt
{ {
my $hash = lc shift; my $hash = lc shift;
my $json = rpcreq( 'eth_getTransactionReceipt', $hash ); my $json = rpcreq( 'eth_getTransactionReceipt', $hash );
...@@ -324,6 +332,41 @@ sub getTransactionReceipt ...@@ -324,6 +332,41 @@ sub getTransactionReceipt
$sql->updateRcptInfo( $hash, $status, $gasUsed, $contractAddress ); $sql->updateRcptInfo( $hash, $status, $gasUsed, $contractAddress );
} }
sub deleteDuplicates()
{
# We don't only check for orphans.. first we see if
# we can create some by deleting duplicate blockheights.
# If any duplicates are found, we will be deleting the
# uncle and the canonical block. The canonical will
# the be listed as parent to an orphans and recreated.
info "Asking for duplicates to delete.";
my @blocks = $sql->selectduplicateblocknumbers;
my $counter = scalar @blocks;
return 0 if $counter == 0;
info "D Deleting duplicate blocks @blocks.";
# Delete the receipts for these blocks, so we can
# remove the blocks.
$sql->deletetransactionsinblocknumbers( @blocks );
# Now we can remove the duplicate blocks.
$sql->deleteblocknumbers( @blocks );
return $counter;
}
sub missingReceipts()
{
my @rcpts = $sql->selecttxwhichmissreceipts( $maxinarow );
my $counter = scalar @rcpts;
return 0 if $counter == 0;
print "R Extra receipts: ";
while ( @rcpts and not $stop_request )
{
print "r";
getTransactionReceipt( shift @rcpts );
}
print "\n";
return $counter;
}
sub main sub main
{ {
$| = 1; $| = 1;
...@@ -354,17 +397,7 @@ sub main ...@@ -354,17 +397,7 @@ sub main
# If any duplicates are found, we will be deleting the # If any duplicates are found, we will be deleting the
# uncle and the canonical block. The canonical will # uncle and the canonical block. The canonical will
# the be listed as parent to an orphans and recreated. # the be listed as parent to an orphans and recreated.
my @blocks = $sql->selectduplicateblocknumbers; $deletecounter = deleteDuplicates();
if ( @blocks )
{
info( "D Deleting duplicate blocks @blocks." );
# Delete the receipts for these blocks, so we can
# remove the blocks.
$sql->deletetransactionsinblocknumbers( @blocks );
# Now we can remove the duplicate blocks.
$sql->deleteblocknumbers( @blocks );
$deletecounter += scalar @blocks;
}
# #
# Orphans happen if some blocks have been deleted # Orphans happen if some blocks have been deleted
# or if the initial synchronization hasn't finished. # or if the initial synchronization hasn't finished.
...@@ -373,30 +406,14 @@ sub main ...@@ -373,30 +406,14 @@ sub main
push @blockqueue, $sql->listOrphans(); push @blockqueue, $sql->listOrphans();
# Look for transactions which hasn't had their receipts # Look for transactions which hasn't had their receipts
# processed (not common that this should happen). # processed (not common that this should happen).
my @rcptqueue = $sql->selecttxwhichmissreceipts( $maxinarow ); $extrarcptcounter = missingReceipts();
while ( @rcptqueue and not $stop_request )
{
print "R Extra receipts: "
if $extrarcptcounter == 0;
$extrarcptcounter++;
print "r";
getTransactionReceipt( shift @rcptqueue );
}
print "\n" if $extrarcptcounter > 0;
# Find out who signed all blocks
my $n = 1;
while ( not $stop_request and $n > 0 )
{
$n = allsnaps();
$snapcounter += $n;
}
} }
my $didsomething = 0; my $didsomething = 0;
do { do {
unshift @blockqueue, 'latest'; unshift @blockqueue, 'latest';
my $blockcounter = 0; my $blockcounter = 0;
my $newtx = 0;; my $newtx = 0;
while ( @blockqueue and $blockcounter <= 2500 and not $stop_request ) while ( @blockqueue and $blockcounter < 2500 and not $stop_request )
{ {
my $block = shift @blockqueue; my $block = shift @blockqueue;
my ( $nextblock, $n ) my ( $nextblock, $n )
...@@ -405,21 +422,27 @@ sub main ...@@ -405,21 +422,27 @@ sub main
next if not defined $n; next if not defined $n;
$blockcounter++; $blockcounter++;
$newtx += $n; $newtx += $n;
$snapcounter += allsnaps( $block );
# Put the parent block in front in @blockqueue # Put the parent block in front in @blockqueue
unshift @blockqueue, $nextblock unshift @blockqueue, $nextblock
if defined $nextblock; if defined $nextblock;
} }
my $n = 1;
# Find out who signed all blocks
while ( not $stop_request and $n > 0 and $snapcounter < 5000 )
{
$n = allsnaps();
$snapcounter += $n;
}
$didsomething = $blockcounter + $snapcounter + $extrarcptcounter; $didsomething = $blockcounter + $snapcounter + $extrarcptcounter;
if ( $didsomething > 0 ) if ( $didsomething > 0 )
{ {
info info
"C Committing %d Delete%s, %d Block%s, %d Transaction%s, %d Snapshot%s, %d extra Receipt%s.", "C Committing %d Delete%s, %d Block%s, %d Transaction%s, %d Snapshot%s, %d extra Receipt%s.",
$deletecounter, $deletecounter==1?'':'s', $deletecounter, plural( $deletecounter ),
$blockcounter, $blockcounter==1?'':'s', $blockcounter, plural( $blockcounter ),
$newtx, $newtx==1?'':'s', $newtx, plural( $newtx ),
$snapcounter, $snapcounter==1?'':'s', $snapcounter, plural( $snapcounter ),
$extrarcptcounter, $extrarcptcounter==1?'':'s'; $extrarcptcounter, plural( $extrarcptcounter );
$sql->commit; $sql->commit;
} }
$extrarcptcounter = 0; $extrarcptcounter = 0;
......
...@@ -16,18 +16,44 @@ services: ...@@ -16,18 +16,44 @@ services:
- pgdata:/var/lib/postgresql/data - pgdata:/var/lib/postgresql/data
ports: ports:
- 5432:5432 - 5432:5432
logging:
driver: "json-file"
options:
max-file: "5"
max-size: "10m"
collector: collector:
build: collector build: collector
#command: "collector/colletor.pl"
restart: always restart: always
depends_on: depends_on:
- postgres - postgres
- geth
environment: environment:
POSTGRES_PASSWORD: theswampthing POSTGRES_PASSWORD: theswampthing
BFANODE: http://public.47525974938.bfa.martin-legene.dk:8545/ BFANODE: http://geth:8545/
volumes: volumes:
- ./collector:/home/bfa/collector - ./collector:/home/bfa/collector
logging:
driver: "json-file"
options:
max-file: "5"
max-size: "10m"
geth:
#image: bfaar/nodo
build: geth
restart: always
ports:
- 8545:8545
- 8546:8546
volumes:
- geth:/home/bfa/bfa
logging:
driver: "json-file"
options:
max-file: "5"
max-size: "10m"
stop_grace_period: 300s
# adminer: # adminer:
# image: adminer # image: adminer
...@@ -37,3 +63,4 @@ services: ...@@ -37,3 +63,4 @@ services:
volumes: volumes:
pgdata: pgdata:
geth:
...@@ -215,7 +215,7 @@ INSERT INTO "account" ("id", "address", "shortname", "name") VALUES ...@@ -215,7 +215,7 @@ INSERT INTO "account" ("id", "address", "shortname", "name") VALUES
(25, '0x9b3ac6719b02ec7bb4820ae178d31c0bbda3a4e0', 'Everis', 'Everis'), (25, '0x9b3ac6719b02ec7bb4820ae178d31c0bbda3a4e0', 'Everis', 'Everis'),
(26, '0x99d6c9fca2a61d4ecdeb403515eb8508dc560c6b', NULL, 'Ultima Milla 0x99 anterior'), (26, '0x99d6c9fca2a61d4ecdeb403515eb8508dc560c6b', NULL, 'Ultima Milla 0x99 anterior'),
(27, '0xc0310a7b3b25f49b11b901a667208a3eda8d7ceb', NULL, 'SyT'), (27, '0xc0310a7b3b25f49b11b901a667208a3eda8d7ceb', NULL, 'SyT'),
(28, '0xabeff859aa6b0fb206d840dbf19de970065d4437', NULL, 'Belatrix'); (28, '0xabeff859aa6b0fb206d840dbf19de970065d4437', NULL, 'Belatrix'),
(401, '0xbfa02a9639318f5ed1291e2cee387aa9f9d68f98', NULL, 'BFA vault 01'), (401, '0xbfa02a9639318f5ed1291e2cee387aa9f9d68f98', NULL, 'BFA vault 01'),
(402, '0xbfa1c42c7381a4ad32e13ce3263b639a0e0488f2', NULL, 'BFA vault 02'), (402, '0xbfa1c42c7381a4ad32e13ce3263b639a0e0488f2', NULL, 'BFA vault 02'),
(403, '0xbfa2c97c3f59cc929e8feb1aee2aca0b38235d18', NULL, 'BFA vault 03'), (403, '0xbfa2c97c3f59cc929e8feb1aee2aca0b38235d18', NULL, 'BFA vault 03'),
...@@ -227,7 +227,7 @@ INSERT INTO "account" ("id", "address", "shortname", "name") VALUES ...@@ -227,7 +227,7 @@ INSERT INTO "account" ("id", "address", "shortname", "name") VALUES
(500, '0xd15dd6dbbe722b451156a013c01b29d91e23c3d6', 'dist1-mgr', 'admin de destileria1'), (500, '0xd15dd6dbbe722b451156a013c01b29d91e23c3d6', 'dist1-mgr', 'admin de destileria1'),
(501, '0xecb6aff6e38dc58c4d9aae2f7927a282bcb77ac2', 'SC dist1', 'SC destileria1'), (501, '0xecb6aff6e38dc58c4d9aae2f7927a282bcb77ac2', 'SC dist1', 'SC destileria1'),
(502, '0xbf1ce9cca7dedea3fdb22d169b49643664602ee1', 'DGSI-dist2', 'Destileria de DGSI'), (502, '0xbf1ce9cca7dedea3fdb22d169b49643664602ee1', 'DGSI-dist2', 'Destileria de DGSI'),
(503, '0xc3cf96af51d3c41dfbe428de75a8e5597d4d7a7b', 'SC-dist2', 'SC destileria de DGSI'), (503, '0xc3cf96af51d3c41dfbe428de75a8e5597d4d7a7b', 'SC-dist2', 'SC destileria de DGSI');
-- 2020-07-13 05:48:47.242334+00 -- 2020-07-13 05:48:47.242334+00
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment