diff --git a/collector/Dockerfile b/collector/Dockerfile index 013a123c35437b3d82a836c8c48b5fc369377c69..0b783d99c51830817d5505c09147f128dbcb4174 100644 --- a/collector/Dockerfile +++ b/collector/Dockerfile @@ -2,9 +2,11 @@ FROM debian RUN useradd --create-home --shell /bin/bash bfa && mkdir /home/bfa/collector RUN apt-get update && apt-get upgrade -y RUN apt-get install -y libjson-perl libnet-dns-perl libdbd-pg-perl libwww-perl vim -COPY collector.pl /home/bfa/collector +RUN apt-get install -y libclass-accessor-perl +COPY collector.pl sql.pm /home/bfa/collector/ RUN chown -R bfa /home/bfa USER bfa -ENV BFANODE public.47525974938.bfa.martin-legene.dk. +WORKDIR /home/bfa +ENV BFANODE http://public.47525974938.bfa.martin-legene.dk:8545/ #CMD /home/bfa/collector/collector.sh -CMD sleep 14d \ No newline at end of file +CMD sleep 14d diff --git a/collector/collector.pl b/collector/collector.pl index e25a1e62f1302f9a598350dc8fa999929a59571f..99e8f8753537b01a3cdeb22b7aeb3b0c521f02d3 100755 --- a/collector/collector.pl +++ b/collector/collector.pl @@ -11,6 +11,8 @@ use LWP; use JSON; use Net::DNS; use Net::DNS::Resolver; +use lib '/home/bfa/collector'; +use lib 'collector'; use lib '.'; use sql; @@ -98,10 +100,20 @@ sub rpcreq sub getblock { - my $block = shift; - my $rpccmd = $block =~ /^(earliest|latest|pending)$/ - ? 'eth_getBlockByNumber' - : 'eth_getBlockByHash'; + my $block = lc shift; + my $rpccmd; + if ( $block =~ /^0x[\da-f]{64}$/ ) + { + $rpccmd = 'eth_getBlockByHash'; + } + else + { + $rpccmd = 'eth_getBlockByNumber'; + if ( $block =~ /^\d+$/ ) + { + $block = Math::BigInt->new( $block )->as_hex; + } + } my $json = rpcreq( $rpccmd, $block, 'false' ); die "We should have received some kind of JSON from our RPC call, " . "but apparently not. Stopped " @@ -121,7 +133,7 @@ sub getblock $json->{'size'}, $json->{'gasUsed'}, $json->{'gasLimit'}; - $sql->sth_insertBlock->execute( + $sql->insertBlock( lc $json->{'hash'}, lc $json->{'parentHash'}, $json->{'number'}, @@ -291,20 +303,18 @@ sub allsnaps sub main { $| = 1; - info "Connecting to database and setting up " - . "prepared statements.\n"; $sql = sql->new(); versioncheck(); info "Looking for orphaned blocks in the " . "database.\n"; - push @blockqueue, sql->listOrphans(); + push @blockqueue, $sql->listOrphans(); while ( 1 ) { unshift @blockqueue, 'latest'; while ( @blockqueue ) { - my $maxinarow = 2500; + my $maxinarow = 100; getblock( shift @blockqueue ) while @blockqueue and --$maxinarow; # Find out who signed all blocks diff --git a/collector/sql.pm b/collector/sql.pm index 249f5d7b08e35ed9b97d38facc28668c977f5e5c..516257fd308ac00340efabfcb9e8751701c4f90d 100644 --- a/collector/sql.pm +++ b/collector/sql.pm @@ -3,7 +3,10 @@ # LGPLv2-only package sql; +use warnings; +use strict; use DBI; +use Data::Dumper; use base qw( Class::Accessor ); __PACKAGE__->mk_accessors( 'dbh' ); @@ -28,11 +31,11 @@ sub listOrphans { my ( $self ) = @_; my $sth = $self->dbh->prepare(q( - SELECT parentBlockhashId,id - FROM blocks - WHERE parentBlockhashId NOT IN ( + SELECT "parentBlockhashId", id + FROM block + WHERE "parentBlockhashId" NOT IN ( SELECT id - FROM blocks + FROM block ) )); my @results; @@ -55,16 +58,19 @@ sub insertTransactionWithContractAddress { $self->sth_insertTransactionWithContractAddress( $self->dbh->prepare(q( - INSERT INTO transactions - ( hash, blockId, nonce, gas, - gasPrice, value, fromAccountId, toAccountId, - status, gasUsed, input, inputlen, - contractaddressAccountId ) + INSERT INTO transaction + ( hash, "blockId", nonce, gas, + "gasPrice", value, "fromAccountId","toAccountId", + status, "gasUsed", input, inputlen, + "contractaddressAccountId" ) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?) )) ); } $_[1] = $self->selectBlockhashByHash( $_[1] ); + $_[6] = $self->selectAccountIdByAddress( $_[6] ); + $_[7] = $self->selectAccountIdByAddress( $_[7] ); + $_[12] = $self->selectAccountIdByAddress( $_[12] ); $self->sth_insertTransactionWithContractAddress->execute( @_ ); } @@ -72,16 +78,21 @@ __PACKAGE__->mk_accessors( 'sth_insertTransaction' ); sub insertTransaction { my $self = shift; - $self->sth_insertTransaction( - $self->dbh->prepare(q( - INSERT INTO transactions - ( hash, blockId, nonce, gas, - gasPrice, value, fromAccountId, toAccountId, - status, gasUsed, input, inputlen ) - VALUES (?,?,?,?,?,?,?,?,?,?,?,?) - )) - ); + if ( not defined $self->sth_insertTransaction ) + { + $self->sth_insertTransaction( + $self->dbh->prepare(q( + INSERT INTO transaction + ( hash, "blockId", nonce, gas, + "gasPrice", value, "fromAccountId","toAccountId", + status, "gasUsed", input, inputlen ) + VALUES (?,?,?,?,?,?,?,?,?,?,?,?) + )) + ); + } $_[1] = $self->selectBlockhashByHash( $_[1] ); + $_[6] = $self->selectAccountIdByAddress( $_[6] ); + $_[7] = $self->selectAccountIdByAddress( $_[7] ); $self->sth_insertTransaction->execute( @_ ); } @@ -111,6 +122,7 @@ __PACKAGE__->mk_accessors( 'sth_selectBlockhashByHash' ); sub selectBlockhashByHash { my ( $self, $hash )= @_; + $hash = lc $hash; if ( not defined $self->sth_selectBlockhashByHash ) { $self->sth_selectBlockhashByHash( @@ -143,6 +155,7 @@ __PACKAGE__->mk_accessors( 'sth_insertBlockhash' ); sub insertBlockhash { my ( $self, $hash )= @_; + $hash = lc $hash; if ( not defined $self->sth_insertBlockhash ) { $self->sth_insertBlockhash( @@ -161,6 +174,7 @@ my %account; sub selectAccountIdByAddress { my ( $self, $address )= @_; + $address = lc $address; if ( not defined $self->sth_selectAccountIdByAddress ) { $self->sth_selectAccountIdByAddress( @@ -171,6 +185,7 @@ sub selectAccountIdByAddress )) ); } + return $account{$address} if exists $account{$address}; while ( not exists $account{$address} ) { # Try to look up the hash @@ -180,16 +195,11 @@ sub selectAccountIdByAddress # If found, return it immediately if ( defined $ref ) { - $account{$address} = $row->[0]; + $account{$address} = $ref->[0]; last; } - # Do maximum 1 attempt to insert a hash - # (this is actually a failure and should not happen) - return if $inserts > 0; # Insert a hash, because it is not already in the database. - $self->insertAccount( $hash ); - # Avoid loops. - $inserts++; + $self->insertAccount( $address ); } return $account{$address}; } @@ -198,6 +208,7 @@ __PACKAGE__->mk_accessors( 'sth_insertAccount' ); sub insertAccount { my ( $self, $address )= @_; + $address = lc $address; if ( not defined $self->sth_insertAccount ) { $self->sth_insertAccount( @@ -208,7 +219,7 @@ sub insertAccount )) ); } - $self->sth_insertAccount->execute( $hash ); + $self->sth_insertAccount->execute( $address ); } __PACKAGE__->mk_accessors( 'sth_insertBlock' ); @@ -220,9 +231,9 @@ sub insertBlock { $self->sth_insertBlock( $self->dbh->prepare(q( - INSERT INTO blocks - ( id, parentBlockhashId, number, timestamp, - difficulty, gasUsed, gasLimit, size ) + INSERT INTO block + ( id, "parentBlockhashId", number, timestamp, + difficulty, "gasUsed", "gasLimit", size ) VALUES (?,?,?,?,?,?,?,?) )) ); @@ -236,64 +247,67 @@ __PACKAGE__->mk_accessors( 'sth_selectBlockByHash' ); sub selectBlockByHash { my ( $self, $hash) = @_; + $hash = lc $hash; if ( not defined $self->sth_selectBlockByHash ) { $self->sth_selectBlockByHash( $self->dbh->prepare(q( - SELECT parenthash,number,sealer - FROM blocks - WHERE hash=? + SELECT "parentBlockhashId",number,"sealerAccountId" + FROM block + WHERE id=? )) ); } - $sth->sth_selectBlockByHash( $hash ); - my $row = $sth->selectBlockByHash->fetchrow_hashref; - $sth->selectBlockByHash->finish; + my $blockid = $self->selectBlockhashByHash( $hash ); + $self->sth_selectBlockByHash->execute( $blockid ); + my $ref = $self->sth_selectBlockByHash->fetchrow_hashref; + $self->sth_selectBlockByHash->finish; return - if not defined $row; - return $row; + if not defined $ref; + return $ref; } __PACKAGE__->mk_accessors( 'sth_selectTransactionByHash' ); sub selectTransactionByHash { my ( $self, $hash) = @_; + $hash = lc $hash; if ( not defined $self->sth_selectTransactionByHash ) { $self->sth_selectTransactionByHash( $self->dbh->prepare(q( SELECT 1 - FROM transactions + FROM transaction WHERE hash=? )) ); } - $sth->sth_selectTransactionByHash( $hash ); - my $row = $sth->selectTransactionByHash->fetchrow_hashref; - $sth->selectTransactionByHash->finish; - return - if not defined $row; - return $row; + $self->sth_selectTransactionByHash->execute( $hash ); + my $ref = $self->sth_selectTransactionByHash->fetchrow_hashref; + $self->sth_selectTransactionByHash->finish; + return $ref + if defined $ref; + return; } -__PACKAGE__->mk_accessors( 'sth_selectAccountIdByAddress' ); +__PACKAGE__->mk_accessors( 'sth_updateWhoSealed' ); sub updateWhoSealed { my ( $self, $sealerhash, $blockhash ) = @_; - if ( not defined $self->sth_updatewhosealed ) + if ( not defined $self->sth_updateWhoSealed ) { - $self->sth_updatewhosealed( + $self->sth_updateWhoSealed( $self->dbh->prepare(q( - UPDATE blocks - SET sealerAccountId=? + UPDATE block + SET "sealerAccountId"=? WHERE id=? - AND sealerAccountId IS NULL + AND "sealerAccountId" IS NULL )) ); } - my $blockid = $self->selectBlockhashByHash( $blockhash ); - my $sealerid = $self->selectSealerByHash( $sealerhash ); - $self->sth_updatewhosealed->execute( $sealerid, $blockid ); + my $blockid = $self->selectBlockhashByHash( $blockhash ); + my $sealerid = $self->selectAccountIdByAddress( $sealerhash ); + $self->sth_updateWhoSealed->execute( $sealerid, $blockid ); } __PACKAGE__->mk_accessors( 'sth_selectmaxunknownsigned' ); @@ -305,19 +319,20 @@ sub selectMaxUnknownSigned $self->sth_selectmaxunknownsigned( $self->dbh->prepare(q( SELECT id - FROM blocks + FROM block WHERE number = ( SELECT MAX(number) - FROM blocks - WHERE sealer IS NULL + FROM block + WHERE "sealerAccountId" IS NULL ) )) ); } $self->sth_selectmaxunknownsigned->execute(); - my $ref = $self->sth_selectmaxunknownsigned->fetchrow_hashref; + my $ref = $self->sth_selectmaxunknownsigned->fetchrow_arrayref; $self->sth_selectmaxunknownsigned->finish(); - my $blockid = $ref->{'number'}; + return if not defined $ref; + my $blockid = $ref->[0]; my $blockhash = $self->selectBlockhashById( $blockid ); return $blockhash; } diff --git a/docker-compose.yml b/docker-compose.yml index fbfa55ca5eabf0e5a58679b56a3f227ee501bbe8..44a981afa2ee70be90694836e2f3154d3e01b200 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,12 +21,13 @@ services: restart: always environment: POSTGRES_PASSWORD: onlythelonely + BFANODE: http://200.108.146.200:8545/ - adminer: - image: adminer - restart: always - ports: - - 8080:8080 +# adminer: +# image: adminer +# restart: always +# ports: +# - 8080:8080 volumes: blockdb_postgres: diff --git a/postgres/10-postgres.sql b/postgres/10-postgres.sql index 4d31f18f584cdf385c55c067a99ab6da7901af91..c970b49ad7031af1fea6aedfdbdd0105ac1456a1 100644 --- a/postgres/10-postgres.sql +++ b/postgres/10-postgres.sql @@ -2,7 +2,7 @@ \connect "postgres"; -CREATE SEQUENCE account_id_seq INCREMENT 1 MINVALUE 1 MAXVALUE 2147483647 START 28 CACHE 1; +CREATE SEQUENCE account_id_seq INCREMENT 1 MINVALUE 1 MAXVALUE 2147483647 START 1000 CACHE 1; CREATE TABLE "public"."account" ( "id" integer DEFAULT nextval('account_id_seq') NOT NULL, "address" character(42) NOT NULL, @@ -12,7 +12,7 @@ CREATE TABLE "public"."account" ( CONSTRAINT "account_hash" UNIQUE ("address") ) WITH (oids = false); -CREATE SEQUENCE blockhash_id_seq INCREMENT 1 MINVALUE 1 MAXVALUE 2147483647 START 1234567 CACHE 1; +CREATE SEQUENCE blockhash_id_seq INCREMENT 1 MINVALUE 1 MAXVALUE 2147483647 START 1000000 CACHE 1; CREATE TABLE "public"."blockhash" ( "id" integer DEFAULT nextval('blockhash_id_seq') NOT NULL, "hash" character(66) NOT NULL, @@ -65,11 +65,11 @@ CREATE INDEX "transaction_toAccountId" ON "public"."transaction" USING btree ("t CREATE INDEX "transaction_blockId" ON "public"."transaction" USING btree ("blockId"); INSERT INTO "account" ("id", "address", "shortname", "name") VALUES -(3, '0x354779914a94ad428d2b53ae96cce3010bb0ce1e', 'RedLink', 'RedLink SA'), (1, '0x377ab0cd00744dbb07b369cd5c0872dcd362c8f0', 'UNER', 'Universidad Nacional de Entre Rios'), (2, '0x2feb6a8876bd9e2116b47834b977506a08ea77bd', 'PNA', 'Prefectura Nacional Argentina'), -(5, '0xd1f17aa41354d58940c300ffd79a200944dda2df', NULL, 'Marandu'), +(3, '0x354779914a94ad428d2b53ae96cce3010bb0ce1e', 'RedLink', 'RedLink SA'), (4, '0x998c2651db6f76ca568c0071667d265bcc1b1e98', NULL, 'ASI'), +(5, '0xd1f17aa41354d58940c300ffd79a200944dda2df', NULL, 'Marandu'), (6, '0x39170a1ce03729d141dfaf8077c08b72c9cfdd0c', 'IXPBB', 'IXP Bahia Blanca'), (7, '0x02665f10cb7b93b4491ac9594d188ef2973c310a', 'CABASE-MZA', 'CABASE Mendoza'), (8, '0x19fe7b9b3a1bebde77c5374c8e13c623e3d1b5b2', 'ARIU', 'Asociación Redes de Interconexión Universitaria'),