Skip to content
Snippets Groups Projects
Commit d1780003 authored by Robert Martin-Legene's avatar Robert Martin-Legene
Browse files

non-working import

parent 08f58abf
No related branches found
No related tags found
No related merge requests found
.vstags
FROM debian
RUN useradd --create-home --shell /bin/bash bfa && mkdir /home/bfa/collector
RUN apt-get update && apt-get upgrade -y
RUN apt-get install -y libjson-perl libnet-dns-perl libdbd-pg-perl libwww-perl vim
COPY collector.pl /home/bfa/collector
RUN chown -R bfa /home/bfa
USER bfa
#CMD /home/bfa/collector/collector.sh
CMD sleep 14d
#!/usr/bin/perl -w
# 20200108 Robert Martin-Legene <robert@martin-legene.dk> / <robert@nic.ar>
# (c)2020 Secretaría Legal y Técnica de la Presidencia de la Nación
# LGPLv2-only
use warnings;
use strict;
use Data::Dumper;
use Math::BigInt;
use LWP;
use JSON;
use Net::DNS;
use Net::DNS::Resolver;
use DBI;
use "sql"
# 47525974938;
my $netversion = Math::BigInt->new( "0xb10c4d39a" );
# 200941592
my $chainid = Math::BigInt->new( "0xbfa2018" );
my $idcounter = 0;
my $ua;
my @endpoints;
my @blockqueue;
my %sealers;
# Set maxruns to 1 if you want to just do a single run to keep
# your database up-to-date.
my $maxruns = 0;
sub info(@)
{
# if 1 param only
unshift @_, '%s'
if $#_ == 0;
my $format = shift;
$format = "%s: " . $format;
printf $format, astime(time), @_;
}
sub astime
{
my @t = localtime( $_[0] );
$t[5] += 1900;
$t[4] += 1;
return sprintf '%04d%02d%02d-%02d%02d%02d', @t[5,4,3,2,1,0];
}
sub shorthash($)
{
local $_ = $_[0];
s/^(0x.......).*(.......)$/$1..$2/;
return $_;
}
sub rpcreq
{
my ( $endpoint,
$opname, @params ) = @_;
# $ua->ssl_opts( 'verify_hostname' => 0 );
my %args = (
jsonrpc => '2.0',
method => $opname,
id => $idcounter++,
);
my @args = map {
my $v = $args{$_};
$v = qq/"${v}"/ if $v !~ /^\d+$/;
sprintf qq/"%s":%s/, $_, $v;
} keys %args;
if ( scalar @params )
{
my $p = '"params":[';
foreach my $param ( @params )
{
$param = '"' . $param . '"'
if $param ne 'true'
and $param ne 'false'
and $param !~ /^\d+$/;
$p .= $param . ',';
}
chop $p;
$p .= ']';
push @args, $p;
}
my $args = '{' . join(',',@args) . '}';
my $res = $ua->post(
$endpoint,
'Content-Type' => 'application/json',
'Content' => $args,
);
die $res->status_line
unless $res->is_success;
my $json = decode_json($res->content);
return $json->{'result'};
}
sub endpoint_find
{
my $networknumber = $netversion->bstr;
my $res = Net::DNS::Resolver->new();
my $origin = 'bfa.martin-legene.dk';
my $fqdn = sprintf
'_rpc._tcp.public.%s.%s.',
$networknumber, $origin;
my $reply = $res->query( $fqdn, 'SRV' );
die "No SRV RR found for public endpoints. Stopped"
if not $reply;
my @answer;
@answer = rrsort( 'SRV', 'priority', $reply->answer );
die "DNS SRV query returned no SRV records. Stopped"
if not @answer;
my %protolookup = (
'_rpc' => 'http',
'_rpcs' => 'https',
);
@endpoints = ();
foreach my $answer ( @answer )
{
my $targetname = $answer->target;
info "Publicly open endpoint found at %s.\n", $targetname;
my $r_a = $res->query( $targetname, 'A' );
my $r_aaaa = $res->query( $targetname, 'AAAA' );
my @addresses = ();
push @addresses, rrsort( 'A', $r_a->answer )
if $r_a;
push @addresses, rrsort( 'AAAA', $r_aaaa->answer )
if $r_aaaa;
warn "No address found for $targetname,"
if not @addresses;
my $label1 = (split( /\./, $fqdn, 2 ))[0];
foreach my $address_rr ( @addresses )
{
next
if not exists $protolookup{$label1};
my $proto = $protolookup{$label1};
push @endpoints, sprintf(
'%s://%s:%s',
$proto,
$address_rr->address,
$answer->port
);
}
}
info "Preferring endpoint at %s\n", $endpoints[0];
}
sub getblock
{
my $blockid = $_[0];
my $json;
$blockid = Math::BigInt->new( $blockid )
if ref($blockid) eq ''
and $blockid =~ /^\d+$/;
if ( ref $blockid eq 'Math::BigInt' or $blockid =~ /^(earliest|latest|pending)$/ )
{
my $specific = ref $blockid eq 'Math::BigInt'
? $blockid->as_hex
: $blockid;
$json = rpcreq(
$endpoints[0],
'eth_getBlockByNumber',
$specific, 'false' );
}
else
{
$sth_selectBlockByHash->execute( $blockid );
return if $sth_selectBlockByHash->fetch;
$json = rpcreq(
$endpoints[0],
'eth_getBlockByHash',
$blockid, 'false'
);
}
die "We should have received some kind of JSON from our RPC call, " .
"but apparently not. Stopped "
if not defined $json;
$sth_selectBlockByHash->execute( $json->{'hash'} );
return if $sth_selectBlockByHash->fetch;
foreach my $key (qw(
timestamp gasUsed gasLimit difficulty number totalDifficulty size
)) {
$json->{$key} = Math::BigInt->new( $json->{$key} )->bstr
if exists $json->{$key};
}
info "B %s %s %s %s %s %s/%s\n",
shorthash $json->{'hash'},
astime($json->{'timestamp'}),
$json->{'number'},
$json->{'difficulty'},
$json->{'size'},
$json->{'gasUsed'},
$json->{'gasLimit'};
$sth_insertBlock->execute(
lc $json->{'hash'},
lc $json->{'parentHash'},
$json->{'number'},
$json->{'timestamp'},
$json->{'difficulty'},
$json->{'gasUsed'},
$json->{'gasLimit'},
$json->{'size'}
);
if ( scalar @{$json->{'transactions'}} )
{
foreach my $txhash ( @{ $json->{'transactions'} } )
{
getTransByHash( $txhash );
}
}
unshift @blockqueue, lc $json->{'parentHash'}
if $json->{'number'} != 0;
}
sub getTransByHash
{
my $hash = $_[0];
$sth_selectTransactionByHash->execute( $hash );
return if $sth_selectTransactionByHash->fetch;
my $json = rpcreq(
$endpoints[0],
'eth_getTransactionByHash',
$hash
);
my $max = 0;
my $rcpt = undef;
do {
$rcpt = rpcreq(
$endpoints[0],
'eth_getTransactionReceipt',
$hash
);
if ( not $rcpt )
{
sleep 1;
warn "Couldn't get the transaction receipt for $hash\n";
}
} while not $rcpt and $max++ < 60;
die "Couldn't get the transaction receipt for $hash\n" if not $rcpt;
$json->{'status'} = $rcpt->{'status'};
$json->{'gasUsed'} = $rcpt->{'gasUsed'};
$json->{'contractAddress'}
= $rcpt->{'contractAddress'};
foreach my $key (qw( nonce value gas gasPrice gasUsed status))
{
$json->{$key} = Math::BigInt->new( $json->{$key} )->bstr;
}
info "T %s %s %s %s -> %s\n",
shorthash $json->{'hash'},
$json->{'nonce'},
$json->{'value'},
shorthash $json->{'from'},
shorthash $json->{'to'};
my $input = $json->{'input'};
$input =~ s/^0x//;
my $inputlen = length($input) / 2;
my @args = (
lc $json->{'hash'},
lc $json->{'blockHash'},
$json->{'nonce'},
$json->{'gas'},
$json->{'gasPrice'},
$json->{'value'},
lc $json->{'from'},
lc $json->{'to'},
$inputlen,
$json->{'gasUsed'},
$json->{'status'},
);
my $function = $sth_insertTransaction;
if ( $json->{'contractAddress'} )
{
push @args, $json->{'contractAddress'};
$function = $sth_insertTransactionWithContractAddress;
}
$function->execute( @args );
}
sub getTransRcptByHash
{
my $hash = $_[0];
}
sub versioncheck
{
my $n;
my $ok = 1;
$n = Math::BigInt->new(
rpcreq( $endpoints[0], "net_version" )
);
if ( $n->bcmp( $netversion ) != 0 )
{
warn "Network says it has net.version "
. $n->bstr
. " ("
. $n->as_hex
. "). Expected $netversion ("
. $netversion->as_hex
. ").\n";
$ok = 0;
}
$n = Math::BigInt->new(
rpcreq( $endpoints[0], "eth_chainId" )
);
if ( $n->bcmp( $chainid ) != 0 )
{
warn sprintf("Network says it has eth.chainId %s (%s). Expected %s (%s).\n", $n->bstr, $n->as_hex, $chainid->bstr, $chainid->as_hex);
$ok = 0;
}
exit 1 if not $ok;
return $ok;
}
sub sealer
{
my $hash = $_[0];
my $row;
die unless defined $hash;
return $sealers{$hash} if exists $sealers{$hash};
$sth_selectsealer->execute( $hash );
return $sealers{$hash} = $row->[0] if $row = $sth_selectsealer->fetch;
$sth_insertsealer->execute( $hash );
$sth_selectsealer->execute( $hash );
die "Apparently we failed to get find/create a new sealer. Stopped"
if not $row = $sth_selectsealer->fetch;
return $sealers{$hash} = $row->[0];
}
sub getsnap
{
my $number = $_[0];
my @blocks;
$number = Math::BigInt->new( $number )
if ref($number) ne 'Math::BigInt';
my $json = rpcreq(
$endpoints[0],
'clique_getSnapshot',
$number->as_hex
);
return if not defined $json;
my $recents = $json->{'recents'};
return if not defined $recents;
my $hash = $json->{'hash'};
my $count = 0;
while ( exists $recents->{$number->bstr} )
{
push @blocks, $number->bstr;
$sth_selectBlockByHash->execute( $hash );
my $row = $sth_selectBlockByHash->fetchrow_hashref;
return if not defined $row;
my $newnumber = Math::BigInt->new( $row->{'number'} );
return if $newnumber->bcmp($number) != 0;
# Stop if we got to a block with sealers.
return if defined $row->{'sealer'};
my $sealerhash = $recents->{$number->bstr};
return if not defined $sealerhash;
my $internalid = sealer( $sealerhash );
return if not defined $internalid;
my $rows_affected = $sth_updatewhosealed->execute(
$internalid,
$hash
);
# For next block...
$hash = $row->{'parentHash'};
$number->bsub(1);
}
info "S Snapshot at block # @blocks.\n";
return $number;
}
sub allsnaps
{
$sth_selectmaxunknownsigned->execute();
my $unkn = $sth_selectmaxunknownsigned->fetchrow_hashref;
my $number = Math::BigInt->new( $unkn->{'number'} );
my $committer = 0;
while ( defined $number and not $number->is_zero )
{
$number = getsnap( $number );
$dbh->commit
if ( $committer++ % 75 ) == 0;
}
$dbh->commit;
}
sub main
{
$| = 1;
info "Connecting to database and setting up "
. "prepared statements.\n";
sql::setup;
#
info "Looking for public RPC endpoints of "
. "the BFA.\n";
$ua = LWP::UserAgent->new( keep_alive => 10 );
endpoint_find();
versioncheck();
info "Looking for orphaned blocks in the "
. "database.\n";
push @blockqueue, sql::listOrphans();
while ( --$maxruns != 0 )
{
unshift @blockqueue, "latest";
while ( @blockqueue )
{
my $maxinarow = 2500;
getblock( shift @blockqueue )
while @blockqueue and --$maxinarow;
# Find out who signed all blocks
allsnaps();
$dbh->commit;
}
sleep 5;
}
}
main();
# 2020 Robert Martin-Legene <robert@martin-legene.dk> / <robert@nic.ar>
# (c)2020 Secretaría Legal y Técnica de la Presidencia de la Nación
# LGPLv2-only
package sql;
use DBI;
my @blockqueue;
my $dbh;
my $sth_insertBlock;
my $sth_selectBlockByHash;
my $sth_selectTransactionByHash;
my $sth_insertTransaction;
my $sth_insertTransactionWithContractAddress;
my $sth_insertsealer;
my $sth_selectsealer;
my $sth_updatewhosealed;
my $sth_selectmaxunknownsigned;
sub setup
{
$dbh = DBI->connect(
'dbi:Pg:dbname=postgres;host=postgres','postgres','onlythelonely',
{AutoCommit=>0,RaiseError=>1}
)
or die DBI::errstr;
$sth_insertBlock = $dbh->prepare(q(
INSERT INTO blocks
( hash, parentHash, number, timestamp,
difficulty, gasUsed, gasLimit, size )
VALUES (?,?,?,?,?,?,?,?)
)) or die $DBI::errstr;
$sth_selectBlockByHash = $dbh->prepare(q(
SELECT parenthash,number,sealer
FROM blocks
WHERE hash=?
)) or die $DBI::errstr;
$sth_selectTransactionByHash
= $dbh->prepare(q(
SELECT "exists"
FROM transactions
WHERE hash=?
)) or die $DBI::errstr;
$sth_insertTransactionWithContractAddress
= $dbh->prepare(q(
INSERT INTO transactions
( hash, blockHash, nonce, gas,
gasPrice, value, _from, _to,
inputlen, gasUsed, status, contractAddress )
VALUES (?,?,?,?,?,?,?,?,?,?,?,?)
)) or die $DBI::errstr;
$sth_insertTransaction = $dbh->prepare(q(
INSERT INTO transactions
( hash, blockHash, nonce, gas,
gasPrice, value, _from, _to,
inputlen, gasUsed, status )
VALUES (?,?,?,?,?,?,?,?,?,?,?)
)) or die $DBI::errstr;
$sth_selectsealer = $dbh->prepare(q(
SELECT internalid
FROM sealers
WHERE hash=?
)) or die $DBI::errstr;
$sth_insertsealer = $dbh->prepare(q(
INSERT INTO sealers
( hash )
VALUES (?)
)) or die $DBI::errstr;
$sth_updatewhosealed = $dbh->prepare(q(
UPDATE blocks
SET sealer=?
WHERE hash=?
AND sealer IS NULL
)) or die $DBI::errstr;
$sth_selectmaxunknownsigned
= $dbh->prepare(q(
SELECT number
FROM blocks
WHERE number = (
SELECT MAX(number)
FROM blocks
WHERE sealer IS NULL
)
)) or die $DBI::errstr;
}
sub listOrphans
{
my @results;
my $sth_selectOrphans
= $dbh->prepare(q(
SELECT parentHash,number
FROM blocks
WHERE parentHash NOT IN (
SELECT hash
FROM blocks
)
)) or die $DBI::errstr;
$sth_selectOrphans->execute();
while ( my $row = $sth_selectOrphans->fetch )
{
# Block 0 has it's parent listed as 0x0{64}
next if $row->[0] =~ /^0x0{64}$/;
push @results, $row->[0]
}
}
1;
\ No newline at end of file
## POSTGRES SUPERUSER user name is postgres
## POSTGRES SUPERUSER password is onlythelonely
## POSTGRES USER dgsi has password bfa
# defined in postgres/90-add-db-user.sh
version: '3.4'
services:
postgres:
build: postgres
restart: always
environment:
POSTGRES_PASSWORD: onlythelonely
volumes:
- blockdb_postgres:/var/lib/postgresql
collector:
build: collector
restart: always
environment:
POSTGRES_PASSWORD: onlythelonely
adminer:
image: adminer
restart: always
ports:
- 8080:8080
volumes:
blockdb_postgres:
-- Adminer 4.7.7 PostgreSQL dump
\connect "postgres";
CREATE SEQUENCE account_id_seq INCREMENT 1 MINVALUE 1 MAXVALUE 2147483647 START 28 CACHE 1;
CREATE TABLE "public"."account" (
"id" integer DEFAULT nextval('account_id_seq') NOT NULL,
"address" character(42) NOT NULL,
"shortname" character varying(16),
"name" character varying(255),
CONSTRAINT "account_id" PRIMARY KEY ("id"),
CONSTRAINT "account_hash" UNIQUE ("address")
) WITH (oids = false);
CREATE SEQUENCE blockhash_id_seq INCREMENT 1 MINVALUE 1 MAXVALUE 2147483647 START 1234567 CACHE 1;
CREATE TABLE "public"."blockhash" (
"id" integer DEFAULT nextval('blockhash_id_seq') NOT NULL,
"hash" character(66) NOT NULL,
CONSTRAINT "blockhash_id" PRIMARY KEY ("id"),
CONSTRAINT "blockhash_hash" UNIQUE ("hash")
) WITH (oids = false);
CREATE TABLE "public"."block" (
"id" integer NOT NULL,
"parentBlockhashId" integer,
"number" integer NOT NULL,
"sealerAccountId" integer,
"timestamp" integer NOT NULL,
"difficulty" smallint NOT NULL,
"gasUsed" integer NOT NULL,
"gasLimit" integer NOT NULL,
"size" integer NOT NULL,
CONSTRAINT "block_id" PRIMARY KEY ("id"),
CONSTRAINT "block_sealerAccountId_fkey" FOREIGN KEY ("sealerAccountId") REFERENCES account(id) NOT DEFERRABLE,
CONSTRAINT "block_id_fkey" FOREIGN KEY (id) REFERENCES blockhash(id) NOT DEFERRABLE,
CONSTRAINT "block_parentBlockhashId_fkey" FOREIGN KEY ("parentBlockhashId") REFERENCES blockhash(id) NOT DEFERRABLE
) WITH (oids = false);
CREATE INDEX "block_parentBlockHashId" ON "public"."block" USING btree ("parentBlockhashId");
CREATE INDEX "block_sealerAccountId" ON "public"."block" USING btree ("sealerAccountId");
CREATE INDEX "block_number" ON "public"."block" USING btree ("number");
CREATE INDEX "block_timestamp" ON "public"."block" USING btree ("timestamp");
CREATE TABLE "public"."transaction" (
"hash" character(66) NOT NULL,
"blockId" integer,
"nonce" integer NOT NULL,
"gas" integer NOT NULL,
"gasPrice" integer NOT NULL,
"value" integer NOT NULL,
"fromAccountId" integer NOT NULL,
"toAccountId" integer NOT NULL,
"contractaddressAccountId" integer,
"status" smallint NOT NULL,
"gasUsed" integer NOT NULL,
"inputlen" smallint NOT NULL,
"input" text NOT NULL,
CONSTRAINT "transaction_contractaddressAccountId_fkey" FOREIGN KEY ("contractaddressAccountId") REFERENCES account(id) NOT DEFERRABLE,
CONSTRAINT "transaction_fromAccountId_fkey" FOREIGN KEY ("fromAccountId") REFERENCES account(id) NOT DEFERRABLE,
CONSTRAINT "transaction_toAccountId_fkey" FOREIGN KEY ("toAccountId") REFERENCES account(id) NOT DEFERRABLE,
CONSTRAINT "transaction_blockId_fkey" FOREIGN KEY ("blockId") REFERENCES block(id) NOT DEFERRABLE
) WITH (oids = false);
CREATE INDEX "transaction_contractaddressAccountId" ON "public"."transaction" USING btree ("contractaddressAccountId");
CREATE INDEX "transaction_fromAccountId" ON "public"."transaction" USING btree ("fromAccountId");
CREATE INDEX "transaction_toAccountId" ON "public"."transaction" USING btree ("toAccountId");
CREATE INDEX "transaction_blockId" ON "public"."transaction" USING btree ("blockId");
INSERT INTO "account" ("id", "address", "shortname", "name") VALUES
(3, '0x354779914a94ad428d2b53ae96cce3010bb0ce1e', 'RedLink', 'RedLink SA'),
(1, '0x377ab0cd00744dbb07b369cd5c0872dcd362c8f0', 'UNER', 'Universidad Nacional de Entre Rios'),
(2, '0x2feb6a8876bd9e2116b47834b977506a08ea77bd', 'PNA', 'Prefectura Nacional Argentina'),
(5, '0xd1f17aa41354d58940c300ffd79a200944dda2df', NULL, 'Marandu'),
(4, '0x998c2651db6f76ca568c0071667d265bcc1b1e98', NULL, 'ASI'),
(6, '0x39170a1ce03729d141dfaf8077c08b72c9cfdd0c', 'IXPBB', 'IXP Bahia Blanca'),
(7, '0x02665f10cb7b93b4491ac9594d188ef2973c310a', 'CABASE-MZA', 'CABASE Mendoza'),
(8, '0x19fe7b9b3a1bebde77c5374c8e13c623e3d1b5b2', 'ARIU', 'Asociación Redes de Interconexión Universitaria'),
(9, '0xe70fbc9d6be2fe509e4de7317637d8ee83d4f13c', 'CABASE-PMY', 'CABASE Puerto Madryn'),
(10, '0xe191ac3108cb2c5d70d0e978876c048d4ba41b03', 'ANSV', 'Agencia Nacional de Seguridad Vial'),
(11, '0xf36475eb25ba0c825455f150b26e24ab9449a443', 'SRT', 'Superintendencia de Riesgos del Trabajo'),
(12, '0xd1420aa9dd092f50f68913e9e53b378a68e76ed7', 'SMGP/OPTIC', 'Secretaría de Modernización de la Gestión Pública / Oficina Provincial de Tecnologías de la Información y la Comunicación - Gobierno de la Provincia del Neuquén'),
(13, '0x2388d2cdb2cd6e7722b4af39c3bb406dd31f560e', 'UNR', 'Universidad Nacional de Rosario'),
(14, '0x342e1d075d820ed3f9d9a05967ec4055ab23fa1e', 'CABASE', 'CABASE CABA'),
(15, '0xb3d1209aefbe00c78b2247656e2ddfa9e3897526', 'Colescriba', 'Colegio de Escribanos de la Provincia de Buenos Aires'),
(16, '0xa14152753515674ae47453bea5e155a20c4ebabc', 'UP', 'Universidad de Palermo'),
(17, '0x97a47d718eab9d660b10de08ef42bd7fd915b783', 'UNLP', 'Universidad Nacional de La Plata'),
(18, '0x850b30dc584b39275a7ddcaf74a5c0e211523a30', 'UM', 'Ultima Milla'),
(19, '0x609043ebde4a06bd28a1de238848e8f82cca9c23', 'UNSJ', 'Universidad Nacional de San Juan'),
(20, '0xb43b53af0db2c3fac788195f4b4dcf2b3d72aa44', NULL, 'IPlan'),
(21, '0x46991ada2a2544468eb3673524641bf293f23ccc', 'UNC', 'Universidad Nacional de Cordoba'),
(22, '0x401d7a8432caa1025d5f093276cc6ec957b87c00', 'ONTI', 'Oficina Nacional de Tecnologias de Informacion'),
(23, '0x91c055c6478bd0ad6d19bcb58f5e7ca7b04e67f1', 'DGSI', 'Dirección General de Sistemas Informáticos'),
(24, '0x52f8a89484947cd29903b6f52ec6beda69965e38', 'CABASE-PSS', 'CABASE Posadas'),
(25, '0x9b3ac6719b02ec7bb4820ae178d31c0bbda3a4e0', 'Everis', 'Everis'),
(26, '0x99d6c9fca2a61d4ecdeb403515eb8508dc560c6b', NULL, NULL),
(27, '0xc0310a7b3b25f49b11b901a667208a3eda8d7ceb', '', 'SyT'),
(28, '0xabeff859aa6b0fb206d840dbf19de970065d4437', NULL, 'Belatrix');
-- 2020-07-13 05:48:47.242334+00
#!/bin/sh
psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
CREATE USER leer WITH PASSWORD 'bfa';
GRANT CONNECT ON DATABASE postgres TO leer;
GRANT SELECT ON ALL TABLES IN SCHEMA public TO leer;
GRANT UPDATE (shortname,name) ON TABLE public.account TO leer;
EOSQL
FROM postgres:13-alpine
COPY ??-* /docker-entrypoint-initdb.d/
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment