From ac1cbdee4d4e3bc82595f1d1322d8af63fb01d98 Mon Sep 17 00:00:00 2001
From: Robert Martin-Legene <robert@nic.ar>
Date: Wed, 21 Jul 2021 18:32:35 +0000
Subject: [PATCH] Importdb
---
bin/bfa | 63 +++++++++++++++++++++++++++++++++++----------------------
1 file changed, 39 insertions(+), 24 deletions(-)
diff --git a/bin/bfa b/bin/bfa
index 152de13..cf05557 100755
--- a/bin/bfa
+++ b/bin/bfa
@@ -139,7 +139,7 @@ function graceful
if [ -z "$pids" ]
then
echo "Nothing to send signals to." >&2
- exit 1
+ return
fi
sendsig -TERM "$pids"
sleep 1
@@ -170,15 +170,15 @@ function initdb
register_subcommand 'exportdb' 'exportdb' 'Export blockchain in chunks of 1 million blocks per file.'
function exportdb
{
- local delta maxblocks blockstart toblock filename
- delta=1000000
+ local million maxblocks blockstart toblock filename
+ million=1000000
graceful
maxblocks=$(bfageth --exec 'eth.blockNumber' console 2> /dev/null)
# 0 is genesis.. shouldn't dump that
blockstart=1
while [ "$blockstart" -lt "$maxblocks" ]
do
- toblock=$(( blockstart + delta - 1 ))
+ toblock=$(( blockstart + million - 1 ))
test "$toblock" -gt "$maxblocks" &&
toblock=$maxblocks
printf -v filename 'bfa2018.blocks.%09d-%09d.export.gz' "$blockstart" "$toblock"
@@ -194,28 +194,43 @@ function exportdb
register_subcommand 'importdb' 'importdb' 'Import blocks safely from previous block exports.'
function importdb
{
- local dumpurl delta blockstart toblock
+ local dumpurl million blockstart continue_at haveblocks theirsize oursize
dumpurl="https://s3.wasabisys.com/bfa/blockdumps"
- delta=1000000
+ million=1000000
graceful
- blockstart=1
+ haveblocks=$(bfageth --exec 'eth.blockNumber' console 2> /dev/null)
+ blockstart=$((haveblocks/million*million+1))
while :
do
- toblock=$(( blockstart + delta - 1 ))
- printf -v filename 'bfa2018.blocks.%09d-%09d.export.gz' "$blockstart" "$toblock"
- curl --fail "${dumpurl}/${filename}" || break
- blockstart=$(( toblock + 1 ))
+ printf -v filename 'bfa2018.blocks.%09d-%09d.export.gz' "$blockstart" "$((blockstart + million - 1))"
+ blockstart=$((blockstart + million))
+ read -r theirsize < <( curl --head "${dumpurl}/${filename}" 2>/dev/null | grep -i '^Content-Length: ' | head -1 )
+ if [ -z "$theirsize" ]
+ then
+ break
+ else
+ theirsize="${theirsize//*: }"
+ # remove trailing newline
+ theirsize="${theirsize%$'\r'}"
+ fi
+ #
+ if [ -r "$filename" ]
+ then
+ oursize=$(stat -c %s "$filename")
+ continue_at="--continue-at $oursize"
+ else
+ oursize=0
+ continue_at=
+ fi
+ if [ "${oursize:-0}" -lt "$theirsize" ]
+ then
+ echo "Downloading $filename"
+ curl --fail $continue_at -o "$filename" "${dumpurl}/${filename}" || break
+ fi
+ # shellcheck disable=SC2086
+ bfageth import <(gzip -dc "$filename")
+ rm -f "$filename"
done
- geth --networkid "${BFANETWORKID}" --datadir "${BFANODEDIR}" --syncmode "full" --gcmode "archive" import <(
- n=1
- printf -v filename 'bfa2018.blocks.%09d-%09d.export.gz' "$n" "$((n + delta - 1))"
- while gzip -dc "$filename" 2>/dev/null
- do
- n=$((n + delta))
- printf -v filename 'bfa2018.blocks.%09d-%09d.export.gz' "$n" "$((n + delta - 1))"
- done
- true
- )
}
register_subcommand 'syncmode' 'admin_syncmode' 'Set initial synchronization mode.'
@@ -381,7 +396,7 @@ function bfageth
echo "You should not run this command without any arguments. Start the background processes with start.sh instead." >&2
exit 1
fi
- exec geth --config "${BFANETWORKDIR}/config.localhost+full+archive" "$@"
+ geth --config "${BFANETWORKDIR}/config.localhost+full+archive" "$@"
}
function bfaadmin
@@ -427,7 +442,7 @@ function main
test -n "$cmd" || _usage
test -n "${commands[$cmd]}" || _usage
# shellcheck disable=SC2086
- eval ${commands[$cmd]} "$*"
+ eval ${commands[$cmd]} "$@"
;;
'admin.sh')
bfaadmin "$@"
@@ -441,4 +456,4 @@ function main
esac
}
-main "$*"
+main "$@"
--
GitLab