-
Notifications
You must be signed in to change notification settings - Fork 3
/
dynamodb-dump
executable file
·107 lines (82 loc) · 2.61 KB
/
dynamodb-dump
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
#!/usr/bin/env bash
# Prerequisites:
# install aws cli
# install jq from https://stedolan.github.io/jq/
S3_FOLDER=${1:?Error: provide the S3 path as the first argument}
TOPIC=${2:?Error: provide the SNS ARN to alert to} # arn:aws:sns:eu-central-1:165664414043:witan-to-slack
TABLE_NAME_TEST=${3:?Error: provide at least one table name as the second argument}
NUM_ITEMS=25 # do not change
type aws
if (($? > 0)); then
printf '%s\n' 'No AWS CLI installed' >&2
exit 1
fi
type jq
if (($? > 0)); then
printf '%s\n' 'No jq installed' >&2
exit 1
fi
current_dir=$(pwd)
cd $(mktemp -d)
dynamo_scan() {
local result=""
if [ -z "$2" ]
then
result=$(aws dynamodb scan --table-name $1 --max-items $NUM_ITEMS --page-size $NUM_ITEMS)
error=$?
else
result=$(aws dynamodb scan --table-name $1 --max-items $NUM_ITEMS --page-size $NUM_ITEMS --starting-token $2)
error=$?
fi
echo $result | jq -c '[{Item: .Items[]} | {PutRequest: .}]' | jq -c "{\"$var\": .}" > $var-$file_num
next_token=$(echo $result | jq .NextToken)
}
set_expected_page_count() {
item_count=$(aws dynamodb describe-table --table-name $1 | jq .Table.ItemCount)
remainder=$(expr $item_count % $NUM_ITEMS)
expected_page_count=$(expr $item_count / $NUM_ITEMS)
if [ ! -z $remainder ]
then
expected_page_count=$((expected_page_count+1))
fi
}
sns-alert() {
local table=$1
local error=${@:2}
result=$(aws sns publish --topic $TOPIC --message "{\"AlarmName\": \"the backup of $table\", \"NewStateValue\": \"failed\", \"NewStateReason\": \"error code $error\"}")
msg_id=$(echo $result | jq .MessageId)
echo "Error for table: $1. Msg_id: $msg_id. Msg: $error"
}
for var in "${@:3}"
do
echo "Exporting table $var"
file_num=1
next_token="null"
error=0
expected_page_count=0
set_expected_page_count $var
echo "Expected page count: $expected_page_count"
echo "Downloading segment: $file_num"
dynamo_scan $var
while [[ "$next_token" != "null" && "$error" -eq 0 ]]; do
file_num=$((file_num+1))
sleep 1
echo "Downloading segment: $file_num, with error: $error, token: $next_token"
dynamo_scan $var $next_token
done
if [ $expected_page_count -gt $file_num ]
then
error="Expected page count not reached. Expected $expected_page_count. Received: $file_num"
fi
if [ "$error" != 0 ]
then
sns-alert $var $error
fi
done
echo "Uploading backup"
aws s3 sync . s3://$S3_FOLDER/$(date +%Y-%m-%d-%H-%M-%S)
error=$?
if [ "$error" -ne 0 ]; then
sns-alert S3 $error
fi
cd $current_dir