avbtool: Add retry to generate_fec_data()

When signing multiple builds at once, we can run out of memory during
this step, let's just attempt to retry it up to 3 times when that
happens.

Change-Id: I308b37823ea4682fb14181735d94be02eb916e5b
diff --git a/avbtool.py b/avbtool.py
index 6ae089f..aceb97f 100755
--- a/avbtool.py
+++ b/avbtool.py
@@ -31,6 +31,8 @@
 import json
 import math
 import os
+import random
+import signal
 import struct
 import subprocess
 import sys
@@ -4129,7 +4131,7 @@
   return int(pout)
 
 
-def generate_fec_data(image_filename, num_roots):
+def generate_fec_data(image_filename, num_roots, attempt=1):
   """Generate FEC codes for an image.
 
   Arguments:
@@ -4149,6 +4151,11 @@
            fec_tmpfile.name],
           stderr=open(os.devnull, 'wb'))
     except subprocess.CalledProcessError as e:
+      if attempt < 3 and e.returncode == -signal.SIGKILL:
+        seconds = random.randrange(30, 120)
+        print('avbtool: fec died, retrying in', seconds, 'seconds')
+        time.sleep(seconds)
+        return generate_fec_data(image_filename, num_roots, attempt + 1)
       raise ValueError('Execution of \'fec\' tool failed: {}.'
                        .format(e)) from e
     fec_data = fec_tmpfile.read()