MA-16048 Fix imx8q u-boot hang

Address 0x8880_0000 is reserved for M4 image on imx8q, which
leaves limited memory region for the malloc pool. The avb
will consume much heap memory to verify the kernel and dtbo
image, memory conflicts may happen as the kernel/dtbo image
size is getting larger.

As the avb will load kernel/dtbo in every avb_slot_verify(), but
will only free the memory after both slots are checked(if needed).
And for trusty enabled platforms, extra heap memory will be used
to do the hash calculation.

This commit will free the slot memory once it's marked as unbootable
and will use fixed memory started from CONFIG_FASTBOOT_BUF_ADDR to
help store the data to do the hash calculation. With above change,
we get a chance to decrease the malloc pool size.

Test: boot on imx8qxp and imx8mn.

Change-Id: Id95b9becf6fa7fae4f120a4dc9e9a3ba90da578c
Signed-off-by: Ji Luo <ji.luo@nxp.com>

17 files changed, 22 insertions, 29 deletions

diff --git a/configs/imx8mq_aiy_android_defconfig b/configs/imx8mq_aiy_android_defconfig
index fc59c08be62..8a0819c7399 100644
--- a/configs/imx8mq_aiy_android_defconfig
+++ b/configs/imx8mq_aiy_android_defconfig
@@ -54,7 +54,7 @@ CONFIG_CMD_FASTBOOT=y
 CONFIG_ANDROID_BOOT_IMAGE=y
 CONFIG_FSL_FASTBOOT=y
 CONFIG_BCB_SUPPORT=y
-CONFIG_FASTBOOT_BUF_ADDR=0x42800000
+CONFIG_FASTBOOT_BUF_ADDR=0x44800000
 CONFIG_FASTBOOT_BUF_SIZE=0x19000000
 CONFIG_FASTBOOT_FLASH=y
 CONFIG_FASTBOOT_FLASH_MMC_DEV=0
diff --git a/configs/imx8mq_aiy_android_trusty_defconfig b/configs/imx8mq_aiy_android_trusty_defconfig
index b6abbc55346..f90fb6e1ce3 100644
--- a/configs/imx8mq_aiy_android_trusty_defconfig
+++ b/configs/imx8mq_aiy_android_trusty_defconfig
@@ -54,7 +54,7 @@ CONFIG_CMD_FASTBOOT=y
 CONFIG_ANDROID_BOOT_IMAGE=y
 CONFIG_FSL_FASTBOOT=y
 CONFIG_BCB_SUPPORT=y
-CONFIG_FASTBOOT_BUF_ADDR=0x42800000
+CONFIG_FASTBOOT_BUF_ADDR=0x44800000
 CONFIG_FASTBOOT_BUF_SIZE=0x19000000
 CONFIG_FASTBOOT_FLASH=y
 CONFIG_FASTBOOT_FLASH_MMC_DEV=0
diff --git a/configs/imx8mq_aiy_android_trusty_dual_defconfig b/configs/imx8mq_aiy_android_trusty_dual_defconfig
index 4987c0232fe..10b13afab11 100644
--- a/configs/imx8mq_aiy_android_trusty_dual_defconfig
+++ b/configs/imx8mq_aiy_android_trusty_dual_defconfig
@@ -54,7 +54,7 @@ CONFIG_CMD_FASTBOOT=y
 CONFIG_ANDROID_BOOT_IMAGE=y
 CONFIG_FSL_FASTBOOT=y
 CONFIG_BCB_SUPPORT=y
-CONFIG_FASTBOOT_BUF_ADDR=0x42800000
+CONFIG_FASTBOOT_BUF_ADDR=0x44800000
 CONFIG_FASTBOOT_BUF_SIZE=0x19000000
 CONFIG_FASTBOOT_FLASH=y
 CONFIG_FASTBOOT_FLASH_MMC_DEV=0
diff --git a/configs/imx8mq_aiy_android_uuu_defconfig b/configs/imx8mq_aiy_android_uuu_defconfig
index 0f1a88dc284..63abbb4ee4f 100644
--- a/configs/imx8mq_aiy_android_uuu_defconfig
+++ b/configs/imx8mq_aiy_android_uuu_defconfig
@@ -54,7 +54,7 @@ CONFIG_CMD_FASTBOOT=y
 CONFIG_ANDROID_BOOT_IMAGE=y
 CONFIG_FSL_FASTBOOT=y
 CONFIG_BCB_SUPPORT=y
-CONFIG_FASTBOOT_BUF_ADDR=0x42800000
+CONFIG_FASTBOOT_BUF_ADDR=0x44800000
 CONFIG_FASTBOOT_BUF_SIZE=0x19000000
 CONFIG_FASTBOOT_FLASH=y
 CONFIG_FASTBOOT_FLASH_MMC_DEV=0
diff --git a/configs/imx8mq_evk_android_defconfig b/configs/imx8mq_evk_android_defconfig
index 38705418ec1..a6b2c0ee952 100644
--- a/configs/imx8mq_evk_android_defconfig
+++ b/configs/imx8mq_evk_android_defconfig
@@ -58,7 +58,7 @@ CONFIG_CMD_FASTBOOT=y
 CONFIG_ANDROID_BOOT_IMAGE=y
 CONFIG_FSL_FASTBOOT=y
 CONFIG_BCB_SUPPORT=y
-CONFIG_FASTBOOT_BUF_ADDR=0x42800000
+CONFIG_FASTBOOT_BUF_ADDR=0x44800000
 CONFIG_FASTBOOT_BUF_SIZE=0x40000000
 CONFIG_FASTBOOT_FLASH=y
 CONFIG_FASTBOOT_FLASH_MMC_DEV=0
diff --git a/configs/imx8mq_evk_android_dual_defconfig b/configs/imx8mq_evk_android_dual_defconfig
index 70dedd9ed08..f0f5abf32ed 100644
--- a/configs/imx8mq_evk_android_dual_defconfig
+++ b/configs/imx8mq_evk_android_dual_defconfig
@@ -58,7 +58,7 @@ CONFIG_CMD_FASTBOOT=y
 CONFIG_ANDROID_BOOT_IMAGE=y
 CONFIG_FSL_FASTBOOT=y
 CONFIG_BCB_SUPPORT=y
-CONFIG_FASTBOOT_BUF_ADDR=0x42800000
+CONFIG_FASTBOOT_BUF_ADDR=0x44800000
 CONFIG_FASTBOOT_BUF_SIZE=0x40000000
 CONFIG_FASTBOOT_FLASH=y
 CONFIG_FASTBOOT_FLASH_MMC_DEV=0
diff --git a/configs/imx8mq_evk_android_uuu_defconfig b/configs/imx8mq_evk_android_uuu_defconfig
index 9bada1f0022..892233c9c23 100644
--- a/configs/imx8mq_evk_android_uuu_defconfig
+++ b/configs/imx8mq_evk_android_uuu_defconfig
@@ -58,7 +58,7 @@ CONFIG_CMD_FASTBOOT=y
 CONFIG_ANDROID_BOOT_IMAGE=y
 CONFIG_FSL_FASTBOOT=y
 CONFIG_BCB_SUPPORT=y
-CONFIG_FASTBOOT_BUF_ADDR=0x42800000
+CONFIG_FASTBOOT_BUF_ADDR=0x44800000
 CONFIG_FASTBOOT_BUF_SIZE=0x40000000
 CONFIG_FASTBOOT_FLASH=y
 CONFIG_FASTBOOT_FLASH_MMC_DEV=0
diff --git a/include/configs/imx8mm_evk_android.h b/include/configs/imx8mm_evk_android.h
index ddfda6cba2b..eeba699ad31 100644
--- a/include/configs/imx8mm_evk_android.h
+++ b/include/configs/imx8mm_evk_android.h
@@ -24,7 +24,7 @@
 
 #ifdef CONFIG_SYS_MALLOC_LEN
 #undef CONFIG_SYS_MALLOC_LEN
-#define CONFIG_SYS_MALLOC_LEN           (96 * SZ_1M)
+#define CONFIG_SYS_MALLOC_LEN           (64 * SZ_1M)
 #endif
 
 #define CONFIG_USB_FUNCTION_FASTBOOT
@@ -36,7 +36,7 @@
 #define CONFIG_FSL_FASTBOOT
 #define CONFIG_ANDROID_RECOVERY
 
-#define CONFIG_FASTBOOT_BUF_ADDR   CONFIG_SYS_LOAD_ADDR
+#define CONFIG_FASTBOOT_BUF_ADDR   0x44800000
 #define CONFIG_FASTBOOT_BUF_SIZE   0x19000000
 
 #define CONFIG_CMD_BOOTA
diff --git a/include/configs/imx8mn_evk_android.h b/include/configs/imx8mn_evk_android.h
index 99d28186dc2..88aec400054 100644
--- a/include/configs/imx8mn_evk_android.h
+++ b/include/configs/imx8mn_evk_android.h
@@ -24,7 +24,7 @@
 
 #ifdef CONFIG_SYS_MALLOC_LEN
 #undef CONFIG_SYS_MALLOC_LEN
-#define CONFIG_SYS_MALLOC_LEN           (96 * SZ_1M)
+#define CONFIG_SYS_MALLOC_LEN           (64 * SZ_1M)
 #endif
 
 #define CONFIG_USB_FUNCTION_FASTBOOT
@@ -36,7 +36,7 @@
 #define CONFIG_FSL_FASTBOOT
 #define CONFIG_ANDROID_RECOVERY
 
-#define CONFIG_FASTBOOT_BUF_ADDR   CONFIG_SYS_LOAD_ADDR
+#define CONFIG_FASTBOOT_BUF_ADDR   0x44800000
 #define CONFIG_FASTBOOT_BUF_SIZE   0x19000000
 
 #define CONFIG_CMD_BOOTA
diff --git a/include/configs/imx8mq_aiy_android.h b/include/configs/imx8mq_aiy_android.h
index 7d1e6945fee..1f31be00966 100644
--- a/include/configs/imx8mq_aiy_android.h
+++ b/include/configs/imx8mq_aiy_android.h
@@ -29,7 +29,7 @@
 
 #ifdef CONFIG_SYS_MALLOC_LEN
 #undef CONFIG_SYS_MALLOC_LEN
-#define CONFIG_SYS_MALLOC_LEN           (96 * SZ_1M)
+#define CONFIG_SYS_MALLOC_LEN           (64 * SZ_1M)
 #endif
 
 #define CONFIG_ANDROID_RECOVERY
diff --git a/include/configs/imx8mq_evk_android.h b/include/configs/imx8mq_evk_android.h
index 066fb15343c..805b2c45407 100644
--- a/include/configs/imx8mq_evk_android.h
+++ b/include/configs/imx8mq_evk_android.h
@@ -23,7 +23,7 @@
 
 #ifdef CONFIG_SYS_MALLOC_LEN
 #undef CONFIG_SYS_MALLOC_LEN
-#define CONFIG_SYS_MALLOC_LEN           (96 * SZ_1M)
+#define CONFIG_SYS_MALLOC_LEN           (64 * SZ_1M)
 #endif
 
 #define CONFIG_ANDROID_RECOVERY
diff --git a/include/configs/imx8qm_mek_android.h b/include/configs/imx8qm_mek_android.h
index 969c66f5041..2446987df4a 100644
--- a/include/configs/imx8qm_mek_android.h
+++ b/include/configs/imx8qm_mek_android.h
@@ -39,7 +39,7 @@
 
 #ifdef CONFIG_SYS_MALLOC_LEN
 #undef CONFIG_SYS_MALLOC_LEN
-#define CONFIG_SYS_MALLOC_LEN           (86 * SZ_1M)
+#define CONFIG_SYS_MALLOC_LEN           (64 * SZ_1M)
 #endif
 
 #define CONFIG_USB_FUNCTION_FASTBOOT
diff --git a/include/configs/imx8qm_mek_android_auto.h b/include/configs/imx8qm_mek_android_auto.h
index e7b29276bb8..760052b7bef 100644
--- a/include/configs/imx8qm_mek_android_auto.h
+++ b/include/configs/imx8qm_mek_android_auto.h
@@ -42,7 +42,7 @@
 
 #ifdef CONFIG_SYS_MALLOC_LEN
 #undef CONFIG_SYS_MALLOC_LEN
-#define CONFIG_SYS_MALLOC_LEN           (76 * SZ_1M)
+#define CONFIG_SYS_MALLOC_LEN           (64 * SZ_1M)
 #endif
 
 #define CONFIG_USB_FUNCTION_FASTBOOT
diff --git a/include/configs/imx8qxp_mek_android.h b/include/configs/imx8qxp_mek_android.h
index 838223ab479..19f567c42df 100644
--- a/include/configs/imx8qxp_mek_android.h
+++ b/include/configs/imx8qxp_mek_android.h
@@ -34,7 +34,7 @@
 
 #ifdef CONFIG_SYS_MALLOC_LEN
 #undef CONFIG_SYS_MALLOC_LEN
-#define CONFIG_SYS_MALLOC_LEN           (86 * SZ_1M)
+#define CONFIG_SYS_MALLOC_LEN           (64 * SZ_1M)
 #endif
 
 #define CONFIG_USB_FUNCTION_FASTBOOT
diff --git a/include/configs/imx8qxp_mek_android_auto.h b/include/configs/imx8qxp_mek_android_auto.h
index 2394f791fcd..bf445fdc29a 100644
--- a/include/configs/imx8qxp_mek_android_auto.h
+++ b/include/configs/imx8qxp_mek_android_auto.h
@@ -41,7 +41,7 @@
 
 #ifdef CONFIG_SYS_MALLOC_LEN
 #undef CONFIG_SYS_MALLOC_LEN
-#define CONFIG_SYS_MALLOC_LEN           (76 * SZ_1M)
+#define CONFIG_SYS_MALLOC_LEN           (64 * SZ_1M)
 #endif
 
 #define CONFIG_USB_FUNCTION_FASTBOOT
diff --git a/lib/avb/fsl/fsl_avb_ab_flow.c b/lib/avb/fsl/fsl_avb_ab_flow.c
index 5637212157f..091304b67c6 100644
--- a/lib/avb/fsl/fsl_avb_ab_flow.c
+++ b/lib/avb/fsl/fsl_avb_ab_flow.c
@@ -1168,6 +1168,10 @@ AvbABFlowResult avb_ab_flow_fast(AvbABOps* ab_ops,
 				fsl_slot_set_unbootable(&ab_data.slots[target_slot]);
 				set_slot_unbootable = false;
 			}
+			if (slot_data[target_slot] != NULL) {
+				avb_slot_verify_data_free(slot_data[target_slot]);
+				slot_data[target_slot] = NULL;
+			}
 		}
 		/* switch to another slot */
 		target_slot = (target_slot == 1 ? 0 : 1);
diff --git a/lib/avb/libavb/avb_slot_verify.c b/lib/avb/libavb/avb_slot_verify.c
index 2ca4ff9bd33..f682007a5e8 100644
--- a/lib/avb/libavb/avb_slot_verify.c
+++ b/lib/avb/libavb/avb_slot_verify.c
@@ -313,12 +313,7 @@ static AvbSlotVerifyResult load_and_verify_hash_partition(
     }
     uint32_t round_buf_size = ROUND(hash_desc.salt_len + hash_desc.image_size,
                                 ARCH_DMA_MINALIGN);
-    hash_buf = memalign(ARCH_DMA_MINALIGN, round_buf_size);
-    if (hash_buf == NULL) {
-        avb_error("failed to alloc memory!\n");
-        ret = AVB_SLOT_VERIFY_RESULT_ERROR_OOM;
-        goto out;
-    }
+    hash_buf = (void *)CONFIG_FASTBOOT_BUF_ADDR;
 
     avb_memcpy(hash_buf, desc_salt, hash_desc.salt_len);
     avb_memcpy(hash_buf + hash_desc.salt_len,
@@ -335,8 +330,6 @@ static AvbSlotVerifyResult load_and_verify_hash_partition(
     }
 
     digest = hash_out;
-    free(hash_buf);
-    hash_buf = NULL;
 #else
     AvbSHA256Ctx sha256_ctx;
     avb_sha256_init(&sha256_ctx);
@@ -399,10 +392,6 @@ out:
     free(hash_out);
     hash_out = NULL;
   }
-  if (hash_buf != NULL) {
-    free(hash_buf);
-    hash_buf = NULL;
-  }
 #endif
   /* If it worked and something was loaded, copy to slot_data. */
   if ((ret == AVB_SLOT_VERIFY_RESULT_OK || result_should_continue(ret)) &&