src/soc/mediatek/common/memory.c

   1 /* SPDX-License-Identifier: GPL-2.0-only */
   2
   3 #include <assert.h>
   4 #include <cbfs.h>
   5 #include <cbmem.h>
   6 #include <commonlib/bsd/mem_chip_info.h>
   7 #include <console/console.h>
   8 #include <fmap_config.h>
   9 #include <soc/dramc_common.h>
  10 #include <mrc_cache.h>
  11 #include <soc/dramc_param.h>
  12 #include <soc/emi.h>
  13 #include <soc/mmu_operations.h>
  14 #include <symbols.h>
  15 #include <timer.h>
  16
  17 _Static_assert(sizeof(struct dramc_param) <= FMAP_SECTION_RW_MRC_CACHE_SIZE,
  18                "sizeof(struct dramc_param) exceeds RW_MRC_CACHE size");
  19
  20 const char *get_dram_geometry_str(u32 ddr_geometry);
  21 const char *get_dram_type_str(u32 ddr_type);
  22
  23 static const struct ddr_base_info *curr_ddr_info;
  24
  25 static int mt_mem_test(const struct dramc_data *dparam)
  26 {
  27         if (CONFIG(MEMORY_TEST)) {
  28                 u8 *addr = _dram;
  29                 const struct ddr_base_info *ddr_info = &dparam->ddr_info;
  30
  31                 for (u8 rank = RANK_0; rank < ddr_info->support_ranks; rank++) {
  32                         int result = complex_mem_test(addr, 0x2000);
  33
  34                         if (result != 0) {
  35                                 printk(BIOS_ERR,
  36                                        "[MEM] complex R/W mem test failed: %d\n", result);
  37                                 return -1;
  38                         }
  39                         printk(BIOS_DEBUG, "[MEM] rank %u complex R/W mem test passed\n", rank);
  40
  41                         addr += ddr_info->rank_size[rank];
  42                 }
  43         }
  44
  45         return 0;
  46 }
  47
  48 const char *get_dram_geometry_str(u32 ddr_geometry)
  49 {
  50         const char *s;
  51
  52         switch (ddr_geometry) {
  53         case DDR_TYPE_2CH_2RK_4GB_2_2:
  54                 s = "2CH_2RK_4GB_2_2";
  55                 break;
  56         case DDR_TYPE_2CH_2RK_6GB_3_3:
  57                 s = "2CH_2RK_6GB_3_3";
  58                 break;
  59         case DDR_TYPE_2CH_2RK_8GB_4_4:
  60                 s = "2CH_2RK_8GB_4_4";
  61                 break;
  62         case DDR_TYPE_2CH_2RK_8GB_4_4_BYTE:
  63                 s = "2CH_2RK_8GB_4_4_BYTE";
  64                 break;
  65         case DDR_TYPE_2CH_1RK_4GB_4_0:
  66                 s = "2CH_1RK_4GB_4_0";
  67                 break;
  68         case DDR_TYPE_2CH_2RK_6GB_2_4:
  69                 s = "2CH_2RK_6GB_2_4";
  70                 break;
  71         default:
  72                 s = "";
  73                 break;
  74         }
  75
  76         return s;
  77 }
  78
  79 const char *get_dram_type_str(u32 ddr_type)
  80 {
  81         const char *s;
  82
  83         switch (ddr_type) {
  84         case DDR_TYPE_DISCRETE:
  85                 s = "DSC";
  86                 break;
  87         case DDR_TYPE_EMCP:
  88                 s = "EMCP";
  89                 break;
  90         default:
  91                 s = "";
  92                 break;
  93         }
  94
  95         return s;
  96 }
  97
  98 size_t mtk_dram_size(void)
  99 {
 100         size_t size = 0;
 101
 102         if (!curr_ddr_info)
 103                 return 0;
 104         for (unsigned int i = 0; i < RANK_MAX; ++i)
 105                 size += curr_ddr_info->mrr_info.mr8_density[i];
 106         return size;
 107 }
 108
 109 __weak enum mem_chip_type map_to_lpddr_dram_type(uint16_t lpddr_type)
 110 {
 111         return MEM_CHIP_LPDDR4X;
 112 }
 113
 114 static void fill_dram_info(struct mem_chip_info *mc, const struct ddr_base_info *ddr)
 115 {
 116         unsigned int c, r;
 117
 118         mc->num_entries = CHANNEL_MAX * ddr->mrr_info.rank_nums;
 119         mc->struct_version = MEM_CHIP_STRUCT_VERSION;
 120
 121         struct mem_chip_entry *entry = mc->entries;
 122         for (c = 0; c < CHANNEL_MAX; c++) {
 123                 for (r = 0; r < ddr->mrr_info.rank_nums; r++) {
 124                         entry->channel = c;
 125                         entry->rank = r;
 126                         entry->type = map_to_lpddr_dram_type(ddr->lpddr_type);
 127                         entry->channel_io_width = DQ_DATA_WIDTH_LP4;
 128                         entry->density_mbits = ddr->mrr_info.mr8_density[r] / CHANNEL_MAX /
 129                                                (MiB / 8);
 130                         entry->io_width = DQ_DATA_WIDTH_LP4;
 131                         entry->manufacturer_id = ddr->mrr_info.mr5_vendor_id;
 132                         entry->revision_id[0] = ddr->mrr_info.mr6_revision_id;
 133                         entry->revision_id[1] = ddr->mrr_info.mr7_revision_id;
 134                         entry++;
 135                 }
 136         }
 137 }
 138
 139 static void add_mem_chip_info(int unused)
 140 {
 141         struct mem_chip_info *mc;
 142         size_t size;
 143
 144         if (!CONFIG(USE_CBMEM_DRAM_INFO)) {
 145                 printk(BIOS_DEBUG,
 146                        "DRAM-K: CBMEM DRAM info is unsupported (USE_CBMEM_DRAM_INFO)\n");
 147                 return;
 148         }
 149
 150         size = mem_chip_info_size(CHANNEL_MAX * curr_ddr_info->mrr_info.rank_nums);
 151         mc = cbmem_add(CBMEM_ID_MEM_CHIP_INFO, size);
 152         assert(mc);
 153         memset(mc, 0, size);
 154
 155         fill_dram_info(mc, curr_ddr_info);
 156 }
 157 CBMEM_CREATION_HOOK(add_mem_chip_info);
 158
 159 static int run_dram_blob(struct dramc_param *dparam)
 160 {
 161         /* Load and run the provided blob for full-calibration if available */
 162         struct prog dram = PROG_INIT(PROG_REFCODE, CONFIG_CBFS_PREFIX "/dram");
 163
 164         dump_param_header(dparam);
 165
 166         if (cbfs_prog_stage_load(&dram)) {
 167                 printk(BIOS_ERR, "DRAM-K: CBFS load program failed\n");
 168                 return -2;
 169         }
 170
 171         dparam->do_putc = do_putchar;
 172
 173         prog_set_entry(&dram, prog_entry(&dram), dparam);
 174         prog_run(&dram);
 175         if (dparam->header.status != DRAMC_SUCCESS) {
 176                 printk(BIOS_ERR, "DRAM-K: calibration failed: status = %d\n",
 177                        dparam->header.status);
 178                 return -3;
 179         }
 180
 181         if (!(dparam->header.config & DRAMC_CONFIG_FAST_K)
 182             && !(dparam->header.flags & DRAMC_FLAG_HAS_SAVED_DATA)) {
 183                 printk(BIOS_ERR,
 184                        "DRAM-K: Full calibration executed without saving parameters. "
 185                        "Please ensure the blob is built properly.\n");
 186                 return -4;
 187         }
 188
 189         return 0;
 190 }
 191
 192 static int dram_run_fast_calibration(struct dramc_param *dparam)
 193 {
 194         const u16 config = CONFIG(MEDIATEK_DRAM_DVFS) ? DRAMC_ENABLE_DVFS : DRAMC_DISABLE_DVFS;
 195
 196         if (dparam->dramc_datas.ddr_info.config_dvfs != config) {
 197                 printk(BIOS_WARNING,
 198                        "DRAM-K: Incompatible config for calibration data from flash "
 199                        "(expected: %#x, saved: %#x)\n",
 200                        config, dparam->dramc_datas.ddr_info.config_dvfs);
 201                 return -1;
 202         }
 203
 204         printk(BIOS_INFO, "DRAM-K: DRAM calibration data valid pass\n");
 205
 206         if (CONFIG(MEDIATEK_DRAM_SCRAMBLE))
 207                 dparam->header.config |= DRAMC_CONFIG_SCRAMBLE;
 208         if (CONFIG(MEDIATEK_DRAM_BLOB_FAST_INIT)) {
 209                 printk(BIOS_INFO, "DRAM-K: Run fast calibration run in blob mode\n");
 210
 211                 /*
 212                  * The loaded config should not contain FAST_K (done in full calibration),
 213                  * so we have to set that now to indicate the blob taking the config instead
 214                  * of generating a new config.
 215                  */
 216                 dparam->header.config |= DRAMC_CONFIG_FAST_K;
 217
 218                 if (run_dram_blob(dparam) < 0)
 219                         return -3;
 220         } else {
 221                 init_dram_by_params(dparam);
 222         }
 223
 224         if (mt_mem_test(&dparam->dramc_datas) < 0)
 225                 return -4;
 226
 227         return 0;
 228 }
 229
 230 static int dram_run_full_calibration(struct dramc_param *dparam)
 231 {
 232         initialize_dramc_param(dparam);
 233
 234         return run_dram_blob(dparam);
 235 }
 236
 237 static void mem_init_set_default_config(struct dramc_param *dparam,
 238                                         const struct sdram_info *dram_info)
 239 {
 240         u32 type, geometry;
 241         memset(dparam, 0, sizeof(*dparam));
 242
 243         type = dram_info->ddr_type;
 244         geometry = dram_info->ddr_geometry;
 245
 246         dparam->dramc_datas.ddr_info.sdram.ddr_type = type;
 247
 248         if (CONFIG(MEDIATEK_DRAM_DVFS))
 249                 dparam->dramc_datas.ddr_info.config_dvfs = DRAMC_ENABLE_DVFS;
 250         if (CONFIG(MEDIATEK_DRAM_SCRAMBLE))
 251                 dparam->header.config |= DRAMC_CONFIG_SCRAMBLE;
 252
 253         dparam->dramc_datas.ddr_info.sdram.ddr_geometry = geometry;
 254
 255         printk(BIOS_INFO, "DRAM-K: ddr_type: %s, config_dvfs: %d, ddr_geometry: %s\n",
 256                get_dram_type_str(type),
 257                dparam->dramc_datas.ddr_info.config_dvfs,
 258                get_dram_geometry_str(geometry));
 259 }
 260
 261 static void mt_mem_init_run(struct dramc_param *dparam,
 262                             const struct sdram_info *dram_info)
 263 {
 264         const ssize_t mrc_cache_size = sizeof(*dparam);
 265         ssize_t data_size;
 266         struct stopwatch sw;
 267         int ret;
 268
 269         /* Load calibration params from flash and run fast calibration */
 270         data_size = mrc_cache_load_current(MRC_TRAINING_DATA,
 271                                            DRAMC_PARAM_HEADER_VERSION,
 272                                            dparam, mrc_cache_size);
 273         if (data_size == mrc_cache_size) {
 274                 printk(BIOS_INFO, "DRAM-K: Running fast calibration\n");
 275                 stopwatch_init(&sw);
 276
 277                 ret = dram_run_fast_calibration(dparam);
 278                 if (ret != 0) {
 279                         printk(BIOS_ERR, "DRAM-K: Failed to run fast calibration "
 280                                "in %lld msecs, error: %d\n",
 281                                stopwatch_duration_msecs(&sw), ret);
 282
 283                         /* Erase flash data after fast calibration failed */
 284                         memset(dparam, 0xa5, mrc_cache_size);
 285                         mrc_cache_stash_data(MRC_TRAINING_DATA,
 286                                              DRAMC_PARAM_HEADER_VERSION,
 287                                              dparam, mrc_cache_size);
 288                 } else {
 289                         printk(BIOS_INFO, "DRAM-K: Fast calibration passed in %lld msecs\n",
 290                                stopwatch_duration_msecs(&sw));
 291                         return;
 292                 }
 293         } else {
 294                 printk(BIOS_WARNING, "DRAM-K: Invalid data in flash (size: %#zx, expected: %#zx)\n",
 295                        data_size, mrc_cache_size);
 296         }
 297
 298         /* Run full calibration */
 299         printk(BIOS_INFO, "DRAM-K: Running full calibration\n");
 300         mem_init_set_default_config(dparam, dram_info);
 301
 302         stopwatch_init(&sw);
 303         int err = dram_run_full_calibration(dparam);
 304         if (err == 0) {
 305                 printk(BIOS_INFO, "DRAM-K: Full calibration passed in %lld msecs\n",
 306                        stopwatch_duration_msecs(&sw));
 307                 mrc_cache_stash_data(MRC_TRAINING_DATA,
 308                                      DRAMC_PARAM_HEADER_VERSION,
 309                                      dparam, mrc_cache_size);
 310         } else {
 311                 printk(BIOS_ERR, "DRAM-K: Full calibration failed in %lld msecs\n",
 312                        stopwatch_duration_msecs(&sw));
 313         }
 314 }
 315
 316 void mt_mem_init(struct dramc_param *dparam)
 317 {
 318         const struct sdram_info *sdram_param = NULL;
 319         static struct sdram_info fake_sdram_param;
 320
 321         if (CONFIG(MEDIATEK_DRAM_ADAPTIVE))
 322                 sdram_param = &fake_sdram_param;
 323         else
 324                 sdram_param = get_sdram_config();
 325
 326         mt_mem_init_run(dparam, sdram_param);
 327 }
 328
 329 void mtk_dram_init(void)
 330 {
 331         /* dramc_param is too large to fit in stack. */
 332         static struct dramc_param dramc_parameter;
 333         mt_mem_init(&dramc_parameter);
 334         curr_ddr_info = &dramc_parameter.dramc_datas.ddr_info;
 335         mtk_mmu_after_dram();
 336 }