src/northbridge/intel/i945/raminit.c

   1 /*
   2  * This file is part of the coreboot project.
   3  *
   4  * Copyright (C) 2007-2009 coresystems GmbH
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation; version 2 of the License.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program; if not, write to the Free Software
  17  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
  18  */
  19
  20 #include <cpu/x86/mtrr.h>
  21 #include <cpu/x86/cache.h>
  22 #include <pc80/mc146818rtc.h>
  23 #include <spd.h>
  24 #include "raminit.h"
  25 #include "i945.h"
  26
  27 /* Debugging macros. */
  28 #if CONFIG_DEBUG_RAM_SETUP
  29 #define PRINTK_DEBUG(x...)      printk(BIOS_DEBUG, x)
  30 #else
  31 #define PRINTK_DEBUG(x...)
  32 #endif
  33
  34 #define RAM_INITIALIZATION_COMPLETE     (1 << 19)
  35
  36 #define RAM_COMMAND_SELF_REFRESH        (0x0 << 16)
  37 #define RAM_COMMAND_NOP                 (0x1 << 16)
  38 #define RAM_COMMAND_PRECHARGE           (0x2 << 16)
  39 #define RAM_COMMAND_MRS                 (0x3 << 16)
  40 #define RAM_COMMAND_EMRS                (0x4 << 16)
  41 #define RAM_COMMAND_CBR                 (0x6 << 16)
  42 #define RAM_COMMAND_NORMAL              (0x7 << 16)
  43
  44 #define RAM_EMRS_1                      (0x0 << 21)
  45 #define RAM_EMRS_2                      (0x1 << 21)
  46 #define RAM_EMRS_3                      (0x2 << 21)
  47
  48 static __attribute__((noinline)) void do_ram_command(u32 command)
  49 {
  50         u32 reg32;
  51
  52         reg32 = MCHBAR32(DCC);
  53         reg32 &= ~( (3<<21) | (1<<20) | (1<<19) | (7 << 16) );
  54         reg32 |= command;
  55
  56         /* Also set Init Complete */
  57         if (command == RAM_COMMAND_NORMAL)
  58                 reg32 |= RAM_INITIALIZATION_COMPLETE;
  59
  60         PRINTK_DEBUG("   Sending RAM command 0x%08x", reg32);
  61
  62         MCHBAR32(DCC) = reg32;  /* This is the actual magic */
  63
  64         PRINTK_DEBUG("...done\n");
  65
  66         udelay(1);
  67 }
  68
  69 static void ram_read32(u32 offset)
  70 {
  71         PRINTK_DEBUG("   ram read: %08x\n", offset);
  72
  73         read32(offset);
  74 }
  75
  76 #if CONFIG_DEBUG_RAM_SETUP
  77 static void sdram_dump_mchbar_registers(void)
  78 {
  79         int i;
  80         printk(BIOS_DEBUG, "Dumping MCHBAR Registers\n");
  81
  82         for (i=0; i<0xfff; i+=4) {
  83                 if (MCHBAR32(i) == 0)
  84                         continue;
  85                 printk(BIOS_DEBUG, "0x%04x: 0x%08x\n", i, MCHBAR32(i));
  86         }
  87 }
  88 #endif
  89
  90 static int memclk(void)
  91 {
  92         int offset = 0;
  93 #if CONFIG_I945GM
  94         offset++;
  95 #endif
  96         switch (((MCHBAR32(CLKCFG) >> 4) & 7) - offset) {
  97         case 1: return 400;
  98         case 2: return 533;
  99         case 3: return 667;
 100         default: printk(BIOS_DEBUG, "memclk: unknown register value %x\n", ((MCHBAR32(CLKCFG) >> 4) & 7) - offset);
 101         }
 102         return -1;
 103 }
 104
 105 #if CONFIG_I945GM
 106 static int fsbclk(void)
 107 {
 108         switch (MCHBAR32(CLKCFG) & 7) {
 109         case 0: return 400;
 110         case 1: return 533;
 111         case 3: return 667;
 112         default: printk(BIOS_DEBUG, "fsbclk: unknown register value %x\n", MCHBAR32(CLKCFG) & 7);
 113         }
 114         return -1;
 115 }
 116 #endif
 117 #if CONFIG_I945GC
 118 static int fsbclk(void)
 119 {
 120         switch (MCHBAR32(CLKCFG) & 7) {
 121         case 0: return 1066;
 122         case 1: return 533;
 123         case 2: return 800;
 124         default: printk(BIOS_DEBUG, "fsbclk: unknown register value %x\n", MCHBAR32(CLKCFG) & 7);
 125         }
 126         return -1;
 127 }
 128 #endif
 129
 130 static int sdram_capabilities_max_supported_memory_frequency(void)
 131 {
 132         u32 reg32;
 133
 134 #if CONFIG_MAXIMUM_SUPPORTED_FREQUENCY
 135         return CONFIG_MAXIMUM_SUPPORTED_FREQUENCY;
 136 #endif
 137
 138         reg32 = pci_read_config32(PCI_DEV(0, 0x00, 0), 0xe4); /* CAPID0 + 4 */
 139         reg32 &= (7 << 0);
 140
 141         switch (reg32) {
 142         case 4: return 400;
 143         case 3: return 533;
 144         case 2: return 667;
 145         }
 146         /* Newer revisions of this chipset rather support faster memory clocks,
 147          * so if it's a reserved value, return the fastest memory clock that we
 148          * know of and can handle
 149          */
 150         return 667;
 151 }
 152
 153 /**
 154  * @brief determine whether chipset is capable of dual channel interleaved mode
 155  *
 156  * @return 1 if interleaving is supported, 0 otherwise
 157  */
 158 static int sdram_capabilities_interleave(void)
 159 {
 160         u32 reg32;
 161
 162         reg32 = pci_read_config32(PCI_DEV(0, 0x00,0), 0xe4); /* CAPID0 + 4 */
 163         reg32 >>= 25;
 164         reg32 &= 1;
 165
 166         return (!reg32);
 167 }
 168
 169 /**
 170  * @brief determine whether chipset is capable of two memory channels
 171  *
 172  * @return 1 if dual channel operation is supported, 0 otherwise
 173  */
 174 static int sdram_capabilities_dual_channel(void)
 175 {
 176         u32 reg32;
 177
 178         reg32 = pci_read_config32(PCI_DEV(0, 0x00,0), 0xe4); /* CAPID0 + 4 */
 179         reg32 >>= 24;
 180         reg32 &= 1;
 181
 182         return (!reg32);
 183 }
 184
 185 static int sdram_capabilities_enhanced_addressing_xor(void)
 186 {
 187         u8 reg8;
 188
 189         reg8 = pci_read_config8(PCI_DEV(0, 0x00, 0), 0xe5); /* CAPID0 + 5 */
 190         reg8 &= (1 << 7);
 191
 192         return (!reg8);
 193 }
 194
 195 static int sdram_capabilities_two_dimms_per_channel(void)
 196 {
 197         u8 reg8;
 198
 199         reg8 = pci_read_config8(PCI_DEV(0, 0x00, 0), 0xe8); /* CAPID0 + 8 */
 200         reg8 &= (1 << 0);
 201
 202         return (reg8 != 0);
 203 }
 204
 205 // TODO check if we ever need this function
 206 #if 0
 207 static int sdram_capabilities_MEM4G_disable(void)
 208 {
 209         u8 reg8;
 210
 211         reg8 = pci_read_config8(PCI_DEV(0, 0x00, 0), 0xe5); /* CAPID0 + 5 */
 212         reg8 &= (1 << 0);
 213
 214         return (reg8 != 0);
 215 }
 216 #endif
 217
 218 #define GFX_FREQUENCY_CAP_166MHZ        0x04
 219 #define GFX_FREQUENCY_CAP_200MHZ        0x03
 220 #define GFX_FREQUENCY_CAP_250MHZ        0x02
 221 #define GFX_FREQUENCY_CAP_ALL           0x00
 222
 223 static int sdram_capabilities_core_frequencies(void)
 224 {
 225         u8 reg8;
 226
 227         reg8 = pci_read_config8(PCI_DEV(0, 0x00, 0), 0xe5); /* CAPID0 + 5 */
 228         reg8 &= (1 << 3) | (1 << 2) | (1 << 1);
 229         reg8 >>= 1;
 230
 231         return (reg8);
 232 }
 233
 234 static void sdram_detect_errors(struct sys_info *sysinfo)
 235 {
 236         u8 reg8;
 237         u8 do_reset = 0;
 238
 239         reg8 = pci_read_config8(PCI_DEV(0, 0x1f, 0), 0xa2);
 240
 241         if (reg8 & ((1<<7)|(1<<2))) {
 242                 if (reg8 & (1<<2)) {
 243                         printk(BIOS_DEBUG, "SLP S4# Assertion Width Violation.\n");
 244                         /* Write back clears bit 2 */
 245                         pci_write_config8(PCI_DEV(0, 0x1f, 0), 0xa2, reg8);
 246                         do_reset = 1;
 247
 248                 }
 249
 250                 if (reg8 & (1<<7)) {
 251                         printk(BIOS_DEBUG, "DRAM initialization was interrupted.\n");
 252                         reg8 &= ~(1<<7);
 253                         pci_write_config8(PCI_DEV(0, 0x1f, 0), 0xa2, reg8);
 254                         do_reset = 1;
 255                 }
 256
 257                 /* Set SLP_S3# Assertion Stretch Enable */
 258                 reg8 = pci_read_config8(PCI_DEV(0, 0x1f, 0), 0xa4); /* GEN_PMCON_3 */
 259                 reg8 |= (1 << 3);
 260                 pci_write_config8(PCI_DEV(0, 0x1f, 0), 0xa4, reg8);
 261
 262                 if (do_reset) {
 263                         printk(BIOS_DEBUG, "Reset required.\n");
 264                         outb(0x00, 0xcf9);
 265                         outb(0x0e, 0xcf9);
 266                         for (;;) asm("hlt"); /* Wait for reset! */
 267                 }
 268         }
 269
 270         /* Set DRAM initialization bit in ICH7 */
 271         reg8 = pci_read_config8(PCI_DEV(0, 0x1f, 0), 0xa2);
 272         reg8 |= (1<<7);
 273         pci_write_config8(PCI_DEV(0, 0x1f, 0), 0xa2, reg8);
 274
 275         /* clear self refresh if not wake-up from suspend */
 276         if (sysinfo->boot_path != 2) {
 277                 MCHBAR8(0xf14) |= 3;
 278         } else {
 279                 /* Validate self refresh config */
 280                 if (((sysinfo->dimm[0] != SYSINFO_DIMM_NOT_POPULATED) ||
 281                      (sysinfo->dimm[1] != SYSINFO_DIMM_NOT_POPULATED)) &&
 282                     !(MCHBAR8(0xf14) & (1<<0))) {
 283                         do_reset = 1;
 284                 }
 285                 if (((sysinfo->dimm[2] != SYSINFO_DIMM_NOT_POPULATED) ||
 286                      (sysinfo->dimm[3] != SYSINFO_DIMM_NOT_POPULATED)) &&
 287                     !(MCHBAR8(0xf14) & (1<<1))) {
 288                         do_reset = 1;
 289                 }
 290         }
 291
 292         if (do_reset) {
 293                 printk(BIOS_DEBUG, "Reset required.\n");
 294                 outb(0x00, 0xcf9);
 295                 outb(0x0e, 0xcf9);
 296                 for (;;) asm("hlt"); /* Wait for reset! */
 297         }
 298 }
 299
 300 /**
 301  * @brief Get generic DIMM parameters.
 302  * @param sysinfo Central memory controller information structure
 303  *
 304  * This function gathers several pieces of information for each system DIMM:
 305  *  o DIMM width (x8 / x16)
 306  *  o DIMM sides (single sided / dual sided)
 307  *
 308  *  Also, some non-supported scenarios are detected.
 309  */
 310
 311 static void sdram_get_dram_configuration(struct sys_info *sysinfo)
 312 {
 313         u32 dimm_mask = 0;
 314         int i;
 315
 316         /**
 317          * i945 supports two DIMMs, in two configurations:
 318          *
 319          * - single channel with two dimms
 320          * - dual channel with one dimm per channel
 321          *
 322          * In practice dual channel mainboards have their spd at 0x50, 0x52
 323          * whereas single channel configurations have their spd at 0x50/x51
 324          *
 325          * The capability register knows a lot about the channel configuration
 326          * but for now we stick with the information we gather from the SPD
 327          * ROMs
 328          */
 329
 330         if (sdram_capabilities_dual_channel()) {
 331                 sysinfo->dual_channel = 1;
 332                 printk(BIOS_DEBUG, "This mainboard supports Dual Channel Operation.\n");
 333         } else {
 334                 sysinfo->dual_channel = 0;
 335                 printk(BIOS_DEBUG, "This mainboard supports only Single Channel Operation.\n");
 336         }
 337
 338         /**
 339          * Since we only support two DIMMs in total, there is a limited number
 340          * of combinations. This function returns the type of DIMMs.
 341          * return value:
 342          *   [0:7]  lower DIMM population
 343          *   [8-15] higher DIMM population
 344          *   [16]   dual channel?
 345          *
 346          * There are 5 different possible populations for a DIMM socket:
 347          * 1. x16 double sided (X16DS)
 348          * 2. x8 double sided  (X8DS)
 349          * 3. x16 single sided (X16SS)
 350          * 4. x8 double stacked (X8DDS)
 351          * 5. not populated (NC)
 352          *
 353          * For the return value we start counting at zero.
 354          *
 355          */
 356
 357         for (i=0; i<(2 * DIMM_SOCKETS); i++) {
 358                 u8 reg8, device = DIMM_SPD_BASE + i;
 359
 360                 /* Initialize the socket information with a sane value */
 361                 sysinfo->dimm[i] = SYSINFO_DIMM_NOT_POPULATED;
 362
 363                 /* Dual Channel not supported, but Channel 1? Bail out */
 364                 if (!sdram_capabilities_dual_channel() && (i >> 1))
 365                         continue;
 366
 367                 /* Two DIMMs per channel not supported, but odd DIMM number? */
 368                 if (!sdram_capabilities_two_dimms_per_channel() && (i& 1))
 369                         continue;
 370
 371                 printk(BIOS_DEBUG, "DDR II Channel %d Socket %d: ", (i >> 1), (i & 1));
 372
 373                 if (spd_read_byte(device, SPD_MEMORY_TYPE) != SPD_MEMORY_TYPE_SDRAM_DDR2) {
 374                         printk(BIOS_DEBUG, "N/A\n");
 375                         continue;
 376                 }
 377
 378                 reg8 = spd_read_byte(device, SPD_DIMM_CONFIG_TYPE);
 379                 if (reg8 == ERROR_SCHEME_ECC)
 380                         die("Error: ECC memory not supported by this chipset\n");
 381
 382                 reg8 = spd_read_byte(device, SPD_MODULE_ATTRIBUTES);
 383                 if (reg8 & MODULE_BUFFERED)
 384                         die("Error: Buffered memory not supported by this chipset\n");
 385                 if (reg8 & MODULE_REGISTERED)
 386                         die("Error: Registered memory not supported by this chipset\n");
 387
 388                 switch (spd_read_byte(device, SPD_PRIMARY_SDRAM_WIDTH)) {
 389                 case 0x08:
 390                         switch (spd_read_byte(device, SPD_NUM_DIMM_BANKS) & 0x0f) {
 391                         case 1:
 392                                 printk(BIOS_DEBUG, "x8DDS\n");
 393                                 sysinfo->dimm[i] = SYSINFO_DIMM_X8DDS;
 394                                 break;
 395                         case 0:
 396                                 printk(BIOS_DEBUG, "x8DS\n");
 397                                 sysinfo->dimm[i] = SYSINFO_DIMM_X8DS;
 398                                 break;
 399                         default:
 400                                 printk(BIOS_DEBUG, "Unsupported.\n");
 401                         }
 402                         break;
 403                 case 0x10:
 404                         switch (spd_read_byte(device, SPD_NUM_DIMM_BANKS) & 0x0f) {
 405                         case 1:
 406                                 printk(BIOS_DEBUG, "x16DS\n");
 407                                 sysinfo->dimm[i] = SYSINFO_DIMM_X16DS;
 408                                 break;
 409                         case 0:
 410                                 printk(BIOS_DEBUG, "x16SS\n");
 411                                 sysinfo->dimm[i] = SYSINFO_DIMM_X16SS;
 412                                 break;
 413                         default:
 414                                 printk(BIOS_DEBUG, "Unsupported.\n");
 415                         }
 416                         break;
 417                 default:
 418                         die("Unsupported DDR-II memory width.\n");
 419                 }
 420
 421                 dimm_mask |= (1 << i);
 422         }
 423
 424         if (!dimm_mask) {
 425                 die("No memory installed.\n");
 426         }
 427
 428         if (!(dimm_mask & ((1 << DIMM_SOCKETS) - 1))) {
 429                 printk(BIOS_INFO, "Channel 0 has no memory populated.\n");
 430         }
 431 }
 432
 433 /**
 434  * @brief determine if any DIMMs are stacked
 435  *
 436  * @param sysinfo central sysinfo data structure.
 437  */
 438 static void sdram_verify_package_type(struct sys_info * sysinfo)
 439 {
 440         int i;
 441
 442         /* Assume no stacked DIMMs are available until we find one */
 443         sysinfo->package = 0;
 444         for (i=0; i<2*DIMM_SOCKETS; i++) {
 445                 if (sysinfo->dimm[i] == SYSINFO_DIMM_NOT_POPULATED)
 446                         continue;
 447
 448                 /* Is the current DIMM a stacked DIMM? */
 449                 if (spd_read_byte(DIMM_SPD_BASE + i, SPD_NUM_DIMM_BANKS) & (1 << 4))
 450                         sysinfo->package = 1;
 451         }
 452 }
 453
 454 static u8 sdram_possible_cas_latencies(struct sys_info * sysinfo)
 455 {
 456         int i;
 457         u8 cas_mask;
 458
 459         /* Setup CAS mask with all supported CAS Latencies */
 460         cas_mask = SPD_CAS_LATENCY_DDR2_3 |
 461                    SPD_CAS_LATENCY_DDR2_4 |
 462                    SPD_CAS_LATENCY_DDR2_5;
 463
 464         for (i=0; i<2*DIMM_SOCKETS; i++) {
 465                 if (sysinfo->dimm[i] != SYSINFO_DIMM_NOT_POPULATED)
 466                         cas_mask &= spd_read_byte(DIMM_SPD_BASE + i, SPD_ACCEPTABLE_CAS_LATENCIES);
 467         }
 468
 469         if(!cas_mask) {
 470                 die("No DDR-II modules with accepted CAS latencies found.\n");
 471         }
 472
 473         return cas_mask;
 474 }
 475
 476 static void sdram_detect_cas_latency_and_ram_speed(struct sys_info * sysinfo, u8 cas_mask)
 477 {
 478         int i, j, idx;
 479         int lowest_common_cas = 0;
 480         int max_ram_speed = 0;
 481
 482         const u8 ddr2_speeds_table[] = {
 483                 0x50, 0x60,     /* DDR2 400: tCLK = 5.0ns  tAC = 0.6ns  */
 484                 0x3d, 0x50,     /* DDR2 533: tCLK = 3.75ns tAC = 0.5ns  */
 485                 0x30, 0x45,     /* DDR2 667: tCLK = 3.0ns  tAC = 0.45ns */
 486         };
 487
 488         const u8 spd_lookup_table[] = {
 489                 SPD_MIN_CYCLE_TIME_AT_CAS_MAX,  SPD_ACCESS_TIME_FROM_CLOCK,
 490                 SPD_SDRAM_CYCLE_TIME_2ND,       SPD_ACCESS_TIME_FROM_CLOCK_2ND,
 491                 SPD_SDRAM_CYCLE_TIME_3RD,       SPD_ACCESS_TIME_FROM_CLOCK_3RD
 492         };
 493
 494         switch (sdram_capabilities_max_supported_memory_frequency()) {
 495         case 400: max_ram_speed = 0; break;
 496         case 533: max_ram_speed = 1; break;
 497         case 667: max_ram_speed = 2; break;
 498         }
 499
 500         if (fsbclk() == 533)
 501                 max_ram_speed = 1;
 502
 503         sysinfo->memory_frequency = 0;
 504         sysinfo->cas = 0;
 505
 506         if (cas_mask & SPD_CAS_LATENCY_DDR2_3) {
 507                 lowest_common_cas = 3;
 508         } else if (cas_mask & SPD_CAS_LATENCY_DDR2_4) {
 509                 lowest_common_cas = 4;
 510         } else if (cas_mask & SPD_CAS_LATENCY_DDR2_5) {
 511                 lowest_common_cas = 5;
 512         }
 513         PRINTK_DEBUG("lowest common cas = %d\n", lowest_common_cas);
 514
 515         for (j = max_ram_speed; j>=0; j--) {
 516                 int freq_cas_mask = cas_mask;
 517
 518                 PRINTK_DEBUG("Probing Speed %d\n", j);
 519                 for (i=0; i<2*DIMM_SOCKETS; i++) {
 520                         int current_cas_mask;
 521
 522                         PRINTK_DEBUG("  DIMM: %d\n", i);
 523                         if (sysinfo->dimm[i] == SYSINFO_DIMM_NOT_POPULATED) {
 524                                 continue;
 525                         }
 526
 527                         current_cas_mask = spd_read_byte(DIMM_SPD_BASE + i, SPD_ACCEPTABLE_CAS_LATENCIES);
 528
 529                         while (current_cas_mask) {
 530                                 int highest_supported_cas = 0, current_cas = 0;
 531                                 PRINTK_DEBUG("    Current CAS mask: %04x; ", current_cas_mask);
 532                                 if (current_cas_mask & SPD_CAS_LATENCY_DDR2_5) {
 533                                         highest_supported_cas = 5;
 534                                 } else if (current_cas_mask & SPD_CAS_LATENCY_DDR2_4) {
 535                                         highest_supported_cas = 4;
 536                                 } else if (current_cas_mask & SPD_CAS_LATENCY_DDR2_3) {
 537                                         highest_supported_cas = 3;
 538                                 }
 539                                 if (current_cas_mask & SPD_CAS_LATENCY_DDR2_3) {
 540                                         current_cas = 3;
 541                                 } else if (current_cas_mask & SPD_CAS_LATENCY_DDR2_4) {
 542                                         current_cas = 4;
 543                                 } else if (current_cas_mask & SPD_CAS_LATENCY_DDR2_5) {
 544                                         current_cas = 5;
 545                                 }
 546
 547                                 idx = highest_supported_cas - current_cas;
 548                                 PRINTK_DEBUG("idx=%d, ", idx);
 549                                 PRINTK_DEBUG("tCLK=%x, ", spd_read_byte(DIMM_SPD_BASE + i, spd_lookup_table[2*idx]));
 550                                 PRINTK_DEBUG("tAC=%x", spd_read_byte(DIMM_SPD_BASE + i, spd_lookup_table[(2*idx)+1]));
 551
 552                                 if (spd_read_byte(DIMM_SPD_BASE + i, spd_lookup_table[2*idx]) <= ddr2_speeds_table[2*j] &&
 553                                                 spd_read_byte(DIMM_SPD_BASE + i, spd_lookup_table[(2*idx)+1]) <= ddr2_speeds_table[(2*j)+1]) {
 554                                         PRINTK_DEBUG(":    OK\n");
 555                                         break;
 556                                 }
 557
 558                                 PRINTK_DEBUG(":    Not fast enough!\n");
 559
 560                                 current_cas_mask &= ~(1 << (current_cas));
 561                         }
 562
 563                         freq_cas_mask &= current_cas_mask;
 564                         if (!current_cas_mask) {
 565                                 PRINTK_DEBUG("    No valid CAS for this speed on DIMM %d\n", i);
 566                                 break;
 567                         }
 568                 }
 569                 PRINTK_DEBUG("  freq_cas_mask for speed %d: %04x\n", j, freq_cas_mask);
 570                 if (freq_cas_mask) {
 571                         switch (j) {
 572                         case 0: sysinfo->memory_frequency = 400; break;
 573                         case 1: sysinfo->memory_frequency = 533; break;
 574                         case 2: sysinfo->memory_frequency = 667; break;
 575                         }
 576                         if (freq_cas_mask & SPD_CAS_LATENCY_DDR2_3) {
 577                                 sysinfo->cas = 3;
 578                         } else if (freq_cas_mask & SPD_CAS_LATENCY_DDR2_4) {
 579                                 sysinfo->cas = 4;
 580                         } else if (freq_cas_mask & SPD_CAS_LATENCY_DDR2_5) {
 581                                 sysinfo->cas = 5;
 582                         }
 583                         break;
 584                 }
 585         }
 586
 587         if (sysinfo->memory_frequency && sysinfo->cas) {
 588                 printk(BIOS_DEBUG, "Memory will be driven at %dMHz with CAS=%d clocks\n",
 589                                 sysinfo->memory_frequency, sysinfo->cas);
 590         } else {
 591                 die("Could not find common memory frequency and CAS\n");
 592         }
 593 }
 594
 595 static void sdram_detect_smallest_tRAS(struct sys_info * sysinfo)
 596 {
 597         int i;
 598         int tRAS_time;
 599         int tRAS_cycles;
 600         int freq_multiplier = 0;
 601
 602         switch (sysinfo->memory_frequency) {
 603         case 400: freq_multiplier = 0x14; break; /* 5ns */
 604         case 533: freq_multiplier = 0x0f; break; /* 3.75ns */
 605         case 667: freq_multiplier = 0x0c; break; /* 3ns */
 606         }
 607
 608         tRAS_cycles = 4; /* 4 clocks minimum */
 609         tRAS_time = tRAS_cycles * freq_multiplier;
 610
 611         for (i=0; i<2*DIMM_SOCKETS; i++) {
 612                 u8 reg8;
 613
 614                 if (sysinfo->dimm[i] == SYSINFO_DIMM_NOT_POPULATED)
 615                         continue;
 616
 617                 reg8 = spd_read_byte(DIMM_SPD_BASE + i, SPD_MIN_ACTIVE_TO_PRECHARGE_DELAY);
 618                 if (!reg8) {
 619                         die("Invalid tRAS value.\n");
 620                 }
 621
 622                 while ((tRAS_time >> 2) < reg8) {
 623                         tRAS_time += freq_multiplier;
 624                         tRAS_cycles++;
 625                 }
 626         }
 627         if(tRAS_cycles > 0x18) {
 628                 die("DDR-II Module does not support this frequency (tRAS error)\n");
 629         }
 630
 631         printk(BIOS_DEBUG, "tRAS = %d cycles\n", tRAS_cycles);
 632         sysinfo->tras = tRAS_cycles;
 633 }
 634
 635 static void sdram_detect_smallest_tRP(struct sys_info * sysinfo)
 636 {
 637         int i;
 638         int tRP_time;
 639         int tRP_cycles;
 640         int freq_multiplier = 0;
 641
 642         switch (sysinfo->memory_frequency) {
 643         case 400: freq_multiplier = 0x14; break; /* 5ns */
 644         case 533: freq_multiplier = 0x0f; break; /* 3.75ns */
 645         case 667: freq_multiplier = 0x0c; break; /* 3ns */
 646         }
 647
 648         tRP_cycles = 2; /* 2 clocks minimum */
 649         tRP_time = tRP_cycles * freq_multiplier;
 650
 651         for (i=0; i<2*DIMM_SOCKETS; i++) {
 652                 u8 reg8;
 653
 654                 if (sysinfo->dimm[i] == SYSINFO_DIMM_NOT_POPULATED)
 655                         continue;
 656
 657                 reg8 = spd_read_byte(DIMM_SPD_BASE + i, SPD_MIN_ROW_PRECHARGE_TIME);
 658                 if (!reg8) {
 659                         die("Invalid tRP value.\n");
 660                 }
 661
 662                 while (tRP_time < reg8) {
 663                         tRP_time += freq_multiplier;
 664                         tRP_cycles++;
 665                 }
 666         }
 667
 668         if(tRP_cycles > 6) {
 669                 die("DDR-II Module does not support this frequency (tRP error)\n");
 670         }
 671
 672         printk(BIOS_DEBUG, "tRP = %d cycles\n", tRP_cycles);
 673         sysinfo->trp = tRP_cycles;
 674 }
 675
 676 static void sdram_detect_smallest_tRCD(struct sys_info * sysinfo)
 677 {
 678         int i;
 679         int tRCD_time;
 680         int tRCD_cycles;
 681         int freq_multiplier = 0;
 682
 683         switch (sysinfo->memory_frequency) {
 684         case 400: freq_multiplier = 0x14; break; /* 5ns */
 685         case 533: freq_multiplier = 0x0f; break; /* 3.75ns */
 686         case 667: freq_multiplier = 0x0c; break; /* 3ns */
 687         }
 688
 689         tRCD_cycles = 2; /* 2 clocks minimum */
 690         tRCD_time = tRCD_cycles * freq_multiplier;
 691
 692         for (i=0; i<2*DIMM_SOCKETS; i++) {
 693                 u8 reg8;
 694
 695                 if (sysinfo->dimm[i] == SYSINFO_DIMM_NOT_POPULATED)
 696                         continue;
 697
 698                 reg8 = spd_read_byte(DIMM_SPD_BASE + i, SPD_MIN_RAS_TO_CAS_DELAY);
 699                 if (!reg8) {
 700                         die("Invalid tRCD value.\n");
 701                 }
 702
 703                 while (tRCD_time < reg8) {
 704                         tRCD_time += freq_multiplier;
 705                         tRCD_cycles++;
 706                 }
 707         }
 708         if(tRCD_cycles > 6) {
 709                 die("DDR-II Module does not support this frequency (tRCD error)\n");
 710         }
 711
 712         printk(BIOS_DEBUG, "tRCD = %d cycles\n", tRCD_cycles);
 713         sysinfo->trcd = tRCD_cycles;
 714 }
 715
 716 static void sdram_detect_smallest_tWR(struct sys_info * sysinfo)
 717 {
 718         int i;
 719         int tWR_time;
 720         int tWR_cycles;
 721         int freq_multiplier = 0;
 722
 723         switch (sysinfo->memory_frequency) {
 724         case 400: freq_multiplier = 0x14; break; /* 5ns */
 725         case 533: freq_multiplier = 0x0f; break; /* 3.75ns */
 726         case 667: freq_multiplier = 0x0c; break; /* 3ns */
 727         }
 728
 729         tWR_cycles = 2; /* 2 clocks minimum */
 730         tWR_time = tWR_cycles * freq_multiplier;
 731
 732         for (i=0; i<2*DIMM_SOCKETS; i++) {
 733                 u8 reg8;
 734
 735                 if (sysinfo->dimm[i] == SYSINFO_DIMM_NOT_POPULATED)
 736                         continue;
 737
 738                 reg8 = spd_read_byte(DIMM_SPD_BASE + i, SPD_WRITE_RECOVERY_TIME);
 739                 if (!reg8) {
 740                         die("Invalid tWR value.\n");
 741                 }
 742
 743                 while (tWR_time < reg8) {
 744                         tWR_time += freq_multiplier;
 745                         tWR_cycles++;
 746                 }
 747         }
 748         if(tWR_cycles > 5) {
 749                 die("DDR-II Module does not support this frequency (tWR error)\n");
 750         }
 751
 752         printk(BIOS_DEBUG, "tWR = %d cycles\n", tWR_cycles);
 753         sysinfo->twr = tWR_cycles;
 754 }
 755
 756 static void sdram_detect_smallest_tRFC(struct sys_info * sysinfo)
 757 {
 758         int i, index = 0;
 759
 760         const u8 tRFC_cycles[] = {
 761              /* 75 105 127.5 */
 762                 15, 21, 26,     /* DDR2-400 */
 763                 20, 28, 34,     /* DDR2-533 */
 764                 25, 35, 43      /* DDR2-667 */
 765         };
 766
 767         for (i=0; i<2*DIMM_SOCKETS; i++) {
 768                 u8 reg8;
 769
 770                 if (sysinfo->dimm[i] == SYSINFO_DIMM_NOT_POPULATED)
 771                         continue;
 772
 773                 reg8 = sysinfo->banksize[i*2];
 774                 switch (reg8) {
 775                 case 0x04: reg8 = 0; break;
 776                 case 0x08: reg8 = 1; break;
 777                 case 0x10: reg8 = 2; break;
 778                 case 0x20: reg8 = 3; break;
 779                 }
 780
 781                 if (sysinfo->dimm[i] == SYSINFO_DIMM_X16DS || sysinfo->dimm[i] == SYSINFO_DIMM_X16SS)
 782                         reg8++;
 783
 784                 if (reg8 > 3) {
 785                         /* Can this happen? Go back to 127.5ns just to be sure
 786                          * we don't run out of the array. This may be wrong
 787                          */
 788                         printk(BIOS_DEBUG, "DIMM %d is 1Gb x16.. Please report.\n", i);
 789                         reg8 = 3;
 790                 }
 791
 792                 if (reg8 > index)
 793                         index = reg8;
 794
 795         }
 796         index--;
 797         switch (sysinfo->memory_frequency) {
 798         case 667: index += 3; /* Fallthrough */
 799         case 533: index += 3; /* Fallthrough */
 800         case 400: break;
 801         }
 802
 803         sysinfo->trfc = tRFC_cycles[index];
 804         printk(BIOS_DEBUG, "tRFC = %d cycles\n", tRFC_cycles[index]);
 805 }
 806
 807 static void sdram_detect_smallest_refresh(struct sys_info * sysinfo)
 808 {
 809         int i;
 810
 811         sysinfo->refresh = 0;
 812
 813         for (i=0; i<2*DIMM_SOCKETS; i++) {
 814                 int refresh;
 815
 816                 if (sysinfo->dimm[i] == SYSINFO_DIMM_NOT_POPULATED)
 817                         continue;
 818
 819                 refresh = spd_read_byte(DIMM_SPD_BASE + i, SPD_REFRESH) & ~(1 << 7);
 820
 821                 /* 15.6us */
 822                 if (!refresh)
 823                         continue;
 824
 825                 /* Refresh is slower than 15.6us, use 15.6us */
 826                 if (refresh > 2)
 827                         continue;
 828
 829                 if (refresh == 2) {
 830                         sysinfo->refresh = 1;
 831                         break;
 832                 }
 833
 834                 die("DDR-II module has unsupported refresh value\n");
 835         }
 836         printk(BIOS_DEBUG, "Refresh: %s\n", sysinfo->refresh?"7.8us":"15.6us");
 837 }
 838
 839 static void sdram_verify_burst_length(struct sys_info * sysinfo)
 840 {
 841         int i;
 842
 843         for (i=0; i<2*DIMM_SOCKETS; i++) {
 844                 if (sysinfo->dimm[i] == SYSINFO_DIMM_NOT_POPULATED)
 845                         continue;
 846
 847                 if (!(spd_read_byte(DIMM_SPD_BASE + i, SPD_SUPPORTED_BURST_LENGTHS) & SPD_BURST_LENGTH_8))
 848                         die("Only DDR-II RAM with burst length 8 is supported by this chipset.\n");
 849         }
 850 }
 851
 852 static void sdram_program_dram_width(struct sys_info * sysinfo)
 853 {
 854         u16 c0dramw=0, c1dramw=0;
 855         int idx;
 856
 857         if (sysinfo->dual_channel)
 858                 idx = 2;
 859         else
 860                 idx = 1;
 861
 862         switch (sysinfo->dimm[0]) {
 863         case 0: c0dramw = 0x0000; break; /* x16DS */
 864         case 1: c0dramw = 0x0001; break; /* x8DS */
 865         case 2: c0dramw = 0x0000; break; /* x16SS */
 866         case 3: c0dramw = 0x0005; break; /* x8DDS */
 867         case 4: c0dramw = 0x0000; break; /* NC */
 868         }
 869
 870         switch (sysinfo->dimm[idx]) {
 871         case 0: c1dramw = 0x0000; break; /* x16DS */
 872         case 1: c1dramw = 0x0010; break; /* x8DS */
 873         case 2: c1dramw = 0x0000; break; /* x16SS */
 874         case 3: c1dramw = 0x0050; break; /* x8DDS */
 875         case 4: c1dramw = 0x0000; break; /* NC */
 876         }
 877
 878         if ( !sdram_capabilities_dual_channel() ) {
 879                 /* Single Channel */
 880                 c0dramw |= c1dramw;
 881                 c1dramw = 0;
 882         }
 883
 884         MCHBAR16(C0DRAMW) = c0dramw;
 885         MCHBAR16(C1DRAMW) = c1dramw;
 886 }
 887
 888 static void sdram_write_slew_rates(u32 offset, const u32 *slew_rate_table)
 889 {
 890         int i;
 891
 892         for (i=0; i<16; i++)
 893                 MCHBAR32(offset+(i*4)) = slew_rate_table[i];
 894 }
 895
 896 static const u32 dq2030[] = {
 897         0x08070706, 0x0a090908, 0x0d0c0b0a, 0x12100f0e,
 898         0x1a181614, 0x22201e1c, 0x2a282624, 0x3934302d,
 899         0x0a090908, 0x0c0b0b0a, 0x0e0d0d0c, 0x1211100f,
 900         0x19171513, 0x211f1d1b, 0x2d292623, 0x3f393531
 901 };
 902
 903 static const u32 dq2330[] = {
 904         0x08070706, 0x0a090908, 0x0d0c0b0a, 0x12100f0e,
 905         0x1a181614, 0x22201e1c, 0x2a282624, 0x3934302d,
 906         0x0a090908, 0x0c0b0b0a, 0x0e0d0d0c, 0x1211100f,
 907         0x19171513, 0x211f1d1b, 0x2d292623, 0x3f393531
 908 };
 909
 910 static const u32 cmd2710[] = {
 911         0x07060605, 0x0f0d0b09, 0x19171411, 0x1f1f1d1b,
 912         0x1f1f1f1f, 0x1f1f1f1f, 0x1f1f1f1f, 0x1f1f1f1f,
 913         0x1110100f, 0x0f0d0b09, 0x19171411, 0x1f1f1d1b,
 914         0x1f1f1f1f, 0x1f1f1f1f, 0x1f1f1f1f, 0x1f1f1f1f
 915 };
 916
 917 static const u32 cmd3210[] = {
 918         0x0f0d0b0a, 0x17151311, 0x1f1d1b19, 0x1f1f1f1f,
 919         0x1f1f1f1f, 0x1f1f1f1f, 0x1f1f1f1f, 0x1f1f1f1f,
 920         0x18171615, 0x1f1f1c1a, 0x1f1f1f1f, 0x1f1f1f1f,
 921         0x1f1f1f1f, 0x1f1f1f1f, 0x1f1f1f1f, 0x1f1f1f1f
 922 };
 923
 924 static const u32 clk2030[] = {
 925         0x0e0d0d0c, 0x100f0f0e, 0x100f0e0d, 0x15131211,
 926         0x1d1b1917, 0x2523211f, 0x2a282927, 0x32302e2c,
 927         0x17161514, 0x1b1a1918, 0x1f1e1d1c, 0x23222120,
 928         0x27262524, 0x2d2b2928, 0x3533312f, 0x3d3b3937
 929 };
 930
 931 static const u32 ctl3215[] = {
 932         0x01010000, 0x03020101, 0x07060504, 0x0b0a0908,
 933         0x100f0e0d, 0x14131211, 0x18171615, 0x1c1b1a19,
 934         0x05040403, 0x07060605, 0x0a090807, 0x0f0d0c0b,
 935         0x14131211, 0x18171615, 0x1c1b1a19, 0x201f1e1d
 936 };
 937
 938 static const u32 ctl3220[] = {
 939         0x05040403, 0x07060505, 0x0e0c0a08, 0x1a171411,
 940         0x2825221f, 0x35322f2b, 0x3e3e3b38, 0x3e3e3e3e,
 941         0x09080807, 0x0b0a0a09, 0x0f0d0c0b, 0x1b171311,
 942         0x2825221f, 0x35322f2b, 0x3e3e3b38, 0x3e3e3e3e
 943 };
 944
 945 static const u32 nc[] = {
 946         0x00000000, 0x00000000, 0x00000000, 0x00000000,
 947         0x00000000, 0x00000000, 0x00000000, 0x00000000,
 948         0x00000000, 0x00000000, 0x00000000, 0x00000000,
 949         0x00000000, 0x00000000, 0x00000000, 0x00000000
 950 };
 951
 952 enum {
 953         DQ2030,
 954         DQ2330,
 955         CMD2710,
 956         CMD3210,
 957         CLK2030,
 958         CTL3215,
 959         CTL3220,
 960         NC,
 961 };
 962
 963 static const u8 dual_channel_slew_group_lookup[] = {
 964         DQ2030, CMD3210, CTL3215, CTL3215, CLK2030, CLK2030, DQ2030, CMD3210,
 965         DQ2030, CMD3210, CTL3215, CTL3215, CLK2030, CLK2030, DQ2030, CMD3210,
 966         DQ2030, CMD3210, NC,      CTL3215, NC,      CLK2030, DQ2030, CMD3210,
 967         DQ2030, CMD3210, CTL3215, CTL3215, CLK2030, CLK2030, DQ2030, CMD2710,
 968         DQ2030, CMD3210, NC,      CTL3215, NC,      CLK2030, NC,     NC,
 969
 970         DQ2030, CMD3210, CTL3215, CTL3215, CLK2030, CLK2030, DQ2030, CMD3210,
 971         DQ2030, CMD3210, CTL3215, NC,      CLK2030, NC,      DQ2030, CMD3210,
 972         DQ2030, CMD3210, CTL3215, CTL3215, CLK2030, CLK2030, DQ2030, CMD3210,
 973         DQ2030, CMD3210, CTL3215, NC,      CLK2030, NC,      DQ2030, CMD2710,
 974         DQ2030, CMD3210, CTL3215, NC,      CLK2030, NC,      NC,     NC,
 975
 976         DQ2030, CMD3210, NC,      CTL3215, NC,      CLK2030, DQ2030, CMD3210,
 977         DQ2030, CMD3210, CTL3215, CTL3215, CLK2030, CLK2030, DQ2030, CMD3210,
 978         DQ2030, CMD3210, NC,      CTL3215, NC,      CLK2030, DQ2030, CMD3210,
 979         DQ2030, CMD3210, CTL3215, CTL3215, CLK2030, CLK2030, DQ2030, CMD2710,
 980         DQ2030, CMD3210, NC,      CTL3215, NC,      CLK2030, NC,     NC,
 981
 982         DQ2030, CMD2710, CTL3215, CTL3215, CLK2030, CLK2030, DQ2030, CMD3210,
 983         DQ2030, CMD2710, CTL3215, NC,      CLK2030, NC,      DQ2030, CMD3210,
 984         DQ2030, CMD2710, CTL3215, CTL3215, CLK2030, CLK2030, DQ2030, CMD3210,
 985         DQ2030, CMD2710, CTL3215, NC,      CLK2030, NC,      DQ2030, CMD2710,
 986         DQ2030, CMD2710, CTL3215, NC,      CLK2030, NC,      NC,     NC,
 987
 988         NC,     NC,      NC,      CTL3215, NC,      CLK2030, DQ2030, CMD3210,
 989         NC,     NC,      CTL3215, NC,      CLK2030, NC,      DQ2030, CMD3210,
 990         NC,     NC,      NC,      CTL3215, NC,      CLK2030, DQ2030, CMD3210,
 991         NC,     NC,      CTL3215, NC,      CLK2030, CLK2030, DQ2030, CMD2710
 992 };
 993
 994 static const u8 single_channel_slew_group_lookup[] = {
 995         DQ2330, CMD3210, CTL3215, CTL3215, CLK2030, CLK2030, DQ2330, CMD3210,
 996         DQ2330, CMD3210, CTL3215, CTL3215, CLK2030, CLK2030, DQ2330, CMD3210,
 997         DQ2330, CMD3210, NC,      CTL3215, NC,      CLK2030, DQ2330, CMD3210,
 998         DQ2330, CMD3210, CTL3215, CTL3215, CLK2030, CLK2030, DQ2330, CMD3210,
 999         DQ2330, CMD3210, NC,      CTL3215, NC,      CLK2030, NC,     NC,
1000
1001         DQ2330, CMD3210, CTL3215, CTL3215, CLK2030, CLK2030, DQ2330, CMD3210,
1002         DQ2330, CMD3210, CTL3215, NC,      CLK2030, NC,      DQ2330, CMD3210,
1003         DQ2330, CMD3210, CTL3215, CTL3215, CLK2030, CLK2030, DQ2330, CMD3210,
1004         DQ2330, CMD3210, CTL3215, NC,      CLK2030, NC,      DQ2330, CMD3210,
1005         DQ2330, CMD3210, CTL3215, NC,      CLK2030, NC,      NC,     NC,
1006
1007         DQ2330, CMD3210, NC,      CTL3215, NC,      CLK2030, DQ2330, CMD3210,
1008         DQ2330, CMD3210, CTL3215, CTL3215, CLK2030, CLK2030, DQ2330, CMD3210,
1009         DQ2330, CMD3210, NC,      CTL3215, NC,      CLK2030, DQ2330, CMD3210,
1010         DQ2330, CMD3210, CTL3215, CTL3215, CLK2030, CLK2030, DQ2330, CMD3210,
1011         DQ2330, CMD3210, NC,      CTL3215, NC,      CLK2030, NC,     NC,
1012
1013         DQ2330, CMD3210, CTL3215, CTL3215, CLK2030, CLK2030, DQ2330, CMD3210,
1014         DQ2330, CMD3210, CTL3215, NC,      CLK2030, NC,      DQ2330, CMD3210,
1015         DQ2330, CMD3210, CTL3215, CTL3215, CLK2030, CLK2030, DQ2330, CMD3210,
1016         DQ2330, CMD3210, CTL3215, NC,      CLK2030, NC,      DQ2330, CMD3210,
1017         DQ2330, CMD3210, CTL3215, NC,      CLK2030, NC,      NC,     NC,
1018
1019         DQ2330, NC,      NC,      CTL3215, NC,      CLK2030, DQ2030, CMD3210,
1020         DQ2330, NC,      CTL3215, NC,      CLK2030, NC,      DQ2030, CMD3210,
1021         DQ2330, NC,      NC,      CTL3215, NC,      CLK2030, DQ2030, CMD3210,
1022         DQ2330, NC,      CTL3215, NC,      CLK2030, CLK2030, DQ2030, CMD3210
1023 };
1024
1025 static const u32 *slew_group_lookup(int dual_channel, int index)
1026 {
1027         const u8 *slew_group;
1028         /* Dual Channel needs different tables. */
1029         if (dual_channel)
1030                 slew_group   = dual_channel_slew_group_lookup;
1031         else
1032                 slew_group   = single_channel_slew_group_lookup;
1033
1034         switch (slew_group[index]) {
1035         case DQ2030:    return dq2030;
1036         case DQ2330:    return dq2330;
1037         case CMD2710:   return cmd2710;
1038         case CMD3210:   return cmd3210;
1039         case CLK2030:   return clk2030;
1040         case CTL3215:   return ctl3215;
1041         case CTL3220:   return ctl3220;
1042         case NC:        return nc;
1043         }
1044
1045         return nc;
1046 }
1047
1048 #if CONFIG_I945GM
1049 /* Strength multiplier tables */
1050 static const u8 dual_channel_strength_multiplier[] = {
1051         0x44, 0x11, 0x11, 0x11, 0x44, 0x44, 0x44, 0x11,
1052         0x44, 0x11, 0x11, 0x11, 0x44, 0x44, 0x44, 0x11,
1053         0x44, 0x11, 0x00, 0x11, 0x00, 0x44, 0x44, 0x11,
1054         0x44, 0x11, 0x11, 0x11, 0x44, 0x44, 0x44, 0x22,
1055         0x44, 0x11, 0x00, 0x11, 0x00, 0x44, 0x00, 0x00,
1056         0x44, 0x11, 0x11, 0x11, 0x44, 0x44, 0x44, 0x11,
1057         0x44, 0x11, 0x11, 0x00, 0x44, 0x00, 0x44, 0x11,
1058         0x44, 0x11, 0x11, 0x11, 0x44, 0x44, 0x44, 0x11,
1059         0x44, 0x11, 0x11, 0x00, 0x44, 0x00, 0x44, 0x22,
1060         0x44, 0x11, 0x11, 0x00, 0x44, 0x00, 0x00, 0x00,
1061         0x44, 0x11, 0x00, 0x11, 0x00, 0x44, 0x44, 0x11,
1062         0x44, 0x11, 0x11, 0x11, 0x44, 0x44, 0x44, 0x11,
1063         0x44, 0x11, 0x00, 0x11, 0x00, 0x44, 0x44, 0x11,
1064         0x44, 0x11, 0x11, 0x11, 0x44, 0x44, 0x44, 0x22,
1065         0x44, 0x11, 0x00, 0x11, 0x00, 0x44, 0x00, 0x00,
1066         0x44, 0x22, 0x11, 0x11, 0x44, 0x44, 0x44, 0x11,
1067         0x44, 0x22, 0x11, 0x00, 0x44, 0x00, 0x44, 0x11,
1068         0x44, 0x22, 0x11, 0x11, 0x44, 0x44, 0x44, 0x11,
1069         0x44, 0x22, 0x11, 0x00, 0x44, 0x00, 0x44, 0x22,
1070         0x44, 0x22, 0x11, 0x00, 0x44, 0x00, 0x00, 0x00,
1071         0x00, 0x00, 0x00, 0x11, 0x00, 0x44, 0x44, 0x11,
1072         0x00, 0x00, 0x11, 0x00, 0x44, 0x00, 0x44, 0x11,
1073         0x00, 0x00, 0x00, 0x11, 0x00, 0x44, 0x44, 0x11,
1074         0x00, 0x00, 0x11, 0x00, 0x44, 0x44, 0x44, 0x22
1075 };
1076
1077 static const u8 single_channel_strength_multiplier[] = {
1078         0x33, 0x11, 0x11, 0x11, 0x44, 0x44, 0x33, 0x11,
1079         0x33, 0x11, 0x11, 0x11, 0x44, 0x44, 0x33, 0x11,
1080         0x33, 0x11, 0x00, 0x11, 0x00, 0x44, 0x33, 0x11,
1081         0x33, 0x11, 0x11, 0x11, 0x44, 0x44, 0x33, 0x11,
1082         0x33, 0x11, 0x00, 0x11, 0x00, 0x44, 0x00, 0x00,
1083         0x33, 0x11, 0x11, 0x11, 0x44, 0x44, 0x33, 0x11,
1084         0x33, 0x11, 0x11, 0x00, 0x44, 0x00, 0x33, 0x11,
1085         0x33, 0x11, 0x11, 0x11, 0x44, 0x44, 0x33, 0x11,
1086         0x33, 0x11, 0x11, 0x00, 0x44, 0x00, 0x33, 0x11,
1087         0x33, 0x11, 0x11, 0x00, 0x44, 0x00, 0x00, 0x00,
1088         0x33, 0x11, 0x00, 0x11, 0x00, 0x44, 0x33, 0x11,
1089         0x33, 0x11, 0x11, 0x11, 0x44, 0x44, 0x33, 0x11,
1090         0x33, 0x11, 0x00, 0x11, 0x00, 0x44, 0x33, 0x11,
1091         0x33, 0x11, 0x11, 0x11, 0x44, 0x44, 0x33, 0x11,
1092         0x33, 0x11, 0x00, 0x11, 0x00, 0x44, 0x00, 0x00,
1093         0x33, 0x11, 0x11, 0x11, 0x44, 0x44, 0x33, 0x11,
1094         0x33, 0x11, 0x11, 0x00, 0x44, 0x00, 0x33, 0x11,
1095         0x33, 0x11, 0x11, 0x11, 0x44, 0x44, 0x33, 0x11,
1096         0x33, 0x11, 0x11, 0x00, 0x44, 0x00, 0x33, 0x11,
1097         0x33, 0x11, 0x11, 0x00, 0x44, 0x00, 0x00, 0x00,
1098         0x33, 0x00, 0x00, 0x11, 0x00, 0x44, 0x33, 0x11,
1099         0x33, 0x00, 0x11, 0x00, 0x44, 0x00, 0x33, 0x11,
1100         0x33, 0x00, 0x00, 0x11, 0x00, 0x44, 0x33, 0x11,
1101         0x33, 0x00, 0x11, 0x00, 0x44, 0x44, 0x33, 0x11
1102 };
1103 #endif
1104 #if CONFIG_I945GC
1105 static const u8 dual_channel_strength_multiplier[] = {
1106         0x44, 0x22, 0x00, 0x00, 0x44, 0x44, 0x44, 0x22,
1107         0x44, 0x22, 0x00, 0x00, 0x44, 0x44, 0x44, 0x22,
1108         0x44, 0x22, 0x00, 0x00, 0x44, 0x44, 0x44, 0x22,
1109         0x44, 0x22, 0x00, 0x00, 0x44, 0x44, 0x44, 0x33,
1110         0x44, 0x22, 0x00, 0x00, 0x44, 0x44, 0x44, 0x00,
1111         0x44, 0x22, 0x00, 0x00, 0x44, 0x44, 0x44, 0x22,
1112         0x44, 0x22, 0x00, 0x00, 0x44, 0x44, 0x44, 0x22,
1113         0x44, 0x22, 0x00, 0x00, 0x44, 0x44, 0x44, 0x22,
1114         0x44, 0x22, 0x00, 0x00, 0x44, 0x44, 0x44, 0x33,
1115         0x44, 0x22, 0x00, 0x00, 0x44, 0x44, 0x44, 0x00,
1116         0x44, 0x22, 0x00, 0x00, 0x44, 0x44, 0x44, 0x22,
1117         0x44, 0x22, 0x00, 0x00, 0x44, 0x44, 0x44, 0x22,
1118         0x44, 0x22, 0x00, 0x00, 0x44, 0x44, 0x44, 0x22,
1119         0x44, 0x22, 0x00, 0x00, 0x44, 0x44, 0x44, 0x33,
1120         0x44, 0x22, 0x00, 0x00, 0x44, 0x44, 0x44, 0x00,
1121         0x44, 0x33, 0x00, 0x00, 0x44, 0x44, 0x44, 0x22,
1122         0x44, 0x33, 0x00, 0x00, 0x44, 0x44, 0x44, 0x22,
1123         0x44, 0x33, 0x00, 0x00, 0x44, 0x44, 0x44, 0x22,
1124         0x44, 0x33, 0x00, 0x00, 0x44, 0x44, 0x44, 0x33,
1125         0x44, 0x33, 0x00, 0x00, 0x44, 0x44, 0x44, 0x00,
1126         0x44, 0x00, 0x00, 0x00, 0x44, 0x44, 0x44, 0x22,
1127         0x44, 0x00, 0x00, 0x00, 0x44, 0x44, 0x44, 0x22,
1128         0x44, 0x00, 0x00, 0x00, 0x44, 0x44, 0x44, 0x22,
1129         0x44, 0x00, 0x00, 0x00, 0x44, 0x44, 0x44, 0x33
1130 };
1131
1132 static const u8 single_channel_strength_multiplier[] = {
1133         0x44, 0x33, 0x00, 0x00, 0x44, 0x44, 0x44, 0x00,
1134         0x44, 0x44, 0x00, 0x00, 0x44, 0x44, 0x44, 0x00,
1135         0x44, 0x33, 0x00, 0x00, 0x44, 0x44, 0x44, 0x00,
1136         0x44, 0x55, 0x00, 0x00, 0x44, 0x44, 0x44, 0x00,
1137         0x44, 0x22, 0x00, 0x00, 0x44, 0x44, 0x44, 0x00,
1138         0x44, 0x44, 0x00, 0x00, 0x44, 0x44, 0x44, 0x00,
1139         0x44, 0x55, 0x00, 0x00, 0x44, 0x44, 0x44, 0x00,
1140         0x44, 0x44, 0x00, 0x00, 0x44, 0x44, 0x44, 0x00,
1141         0x44, 0x88, 0x00, 0x00, 0x44, 0x44, 0x44, 0x00,
1142         0x44, 0x22, 0x00, 0x00, 0x44, 0x44, 0x44, 0x00,
1143         0x44, 0x33, 0x00, 0x00, 0x44, 0x44, 0x44, 0x00,
1144         0x44, 0x44, 0x00, 0x00, 0x44, 0x44, 0x44, 0x00,
1145         0x44, 0x33, 0x00, 0x00, 0x44, 0x44, 0x44, 0x00,
1146         0x44, 0x55, 0x00, 0x00, 0x44, 0x44, 0x44, 0x00,
1147         0x44, 0x22, 0x00, 0x00, 0x44, 0x44, 0x44, 0x00,
1148         0x44, 0x55, 0x00, 0x00, 0x44, 0x44, 0x44, 0x00,
1149         0x44, 0x88, 0x00, 0x00, 0x44, 0x44, 0x44, 0x00,
1150         0x44, 0x55, 0x00, 0x00, 0x44, 0x44, 0x44, 0x00,
1151         0x44, 0x88, 0x00, 0x00, 0x44, 0x44, 0x44, 0x00,
1152         0x44, 0x33, 0x00, 0x00, 0x44, 0x44, 0x44, 0x00,
1153         0x44, 0x22, 0x00, 0x00, 0x44, 0x44, 0x44, 0x00,
1154         0x44, 0x22, 0x00, 0x00, 0x44, 0x44, 0x44, 0x00,
1155         0x44, 0x22, 0x00, 0x00, 0x44, 0x44, 0x44, 0x00,
1156         0x44, 0x33, 0x00, 0x00, 0x44, 0x44, 0x44, 0x00
1157 };
1158 #endif
1159
1160 static void sdram_rcomp_buffer_strength_and_slew(struct sys_info *sysinfo)
1161 {
1162         const u8 * strength_multiplier;
1163         int idx, dual_channel;
1164
1165         /* Set Strength Multipliers */
1166
1167         /* Dual Channel needs different tables. */
1168         if (sdram_capabilities_dual_channel()) {
1169                 printk(BIOS_DEBUG, "Programming Dual Channel RCOMP\n");
1170                 strength_multiplier = dual_channel_strength_multiplier;
1171                 dual_channel = 1;
1172                 idx = 5 * sysinfo->dimm[0] +  sysinfo->dimm[2];
1173         } else {
1174                 printk(BIOS_DEBUG, "Programming Single Channel RCOMP\n");
1175                 strength_multiplier = single_channel_strength_multiplier;
1176                 dual_channel = 0;
1177                 idx = 5 * sysinfo->dimm[0] + sysinfo->dimm[1];
1178         }
1179
1180         printk(BIOS_DEBUG, "Table Index: %d\n", idx);
1181
1182         MCHBAR8(G1SC) = strength_multiplier[idx * 8 + 0];
1183         MCHBAR8(G2SC) = strength_multiplier[idx * 8 + 1];
1184         MCHBAR8(G3SC) = strength_multiplier[idx * 8 + 2];
1185         MCHBAR8(G4SC) = strength_multiplier[idx * 8 + 3];
1186         MCHBAR8(G5SC) = strength_multiplier[idx * 8 + 4];
1187         MCHBAR8(G6SC) = strength_multiplier[idx * 8 + 5];
1188         MCHBAR8(G7SC) = strength_multiplier[idx * 8 + 6];
1189         MCHBAR8(G8SC) = strength_multiplier[idx * 8 + 7];
1190
1191         /* Channel 0 */
1192         sdram_write_slew_rates(G1SRPUT, slew_group_lookup(dual_channel, idx * 8 + 0));
1193         sdram_write_slew_rates(G2SRPUT, slew_group_lookup(dual_channel, idx * 8 + 1));
1194         if ((slew_group_lookup(dual_channel, idx * 8 + 2) != nc) && (sysinfo->package == SYSINFO_PACKAGE_STACKED)) {
1195
1196                 sdram_write_slew_rates(G3SRPUT, ctl3220);
1197         } else {
1198                 sdram_write_slew_rates(G3SRPUT, slew_group_lookup(dual_channel, idx * 8 + 2));
1199         }
1200         sdram_write_slew_rates(G4SRPUT, slew_group_lookup(dual_channel, idx * 8 + 3));
1201         sdram_write_slew_rates(G5SRPUT, slew_group_lookup(dual_channel, idx * 8 + 4));
1202         sdram_write_slew_rates(G6SRPUT, slew_group_lookup(dual_channel, idx * 8 + 5));
1203
1204         /* Channel 1 */
1205         if (sysinfo->dual_channel) {
1206                 sdram_write_slew_rates(G7SRPUT, slew_group_lookup(dual_channel, idx * 8 + 6));
1207                 sdram_write_slew_rates(G8SRPUT, slew_group_lookup(dual_channel, idx * 8 + 7));
1208         } else {
1209                 sdram_write_slew_rates(G7SRPUT, nc);
1210                 sdram_write_slew_rates(G8SRPUT, nc);
1211         }
1212 }
1213
1214 static void sdram_enable_rcomp(void)
1215 {
1216         u32 reg32;
1217         /* Enable Global Periodic RCOMP */
1218         udelay(300);
1219         reg32 = MCHBAR32(GBRCOMPCTL);
1220         reg32 &= ~(1 << 23);
1221         MCHBAR32(GBRCOMPCTL) = reg32;
1222 }
1223
1224 static void sdram_program_dll_timings(struct sys_info *sysinfo)
1225 {
1226         u32 chan0dll = 0, chan1dll = 0;
1227         int i;
1228
1229         printk(BIOS_DEBUG, "Programming DLL Timings... \n");
1230
1231         MCHBAR16(DQSMT) &= ~( (3 << 12) | (1 << 10) | ( 0xf << 0) );
1232         MCHBAR16(DQSMT) |= (1 << 13) | (0xc << 0);
1233
1234         /* We drive both channels with the same speed */
1235         switch (sysinfo->memory_frequency) {
1236         case 400: chan0dll = 0x26262626; chan1dll=0x26262626; break; /* 400MHz */
1237         case 533: chan0dll = 0x22222222; chan1dll=0x22222222; break; /* 533MHz */
1238         case 667: chan0dll = 0x11111111; chan1dll=0x11111111; break; /* 667MHz */
1239         }
1240
1241         for (i=0; i < 4; i++) {
1242                 MCHBAR32(C0R0B00DQST + (i * 0x10) + 0) = chan0dll;
1243                 MCHBAR32(C0R0B00DQST + (i * 0x10) + 4) = chan0dll;
1244                 MCHBAR32(C1R0B00DQST + (i * 0x10) + 0) = chan1dll;
1245                 MCHBAR32(C1R0B00DQST + (i * 0x10) + 4) = chan1dll;
1246         }
1247 }
1248
1249 static void sdram_force_rcomp(void)
1250 {
1251         u32 reg32;
1252         u8 reg8;
1253
1254         reg32 = MCHBAR32(ODTC);
1255         reg32 |= (1 << 28);
1256         MCHBAR32(ODTC) = reg32;
1257
1258         reg32 = MCHBAR32(SMSRCTL);
1259         reg32 |= (1 << 0);
1260         MCHBAR32(SMSRCTL) = reg32;
1261
1262         /* Start initial RCOMP */
1263         reg32 = MCHBAR32(GBRCOMPCTL);
1264         reg32 |= (1 << 8);
1265         MCHBAR32(GBRCOMPCTL) = reg32;
1266
1267         reg8 = i945_silicon_revision();
1268         if ((reg8 == 0 && (MCHBAR32(DCC) & (3 << 0)) == 0) || (reg8 == 1)) {
1269
1270                 reg32 = MCHBAR32(GBRCOMPCTL);
1271                 reg32 |= (3 << 5);
1272                 MCHBAR32(GBRCOMPCTL) = reg32;
1273         }
1274 }
1275
1276 static void sdram_initialize_system_memory_io(struct sys_info *sysinfo)
1277 {
1278         u8 reg8;
1279         u32 reg32;
1280
1281         printk(BIOS_DEBUG, "Initializing System Memory IO... \n");
1282         /* Enable Data Half Clock Pushout */
1283         reg8 = MCHBAR8(C0HCTC);
1284         reg8 &= ~0x1f;
1285         reg8 |= ( 1 << 0);
1286         MCHBAR8(C0HCTC) = reg8;
1287
1288         reg8 = MCHBAR8(C1HCTC);
1289         reg8 &= ~0x1f;
1290         reg8 |= ( 1 << 0);
1291         MCHBAR8(C1HCTC) = reg8;
1292
1293         MCHBAR16(WDLLBYPMODE) &= ~( (1 << 9) | (1 << 6) | (1 << 4) | (1 << 3) | (1 << 1) );
1294         MCHBAR16(WDLLBYPMODE) |= (1 << 8) | (1 << 7) | (1 << 5) | (1 << 2) | (1 << 0);
1295
1296         MCHBAR8(C0WDLLCMC) = 0;
1297         MCHBAR8(C1WDLLCMC) = 0;
1298
1299         /* Program RCOMP Settings */
1300         sdram_program_dram_width(sysinfo);
1301
1302         sdram_rcomp_buffer_strength_and_slew(sysinfo);
1303
1304         /* Indicate that RCOMP programming is done */
1305         reg32 = MCHBAR32(GBRCOMPCTL);
1306         reg32 &= ~( (1 << 29) | (1 << 26) | (3 << 21) | (3 << 2) );
1307         reg32 |= (3 << 27) | (3 << 0);
1308         MCHBAR32(GBRCOMPCTL) = reg32;
1309
1310         MCHBAR32(GBRCOMPCTL) |= (1 << 10);
1311
1312         /* Program DLL Timings */
1313         sdram_program_dll_timings(sysinfo);
1314
1315         /* Force RCOMP cycle */
1316         sdram_force_rcomp();
1317 }
1318
1319 static void sdram_enable_system_memory_io(struct sys_info *sysinfo)
1320 {
1321         u32 reg32;
1322
1323         printk(BIOS_DEBUG, "Enabling System Memory IO... \n");
1324
1325         reg32 = MCHBAR32(RCVENMT);
1326         reg32 &= ~(0x3f << 6);
1327         MCHBAR32(RCVENMT) = reg32; /* [11:6] = 0 */
1328
1329         reg32 |= (1 << 11) | (1 << 9);
1330         MCHBAR32(RCVENMT) = reg32;
1331
1332         reg32 = MCHBAR32(DRTST);
1333         reg32 |= (1 << 3) | (1 << 2);
1334         MCHBAR32(DRTST) = reg32;
1335
1336         reg32 = MCHBAR32(DRTST);
1337         reg32 |= (1 << 6) | (1 << 4);
1338         MCHBAR32(DRTST) = reg32;
1339
1340         asm volatile ("nop; nop;" ::: "memory");
1341
1342         reg32 = MCHBAR32(DRTST);
1343
1344         /* Is channel 0 populated? */
1345         if (sysinfo->dimm[0] != SYSINFO_DIMM_NOT_POPULATED ||
1346                         sysinfo->dimm[1] != SYSINFO_DIMM_NOT_POPULATED)
1347                 reg32 |= (1 << 7) | (1 << 5);
1348         else
1349                 reg32 |= (1 << 31);
1350
1351         /* Is channel 1 populated? */
1352         if (sysinfo->dimm[2] != SYSINFO_DIMM_NOT_POPULATED ||
1353                         sysinfo->dimm[3] != SYSINFO_DIMM_NOT_POPULATED)
1354                 reg32 |= (1 << 9) | (1 << 8);
1355         else
1356                 reg32 |= (1 << 30);
1357
1358         MCHBAR32(DRTST) = reg32;
1359
1360         /* Activate DRAM Channel IO Buffers */
1361         if (sysinfo->dimm[0] != SYSINFO_DIMM_NOT_POPULATED ||
1362                         sysinfo->dimm[1] != SYSINFO_DIMM_NOT_POPULATED) {
1363                 reg32 = MCHBAR32(C0DRC1);
1364                 reg32 |= (1 << 8);
1365                 MCHBAR32(C0DRC1) = reg32;
1366         }
1367         if (sysinfo->dimm[2] != SYSINFO_DIMM_NOT_POPULATED ||
1368                         sysinfo->dimm[3] != SYSINFO_DIMM_NOT_POPULATED) {
1369                 reg32 = MCHBAR32(C1DRC1);
1370                 reg32 |= (1 << 8);
1371                 MCHBAR32(C1DRC1) = reg32;
1372         }
1373 }
1374
1375 struct dimm_size {
1376         unsigned long side1;
1377         unsigned long side2;
1378 };
1379
1380 static struct dimm_size sdram_get_dimm_size(u16 device)
1381 {
1382         /* Calculate the log base 2 size of a DIMM in bits */
1383         struct dimm_size sz;
1384         int value, low, rows, columns;
1385
1386         sz.side1 = 0;
1387         sz.side2 = 0;
1388
1389         rows = spd_read_byte(device, SPD_NUM_ROWS);     /* rows */
1390         if (rows < 0) goto hw_err;
1391         if ((rows & 0xf) == 0) goto val_err;
1392         sz.side1 += rows & 0xf;
1393
1394         columns = spd_read_byte(device, SPD_NUM_COLUMNS);       /* columns */
1395         if (columns < 0) goto hw_err;
1396         if ((columns & 0xf) == 0) goto val_err;
1397         sz.side1 += columns & 0xf;
1398
1399         value = spd_read_byte(device, SPD_NUM_BANKS_PER_SDRAM); /* banks */
1400         if (value < 0) goto hw_err;
1401         if ((value & 0xff) == 0) goto val_err;
1402         sz.side1 += log2(value & 0xff);
1403
1404         /* Get the module data width and convert it to a power of two */
1405         value = spd_read_byte(device, SPD_MODULE_DATA_WIDTH_MSB);       /* (high byte) */
1406         if (value < 0) goto hw_err;
1407         value &= 0xff;
1408         value <<= 8;
1409
1410         low = spd_read_byte(device, SPD_MODULE_DATA_WIDTH_LSB); /* (low byte) */
1411         if (low < 0) goto hw_err;
1412         value = value | (low & 0xff);
1413         if ((value != 72) && (value != 64)) goto val_err;
1414         sz.side1 += log2(value);
1415
1416         /* side 2 */
1417         value = spd_read_byte(device, SPD_NUM_DIMM_BANKS);      /* number of physical banks */
1418
1419         if (value < 0) goto hw_err;
1420         value &= 7;
1421         value++;
1422         if (value == 1) goto out;
1423         if (value != 2) goto val_err;
1424
1425         /* Start with the symmetrical case */
1426         sz.side2 = sz.side1;
1427
1428         if ((rows & 0xf0) == 0) goto out;       /* If symmetrical we are done */
1429
1430         /* Don't die here, I have not come across any of these to test what
1431          * actually happens.
1432          */
1433         printk(BIOS_ERR, "Assymetric DIMMs are not supported by this chipset\n");
1434
1435         sz.side2 -= (rows & 0x0f);              /* Subtract out rows on side 1 */
1436         sz.side2 += ((rows >> 4) & 0x0f);       /* Add in rows on side 2 */
1437
1438         sz.side2 -= (columns & 0x0f);           /* Subtract out columns on side 1 */
1439         sz.side2 += ((columns >> 4) & 0x0f);    /* Add in columns on side 2 */
1440
1441         goto out;
1442
1443  val_err:
1444         die("Bad SPD value\n");
1445  hw_err:
1446         /* If a hardware error occurs the spd rom probably does not exist.
1447          * In this case report that there is no memory
1448          */
1449         sz.side1 = 0;
1450         sz.side2 = 0;
1451  out:
1452         return sz;
1453 }
1454
1455 static void sdram_detect_dimm_size(struct sys_info * sysinfo)
1456 {
1457         int i;
1458
1459         for(i = 0; i < 2 * DIMM_SOCKETS; i++) {
1460                 struct dimm_size sz;
1461
1462                 sysinfo->banksize[i * 2] = 0;
1463                 sysinfo->banksize[(i * 2) + 1] = 0;
1464
1465                 if (sysinfo->dimm[i] == SYSINFO_DIMM_NOT_POPULATED)
1466                         continue;
1467
1468                 sz = sdram_get_dimm_size(DIMM_SPD_BASE + i);
1469
1470                 sysinfo->banks[i] = spd_read_byte(DIMM_SPD_BASE + i, SPD_NUM_BANKS_PER_SDRAM);  /* banks */
1471
1472                 if (sz.side1 < 30)
1473                         die("DDR-II rank size smaller than 128MB is not supported.\n");
1474
1475                 sysinfo->banksize[i * 2] = 1 << (sz.side1 - 28);
1476
1477                 printk(BIOS_DEBUG, "DIMM %d side 0 = %d MB\n", i, sysinfo->banksize[i * 2] * 32 );
1478
1479                 if (!sz.side2)
1480                         continue;
1481
1482                 /* If there is a second side, it has to have at least 128M, too */
1483                 if (sz.side2 < 30)
1484                         die("DDR-II rank size smaller than 128MB is not supported.\n");
1485
1486                 sysinfo->banksize[(i * 2) + 1] = 1 << (sz.side2 - 28);
1487
1488                 printk(BIOS_DEBUG, "DIMM %d side 1 = %d MB\n", i, sysinfo->banksize[(i * 2) + 1] * 32);
1489         }
1490 }
1491
1492 static int sdram_program_row_boundaries(struct sys_info *sysinfo)
1493 {
1494         int i;
1495         int cum0, cum1, tolud, tom;
1496
1497         printk(BIOS_DEBUG, "Setting RAM size... \n");
1498
1499         cum0 = 0;
1500         for(i = 0; i < 2 * DIMM_SOCKETS; i++) {
1501                 cum0 += sysinfo->banksize[i];
1502                 MCHBAR8(C0DRB0+i) = cum0;
1503         }
1504
1505         /* Assume we continue in Channel 1 where we stopped in Channel 0 */
1506         cum1 = cum0;
1507
1508         /* Exception: Interleaved starts from the beginning */
1509         if (sysinfo->interleaved)
1510                 cum1 = 0;
1511
1512 #if 0
1513         /* Exception: Channel 1 is not populated. C1DRB stays zero */
1514         if (sysinfo->dimm[2] == SYSINFO_DIMM_NOT_POPULATED &&
1515                         sysinfo->dimm[3] == SYSINFO_DIMM_NOT_POPULATED)
1516                 cum1 = 0;
1517 #endif
1518
1519         for(i = 0; i < 2 * DIMM_SOCKETS; i++) {
1520                 cum1 += sysinfo->banksize[i + 4];
1521                 MCHBAR8(C1DRB0+i) = cum1;
1522         }
1523
1524         /* Set TOLUD Top Of Low Usable DRAM */
1525         if (sysinfo->interleaved)
1526                 tolud = (cum0 + cum1) << 1;
1527         else
1528                 tolud = (cum1 ? cum1 : cum0)  << 1;
1529
1530         /* The TOM register has a different format */
1531         tom = tolud >> 3;
1532
1533         /* Limit the value of TOLUD to leave some space for PCI memory. */
1534         if (tolud > 0xd0)
1535                 tolud = 0xd0;   /* 3.25GB : 0.75GB */
1536
1537         pci_write_config8(PCI_DEV(0,0,0), TOLUD, tolud);
1538
1539         printk(BIOS_DEBUG, "C0DRB = 0x%08x\n", MCHBAR32(C0DRB0));
1540         printk(BIOS_DEBUG, "C1DRB = 0x%08x\n", MCHBAR32(C1DRB0));
1541         printk(BIOS_DEBUG, "TOLUD = 0x%04x\n", pci_read_config8(PCI_DEV(0,0,0), TOLUD));
1542
1543         pci_write_config16(PCI_DEV(0,0,0), TOM, tom);
1544
1545         return 0;
1546 }
1547
1548 static int sdram_set_row_attributes(struct sys_info *sysinfo)
1549 {
1550         int i, value;
1551         u16 dra0=0, dra1=0, dra = 0;
1552
1553         printk(BIOS_DEBUG, "Setting row attributes... \n");
1554         for(i=0; i < 2 * DIMM_SOCKETS; i++) {
1555                 u16 device;
1556                 u8 columnsrows;
1557
1558                 if (sysinfo->dimm[i] == SYSINFO_DIMM_NOT_POPULATED) {
1559                         continue;
1560                 }
1561
1562                 device = DIMM_SPD_BASE + i;
1563
1564                 value = spd_read_byte(device, SPD_NUM_ROWS);    /* rows */
1565                 columnsrows = (value & 0x0f);
1566
1567                 value = spd_read_byte(device, SPD_NUM_COLUMNS); /* columns */
1568                 columnsrows |= (value & 0xf) << 4;
1569
1570                 switch (columnsrows) {
1571                 case 0x9d: dra = 2; break;
1572                 case 0xad: dra = 3; break;
1573                 case 0xbd: dra = 4; break;
1574                 case 0xae: dra = 3; break;
1575                 case 0xbe: dra = 4; break;
1576                 default: die("Unsupported Rows/Columns. (DRA)");
1577                 }
1578
1579                 /* Double Sided DIMMs? */
1580                 if (sysinfo->banksize[(2 * i) + 1] != 0) {
1581                         dra = (dra << 4) | dra;
1582                 }
1583
1584                 if (i < DIMM_SOCKETS)
1585                         dra0 |= (dra << (i*8));
1586                 else
1587                         dra1 |= (dra << ((i - DIMM_SOCKETS)*8));
1588         }
1589
1590         MCHBAR16(C0DRA0) = dra0;
1591         MCHBAR16(C1DRA0) = dra1;
1592
1593         printk(BIOS_DEBUG, "C0DRA = 0x%04x\n", dra0);
1594         printk(BIOS_DEBUG, "C1DRA = 0x%04x\n", dra1);
1595
1596         return 0;
1597 }
1598
1599 static void sdram_set_bank_architecture(struct sys_info *sysinfo)
1600 {
1601         u32 off32;
1602         int i;
1603
1604         MCHBAR16(C1BNKARC) &= 0xff00;
1605         MCHBAR16(C0BNKARC) &= 0xff00;
1606
1607         off32 = C0BNKARC;
1608         for (i=0; i < 2 * DIMM_SOCKETS; i++) {
1609                 /* Switch to second channel */
1610                 if (i == DIMM_SOCKETS)
1611                         off32 = C1BNKARC;
1612
1613                 if (sysinfo->dimm[i] == SYSINFO_DIMM_NOT_POPULATED)
1614                         continue;
1615
1616                 if (sysinfo->banks[i] != 8)
1617                         continue;
1618
1619                 printk(BIOS_SPEW, "DIMM%d has 8 banks.\n", i);
1620
1621                 if (i & 1)
1622                         MCHBAR16(off32) |= 0x50;
1623                 else
1624                         MCHBAR16(off32) |= 0x05;
1625         }
1626 }
1627
1628 #define REFRESH_7_8US   1
1629 #define REFRESH_15_6US  0
1630 static void sdram_program_refresh_rate(struct sys_info *sysinfo)
1631 {
1632         u32 reg32;
1633
1634         if (sysinfo->refresh == REFRESH_7_8US) {
1635                 reg32 = (2 << 8); /* Refresh enabled at 7.8us */
1636         } else {
1637                 reg32 = (1 << 8); /* Refresh enabled at 15.6us */
1638         }
1639
1640         MCHBAR32(C0DRC0) &= ~(7 << 8);
1641         MCHBAR32(C0DRC0) |= reg32;
1642
1643         MCHBAR32(C1DRC0) &= ~(7 << 8);
1644         MCHBAR32(C1DRC0) |= reg32;
1645 }
1646
1647 static void sdram_program_cke_tristate(struct sys_info *sysinfo)
1648 {
1649         u32 reg32;
1650         int i;
1651
1652         reg32 = MCHBAR32(C0DRC1);
1653
1654         for (i=0; i < 4; i++) {
1655                 if (sysinfo->banksize[i] == 0) {
1656                         reg32 |= (1 << (16 + i));
1657                 }
1658         }
1659
1660         reg32 |= (1 << 12);
1661
1662         reg32 |= (1 << 11);
1663         MCHBAR32(C0DRC1) = reg32;
1664
1665         /* Do we have to do this if we're in Single Channel Mode?  */
1666         reg32 = MCHBAR32(C1DRC1);
1667
1668         for (i=4; i < 8; i++) {
1669                 if (sysinfo->banksize[i] == 0) {
1670                         reg32 |= (1 << (12 + i));
1671                 }
1672         }
1673
1674         reg32 |= (1 << 12);
1675
1676         reg32 |= (1 << 11);
1677         MCHBAR32(C1DRC1) = reg32;
1678 }
1679
1680 static void sdram_program_odt_tristate(struct sys_info *sysinfo)
1681 {
1682         u32 reg32;
1683         int i;
1684
1685         reg32 = MCHBAR32(C0DRC2);
1686
1687         for (i=0; i < 4; i++) {
1688                 if (sysinfo->banksize[i] == 0) {
1689                         reg32 |= (1 << (24 + i));
1690                 }
1691         }
1692         MCHBAR32(C0DRC2) = reg32;
1693
1694         reg32 = MCHBAR32(C1DRC2);
1695
1696         for (i=4; i < 8; i++) {
1697                 if (sysinfo->banksize[i] == 0) {
1698                         reg32 |= (1 << (20 + i));
1699                 }
1700         }
1701         MCHBAR32(C1DRC2) = reg32;
1702 }
1703
1704 static void sdram_set_timing_and_control(struct sys_info *sysinfo)
1705 {
1706         u32 reg32, off32;
1707         u32 tWTR;
1708         u32 temp_drt;
1709         int i, page_size;
1710
1711         static const u8 const drt0_table[] = {
1712           /* CL 3, 4, 5 */
1713                 3, 4, 5,        /* FSB533/400, DDR533/400 */
1714                 4, 5, 6,        /* FSB667, DDR533/400 */
1715                 4, 5, 6,        /* FSB667, DDR667 */
1716         };
1717
1718         static const u8 const cas_table[] = {
1719                 2, 1, 0, 3
1720         };
1721
1722         reg32 = MCHBAR32(C0DRC0);
1723         reg32 |= (1 << 2);      /* Burst Length 8 */
1724         reg32 &= ~( (1 << 13) | (1 << 12) );
1725         MCHBAR32(C0DRC0) = reg32;
1726
1727         reg32 = MCHBAR32(C1DRC0);
1728         reg32 |= (1 << 2);      /* Burst Length 8 */
1729         reg32 &= ~( (1 << 13) | (1 << 12) );
1730         MCHBAR32(C1DRC0) = reg32;
1731
1732         if (!sysinfo->dual_channel && sysinfo->dimm[1] !=
1733                         SYSINFO_DIMM_NOT_POPULATED) {
1734                 reg32 = MCHBAR32(C0DRC0);
1735                 reg32 |= (1 << 15);
1736                 MCHBAR32(C0DRC0) = reg32;
1737         }
1738
1739         sdram_program_refresh_rate(sysinfo);
1740
1741         sdram_program_cke_tristate(sysinfo);
1742
1743         sdram_program_odt_tristate(sysinfo);
1744
1745         /* Calculate DRT0 */
1746
1747         temp_drt = 0;
1748
1749         /* B2B Write Precharge (same bank) = CL-1 + BL/2 + tWR */
1750         reg32 = (sysinfo->cas - 1) + (BURSTLENGTH / 2) + sysinfo->twr;
1751         temp_drt |= (reg32 << 28);
1752
1753         /* Write Auto Precharge (same bank) = CL-1 + BL/2 + tWR + tRP */
1754         reg32 += sysinfo->trp;
1755         temp_drt |= (reg32 << 4);
1756
1757         if (sysinfo->memory_frequency == 667) {
1758                 tWTR = 3; /* 667MHz */
1759         } else {
1760                 tWTR = 2; /* 400 and 533 */
1761         }
1762
1763         /* B2B Write to Read Command Spacing */
1764         reg32 = (sysinfo->cas - 1) + (BURSTLENGTH / 2) + tWTR;
1765         temp_drt |= (reg32 << 24);
1766
1767         /* CxDRT0 [23:22], [21:20], [19:18] [16] have fixed values */
1768         temp_drt |= ( (1 << 22) | (3 << 20) | (1 << 18) | (0 << 16) );
1769
1770         /* Program Write Auto Precharge to Activate */
1771         off32 = 0;
1772         if (sysinfo->fsb_frequency == 667) { /* 667MHz FSB */
1773                 off32 += 3;
1774         }
1775         if (sysinfo->memory_frequency == 667) {
1776                 off32 += 3;
1777         }
1778         off32 += sysinfo->cas - 3;
1779         reg32 = drt0_table[off32];
1780         temp_drt |= (reg32 << 11);
1781
1782         /* Read Auto Precharge to Activate */
1783
1784         temp_drt |= (8 << 0);
1785
1786         MCHBAR32(C0DRT0) = temp_drt;
1787         MCHBAR32(C1DRT0) = temp_drt;
1788
1789         /* Calculate DRT1 */
1790
1791         temp_drt = MCHBAR32(C0DRT1) & 0x00020088;
1792
1793         /* DRAM RASB Precharge */
1794         temp_drt |= (sysinfo->trp - 2) << 0;
1795
1796         /* DRAM RASB to CASB Delay */
1797         temp_drt |= (sysinfo->trcd - 2) << 4;
1798
1799         /* CASB Latency */
1800         temp_drt |= (cas_table[sysinfo->cas - 3]) << 8;
1801
1802         /* Refresh Cycle Time */
1803         temp_drt |= (sysinfo->trfc) << 10;
1804
1805         /* Pre-All to Activate Delay */
1806         temp_drt |= (0 << 16);
1807
1808         /* Precharge to Precharge Delay stays at 1 clock */
1809         temp_drt |= (0 << 18);
1810
1811         /* Activate to Precharge Delay */
1812         temp_drt |= (sysinfo->tras << 19);
1813
1814         /* Read to Precharge (tRTP) */
1815         if (sysinfo->memory_frequency == 667) {
1816                 temp_drt |= (1 << 28);
1817         } else {
1818                 temp_drt |= (0 << 28);
1819         }
1820
1821         /* Determine page size */
1822         reg32 = 0;
1823         page_size = 1; /* Default: 1k pagesize */
1824         for (i=0; i< 2*DIMM_SOCKETS; i++) {
1825                 if (sysinfo->dimm[i] == SYSINFO_DIMM_X16DS ||
1826                                 sysinfo->dimm[i] == SYSINFO_DIMM_X16SS)
1827                         page_size = 2; /* 2k pagesize */
1828         }
1829
1830         if (sysinfo->memory_frequency == 533 && page_size == 2) {
1831                 reg32 = 1;
1832         }
1833         if (sysinfo->memory_frequency == 667) {
1834                 reg32 = page_size;
1835         }
1836
1837         temp_drt |= (reg32 << 30);
1838
1839         MCHBAR32(C0DRT1) = temp_drt;
1840         MCHBAR32(C1DRT1) = temp_drt;
1841
1842         /* Program DRT2 */
1843         reg32 = MCHBAR32(C0DRT2);
1844         reg32 &= ~(1 << 8);
1845         MCHBAR32(C0DRT2) = reg32;
1846
1847         reg32 = MCHBAR32(C1DRT2);
1848         reg32 &= ~(1 << 8);
1849         MCHBAR32(C1DRT2) = reg32;
1850
1851         /* Calculate DRT3 */
1852         temp_drt = MCHBAR32(C0DRT3) & ~0x07ffffff;
1853
1854         /* Get old tRFC value */
1855         reg32 = MCHBAR32(C0DRT1) >> 10;
1856         reg32 &= 0x3f;
1857
1858         /* 788nS - tRFC */
1859         switch (sysinfo->memory_frequency) {
1860         case 400: /* 5nS */
1861                 reg32 = ((78800 / 500) - reg32) & 0x1ff;
1862                 reg32 |= (0x8c << 16) | (0x0c << 10); /* 1 us */
1863                 break;
1864         case 533: /* 3.75nS */
1865                 reg32 = ((78800 / 375) - reg32) & 0x1ff;
1866                 reg32 |= (0xba << 16) | (0x10 << 10); /* 1 us */
1867                 break;
1868         case 667: /* 3nS */
1869                 reg32 = ((78800 / 300) - reg32) & 0x1ff;
1870                 reg32 |= (0xe9 << 16) | (0x14 << 10); /* 1 us */
1871                 break;
1872         }
1873
1874         temp_drt |= reg32;
1875
1876         MCHBAR32(C0DRT3) = temp_drt;
1877         MCHBAR32(C1DRT3) = temp_drt;
1878 }
1879
1880 static void sdram_set_channel_mode(struct sys_info *sysinfo)
1881 {
1882         u32 reg32;
1883
1884         printk(BIOS_DEBUG, "Setting mode of operation for memory channels...");
1885
1886         if (sdram_capabilities_interleave() &&
1887                     ( ( sysinfo->banksize[0] + sysinfo->banksize[1] +
1888                         sysinfo->banksize[2] + sysinfo->banksize[3] ) ==
1889                       ( sysinfo->banksize[4] + sysinfo->banksize[5] +
1890                         sysinfo->banksize[6] + sysinfo->banksize[7] ) ) ) {
1891                 /* Both channels equipped with DIMMs of the same size */
1892                 sysinfo->interleaved = 1;
1893         } else {
1894                 sysinfo->interleaved = 0;
1895         }
1896
1897         reg32 = MCHBAR32(DCC);
1898         reg32 &= ~(7 << 0);
1899
1900         if(sysinfo->interleaved) {
1901                 /* Dual Channel Interleaved */
1902                 printk(BIOS_DEBUG, "Dual Channel Interleaved.\n");
1903                 reg32 |= (1 << 1);
1904         } else if (sysinfo->dimm[0] == SYSINFO_DIMM_NOT_POPULATED &&
1905                         sysinfo->dimm[1] == SYSINFO_DIMM_NOT_POPULATED) {
1906                 /* Channel 1 only */
1907                 printk(BIOS_DEBUG, "Single Channel 1 only.\n");
1908                 reg32 |= (1 << 2);
1909         } else if (sdram_capabilities_dual_channel() && sysinfo->dimm[2] !=
1910                         SYSINFO_DIMM_NOT_POPULATED) {
1911                 /* Dual Channel Assymetric */
1912                 printk(BIOS_DEBUG, "Dual Channel Assymetric.\n");
1913                 reg32 |= (1 << 0);
1914         } else {
1915                 /* All bits 0 means Single Channel 0 operation */
1916                 printk(BIOS_DEBUG, "Single Channel 0 only.\n");
1917         }
1918
1919         /* Now disable channel XORing */
1920         reg32 |= (1 << 10);
1921
1922         MCHBAR32(DCC) = reg32;
1923
1924         PRINTK_DEBUG("DCC=0x%08x\n", MCHBAR32(DCC));
1925 }
1926
1927 static void sdram_program_pll_settings(struct sys_info *sysinfo)
1928 {
1929         volatile u16 reg16;
1930
1931         MCHBAR32(PLLMON) = 0x80800000;
1932
1933         sysinfo->fsb_frequency = fsbclk();
1934         if (sysinfo->fsb_frequency == -1)
1935                 die("Unsupported FSB speed");
1936
1937         /* Program CPCTL according to FSB speed */
1938         /* Only write the lower byte */
1939         switch (sysinfo->fsb_frequency) {
1940         case 400: MCHBAR8(CPCTL) = 0x90; break; /* FSB400 */
1941         case 533: MCHBAR8(CPCTL) = 0x95; break; /* FSB533 */
1942         case 667: MCHBAR8(CPCTL) = 0x8d; break; /* FSB667 */
1943         }
1944
1945         MCHBAR16(CPCTL) &= ~(1 << 11);
1946
1947         reg16 = MCHBAR16(CPCTL); /* Read back register to activate settings */
1948 }
1949
1950 static void sdram_program_graphics_frequency(struct sys_info *sysinfo)
1951 {
1952         u8  reg8;
1953         u16 reg16;
1954         u8  freq, second_vco, voltage;
1955
1956 #define CRCLK_166MHz    0x00
1957 #define CRCLK_200MHz    0x01
1958 #define CRCLK_250MHz    0x03
1959 #define CRCLK_400MHz    0x05
1960
1961 #define CDCLK_200MHz    0x00
1962 #define CDCLK_320MHz    0x40
1963
1964 #define VOLTAGE_1_05    0x00
1965 #define VOLTAGE_1_50    0x01
1966
1967         printk(BIOS_DEBUG, "Setting Graphics Frequency... \n");
1968
1969         printk(BIOS_DEBUG, "FSB: %d MHz ", sysinfo->fsb_frequency);
1970
1971         voltage = VOLTAGE_1_05;
1972         if (MCHBAR32(DFT_STRAP1) & (1 << 20))
1973                 voltage = VOLTAGE_1_50;
1974         printk(BIOS_DEBUG, "Voltage: %s ", (voltage==VOLTAGE_1_05)?"1.05V":"1.5V");
1975
1976         /* Gate graphics hardware for frequency change */
1977         reg8 = pci_read_config16(PCI_DEV(0,2,0), GCFC + 1);
1978         reg8 = (1<<3) | (1<<1); /* disable crclk, gate cdclk */
1979         pci_write_config8(PCI_DEV(0,2,0), GCFC + 1, reg8);
1980
1981         /* Get graphics frequency capabilities */
1982         reg8 = sdram_capabilities_core_frequencies();
1983
1984         freq = CRCLK_250MHz;
1985         switch (reg8) {
1986         case GFX_FREQUENCY_CAP_ALL:
1987                 if (voltage == VOLTAGE_1_05)
1988                         freq = CRCLK_250MHz;
1989                 else
1990                         freq = CRCLK_400MHz; /* 1.5V requires 400MHz */
1991                 break;
1992         case GFX_FREQUENCY_CAP_250MHZ: freq = CRCLK_250MHz; break;
1993         case GFX_FREQUENCY_CAP_200MHZ: freq = CRCLK_200MHz; break;
1994         case GFX_FREQUENCY_CAP_166MHZ: freq = CRCLK_166MHz; break;
1995         }
1996
1997         if (freq != CRCLK_400MHz) {
1998                 /* What chipset are we? Force 166MHz for GMS */
1999                 reg8 = (pci_read_config8(PCI_DEV(0, 0x00,0), 0xe7) & 0x70) >> 4;
2000                 if (reg8==2)
2001                         freq = CRCLK_166MHz;
2002         }
2003
2004         printk(BIOS_DEBUG, "Render: ");
2005         switch (freq) {
2006         case CRCLK_166MHz: printk(BIOS_DEBUG, "166Mhz"); break;
2007         case CRCLK_200MHz: printk(BIOS_DEBUG, "200Mhz"); break;
2008         case CRCLK_250MHz: printk(BIOS_DEBUG, "250Mhz"); break;
2009         case CRCLK_400MHz: printk(BIOS_DEBUG, "400Mhz"); break;
2010         }
2011
2012         if (i945_silicon_revision() == 0) {
2013                 sysinfo->mvco4x = 1;
2014         } else {
2015                 sysinfo->mvco4x = 0;
2016         }
2017
2018         second_vco = 0;
2019
2020         if (voltage == VOLTAGE_1_50) {
2021                 second_vco = 1;
2022         } else if ((i945_silicon_revision() > 0) && (freq == CRCLK_250MHz))  {
2023                 u16 mem = sysinfo->memory_frequency;
2024                 u16 fsb = sysinfo->fsb_frequency;
2025
2026                 if ( (fsb == 667 && mem == 533) ||
2027                      (fsb == 533 && mem == 533) ||
2028                      (fsb == 533 && mem == 400)) {
2029                         second_vco = 1;
2030                 }
2031
2032                 if (fsb == 667 && mem == 533)
2033                         sysinfo->mvco4x = 1;
2034         }
2035
2036         if (second_vco) {
2037                 sysinfo->clkcfg_bit7=1;
2038         } else {
2039                 sysinfo->clkcfg_bit7=0;
2040         }
2041
2042         /* Graphics Core Render Clock */
2043         reg16 = pci_read_config16(PCI_DEV(0,2,0), GCFC);
2044         reg16 &= ~( (7 << 0) | (1 << 13) );
2045         reg16 |= freq;
2046         pci_write_config16(PCI_DEV(0,2,0), GCFC, reg16);
2047
2048         /* Graphics Core Display Clock */
2049         reg8 = pci_read_config8(PCI_DEV(0,2,0), GCFC);
2050         reg8 &= ~( (1<<7) | (7<<4) );
2051
2052         if (voltage == VOLTAGE_1_05) {
2053                 reg8 |= CDCLK_200MHz;
2054                 printk(BIOS_DEBUG, " Display: 200MHz\n");
2055         } else {
2056                 reg8 |= CDCLK_320MHz;
2057                 printk(BIOS_DEBUG, " Display: 320MHz\n");
2058         }
2059         pci_write_config8(PCI_DEV(0,2,0), GCFC, reg8);
2060
2061         reg8 = pci_read_config8(PCI_DEV(0,2,0), GCFC + 1);
2062
2063         reg8 |= (1<<3) | (1<<1);
2064         pci_write_config8(PCI_DEV(0,2,0), GCFC + 1, reg8);
2065
2066         reg8 |= 0x0f;
2067         pci_write_config8(PCI_DEV(0,2,0), GCFC + 1, reg8);
2068
2069         /* Ungate core render and display clocks */
2070         reg8 &= 0xf0;
2071         pci_write_config8(PCI_DEV(0,2,0), GCFC + 1, reg8);
2072 }
2073
2074 static void sdram_program_memory_frequency(struct sys_info *sysinfo)
2075 {
2076         u32 clkcfg;
2077         u8 reg8;
2078
2079         printk(BIOS_DEBUG, "Setting Memory Frequency... ");
2080
2081         clkcfg = MCHBAR32(CLKCFG);
2082
2083         printk(BIOS_DEBUG, "CLKCFG=0x%08x, ", clkcfg);
2084
2085         clkcfg &= ~( (1 << 12) | (1 << 7) | ( 7 << 4) );
2086
2087         if (sysinfo->mvco4x) {
2088                 printk(BIOS_DEBUG, "MVCO 4x, ");
2089                 clkcfg &= ~(1 << 12);
2090         }
2091
2092         if (sysinfo->clkcfg_bit7) {
2093                 printk(BIOS_DEBUG, "second VCO, ");
2094
2095                 clkcfg |= (1 << 7);
2096         }
2097
2098         switch (sysinfo->memory_frequency) {
2099         case 400: clkcfg |= (2 << 4); break;
2100         case 533: clkcfg |= (3 << 4); break;
2101         case 667: clkcfg |= (4 << 4); break;
2102         default: die("Target Memory Frequency Error");
2103         }
2104
2105         if (MCHBAR32(CLKCFG) == clkcfg) {
2106                 printk(BIOS_DEBUG, "ok (unchanged)\n");
2107                 return;
2108         }
2109
2110         MCHBAR32(CLKCFG) = clkcfg;
2111
2112         /* Make sure the following code is in the
2113          * cache before we execute it.
2114          */
2115         goto cache_code;
2116 vco_update:
2117         reg8 = pci_read_config8(PCI_DEV(0,0x1f,0), 0xa2);
2118         reg8 &= ~(1 << 7);
2119         pci_write_config8(PCI_DEV(0, 0x1f, 0), 0xa2, reg8);
2120
2121         clkcfg &= ~(1 << 10);
2122         MCHBAR32(CLKCFG) = clkcfg;
2123         clkcfg |= (1 << 10);
2124         MCHBAR32(CLKCFG) = clkcfg;
2125
2126         asm volatile (
2127                 "       movl $0x100, %%ecx\n"
2128                 "delay_update:\n"
2129                 "       nop\n"
2130                 "       nop\n"
2131                 "       nop\n"
2132                 "       nop\n"
2133                 "       loop delay_update\n"
2134                 : /* No outputs */
2135                 : /* No inputs */
2136                 : "%ecx", "memory"
2137                 );
2138
2139         clkcfg &= ~(1 << 10);
2140         MCHBAR32(CLKCFG) = clkcfg;
2141
2142         goto out;
2143 cache_code:
2144         goto vco_update;
2145 out:
2146
2147         printk(BIOS_DEBUG, "CLKCFG=0x%08x, ", MCHBAR32(CLKCFG));
2148         printk(BIOS_DEBUG, "ok\n");
2149 }
2150
2151 static void sdram_program_clock_crossing(void)
2152 {
2153         int idx = 0;
2154
2155         /**
2156          * We add the indices according to our clocks from CLKCFG.
2157          */
2158 #if CONFIG_I945GM
2159         static const u32 data_clock_crossing[] = {
2160                 0x00100401, 0x00000000, /* DDR400 FSB400 */
2161                 0xffffffff, 0xffffffff, /*  nonexistant  */
2162                 0xffffffff, 0xffffffff, /*  nonexistant  */
2163
2164                 0x08040120, 0x00000000, /* DDR400 FSB533 */
2165                 0x00100401, 0x00000000, /* DDR533 FSB533 */
2166                 0x00010402, 0x00000000, /* DDR667 FSB533 - fake values */
2167
2168                 0x04020120, 0x00000010, /* DDR400 FSB667 */
2169                 0x10040280, 0x00000040, /* DDR533 FSB667 */
2170                 0x00100401, 0x00000000, /* DDR667 FSB667 */
2171
2172                 0xffffffff, 0xffffffff, /*  nonexistant  */
2173                 0xffffffff, 0xffffffff, /*  nonexistant  */
2174                 0xffffffff, 0xffffffff, /*  nonexistant  */
2175
2176                 0xffffffff, 0xffffffff, /*  nonexistant  */
2177                 0xffffffff, 0xffffffff, /*  nonexistant  */
2178                 0xffffffff, 0xffffffff, /*  nonexistant  */
2179         };
2180
2181         static const u32 command_clock_crossing[] = {
2182                 0x04020208, 0x00000000, /* DDR400 FSB400 */
2183                 0xffffffff, 0xffffffff, /*  nonexistant  */
2184                 0xffffffff, 0xffffffff, /*  nonexistant  */
2185
2186                 0x00060108, 0x00000000, /* DDR400 FSB533 */
2187                 0x04020108, 0x00000000, /* DDR533 FSB533 */
2188                 0xffffffff, 0xffffffff, /*  nonexistant  */
2189
2190                 0x00040318, 0x00000000, /* DDR400 FSB667 */
2191                 0x04020118, 0x00000000, /* DDR533 FSB667 */
2192                 0x02010804, 0x00000000, /* DDR667 FSB667 */
2193
2194                 0xffffffff, 0xffffffff, /*  nonexistant  */
2195                 0xffffffff, 0xffffffff, /*  nonexistant  */
2196                 0xffffffff, 0xffffffff, /*  nonexistant  */
2197
2198                 0xffffffff, 0xffffffff, /*  nonexistant  */
2199                 0xffffffff, 0xffffffff, /*  nonexistant  */
2200                 0xffffffff, 0xffffffff, /*  nonexistant  */
2201         };
2202
2203 #endif
2204 #if CONFIG_I945GC
2205         /* i945 G/P */
2206         static const u32 data_clock_crossing[] = {
2207                 0xffffffff, 0xffffffff, /*  nonexistant  */
2208                 0xffffffff, 0xffffffff, /*  nonexistant  */
2209                 0xffffffff, 0xffffffff, /*  nonexistant  */
2210
2211                 0x10080201, 0x00000000, /* DDR400 FSB533 */
2212                 0x00100401, 0x00000000, /* DDR533 FSB533 */
2213                 0x00010402, 0x00000000, /* DDR667 FSB533 - fake values */
2214
2215                 0xffffffff, 0xffffffff, /*  nonexistant  */
2216                 0xffffffff, 0xffffffff, /*  nonexistant  */
2217                 0xffffffff, 0xffffffff, /*  nonexistant  */
2218
2219                 0x04020108, 0x00000000, /* DDR400 FSB800 */
2220                 0x00020108, 0x00000000, /* DDR533 FSB800 */
2221                 0x00080201, 0x00000000, /* DDR667 FSB800 */
2222
2223                 0x00010402, 0x00000000, /* DDR400 FSB1066 */
2224                 0x04020108, 0x00000000, /* DDR533 FSB1066 */
2225                 0x08040110, 0x00000000, /* DDR667 FSB1066 */
2226         };
2227
2228         static const u32 command_clock_crossing[] = {
2229                 0xffffffff, 0xffffffff, /*  nonexistant  */
2230                 0xffffffff, 0xffffffff, /*  nonexistant  */
2231                 0xffffffff, 0xffffffff, /*  nonexistant  */
2232
2233                 0x00010800, 0x00000402, /* DDR400 FSB533 */
2234                 0x01000400, 0x00000200, /* DDR533 FSB533 */
2235                 0x00020904, 0x00000000, /* DDR667 FSB533 - fake values */
2236
2237                 0xffffffff, 0xffffffff, /*  nonexistant  */
2238                 0xffffffff, 0xffffffff, /*  nonexistant  */
2239                 0xffffffff, 0xffffffff, /*  nonexistant  */
2240
2241                 0x02010804, 0x00000000, /* DDR400 FSB800 */
2242                 0x00010402, 0x00000000, /* DDR533 FSB800 */
2243                 0x04020180, 0x00000008, /* DDR667 FSB800 */
2244
2245                 0x00020904, 0x00000000, /* DDR400 FSB1066 */
2246                 0x02010804, 0x00000000, /* DDR533 FSB1066 */
2247                 0x180601c0, 0x00000020, /* DDR667 FSB1066 */
2248         };
2249 #endif
2250
2251         printk(BIOS_DEBUG, "Programming Clock Crossing...");
2252
2253         printk(BIOS_DEBUG, "MEM=");
2254         switch (memclk()) {
2255         case 400:       printk(BIOS_DEBUG, "400"); idx += 0; break;
2256         case 533:       printk(BIOS_DEBUG, "533"); idx += 2; break;
2257         case 667:       printk(BIOS_DEBUG, "667"); idx += 4; break;
2258         default: printk(BIOS_DEBUG, "RSVD %x", memclk()); return;
2259         }
2260
2261         printk(BIOS_DEBUG, " FSB=");
2262         switch (fsbclk()) {
2263         case 400:       printk(BIOS_DEBUG, "400"); idx += 0; break;
2264         case 533:       printk(BIOS_DEBUG, "533"); idx += 6; break;
2265         case 667:       printk(BIOS_DEBUG, "667"); idx += 12; break;
2266         case 800:       printk(BIOS_DEBUG, "800"); idx += 18; break;
2267         case 1066:      printk(BIOS_DEBUG, "1066"); idx += 24; break;
2268         default: printk(BIOS_DEBUG, "RSVD %x\n", fsbclk()); return;
2269         }
2270
2271         if (command_clock_crossing[idx]==0xffffffff) {
2272                 printk(BIOS_DEBUG, "Invalid MEM/FSB combination!\n");
2273         }
2274
2275         MCHBAR32(CCCFT + 0) = command_clock_crossing[idx];
2276         MCHBAR32(CCCFT + 4) = command_clock_crossing[idx + 1];
2277
2278         MCHBAR32(C0DCCFT + 0) = data_clock_crossing[idx];
2279         MCHBAR32(C0DCCFT + 4) = data_clock_crossing[idx + 1];
2280         MCHBAR32(C1DCCFT + 0) = data_clock_crossing[idx];
2281         MCHBAR32(C1DCCFT + 4) = data_clock_crossing[idx + 1];
2282
2283         printk(BIOS_DEBUG, "... ok\n");
2284 }
2285
2286 static void sdram_disable_fast_dispatch(void)
2287 {
2288         u32 reg32;
2289
2290         reg32 = MCHBAR32(FSBPMC3);
2291         reg32 |= (1 << 1);
2292         MCHBAR32(FSBPMC3) = reg32;
2293
2294         reg32 = MCHBAR32(SBTEST);
2295         reg32 |= (3 << 1);
2296         MCHBAR32(SBTEST) = reg32;
2297 }
2298
2299 static void sdram_pre_jedec_initialization(void)
2300 {
2301         u32 reg32;
2302
2303         reg32 = MCHBAR32(WCC);
2304         reg32 &= 0x113ff3ff;
2305         reg32 |= (4 << 29) | (3 << 25) | (1 << 10);
2306         MCHBAR32(WCC) = reg32;
2307
2308         MCHBAR32(SMVREFC) |= (1 << 6);
2309
2310         MCHBAR32(MMARB0) &= ~(3 << 17);
2311         MCHBAR32(MMARB0) |= (1 << 21) | (1 << 16);
2312
2313         MCHBAR32(MMARB1) &= ~(7 << 8);
2314         MCHBAR32(MMARB1) |= (3 << 8);
2315
2316         /* Adaptive Idle Timer Control */
2317         MCHBAR32(C0AIT) = 0x000006c4;
2318         MCHBAR32(C0AIT+4) = 0x871a066d;
2319
2320         MCHBAR32(C1AIT) = 0x000006c4;
2321         MCHBAR32(C1AIT+4) = 0x871a066d;
2322 }
2323
2324 #define EA_DUALCHANNEL_XOR_BANK_RANK_MODE       (0xd4 << 24)
2325 #define EA_DUALCHANNEL_XOR_BANK_MODE            (0xf4 << 24)
2326 #define EA_DUALCHANNEL_BANK_RANK_MODE           (0xc2 << 24)
2327 #define EA_DUALCHANNEL_BANK_MODE                (0xe2 << 24)
2328 #define EA_SINGLECHANNEL_XOR_BANK_RANK_MODE     (0x91 << 24)
2329 #define EA_SINGLECHANNEL_XOR_BANK_MODE          (0xb1 << 24)
2330 #define EA_SINGLECHANNEL_BANK_RANK_MODE         (0x80 << 24)
2331 #define EA_SINGLECHANNEL_BANK_MODE              (0xa0 << 24)
2332
2333 static void sdram_enhanced_addressing_mode(struct sys_info *sysinfo)
2334 {
2335         u32 chan0 = 0, chan1 = 0;
2336         int chan0_dualsided, chan1_dualsided, chan0_populated, chan1_populated;
2337
2338         chan0_populated =  (sysinfo->dimm[0] != SYSINFO_DIMM_NOT_POPULATED ||
2339                         sysinfo->dimm[1] != SYSINFO_DIMM_NOT_POPULATED);
2340         chan1_populated = (sysinfo->dimm[0] != SYSINFO_DIMM_NOT_POPULATED ||
2341                         sysinfo->dimm[1] != SYSINFO_DIMM_NOT_POPULATED);
2342         chan0_dualsided = (sysinfo->banksize[1] || sysinfo->banksize[3]);
2343         chan1_dualsided = (sysinfo->banksize[5] || sysinfo->banksize[7]);
2344
2345         if (sdram_capabilities_enhanced_addressing_xor()) {
2346                 if (!sysinfo->interleaved) {
2347                         /* Single Channel & Dual Channel Assymetric */
2348                         if (chan0_populated) {
2349                                 if (chan0_dualsided) {
2350                                         chan0 = EA_SINGLECHANNEL_XOR_BANK_RANK_MODE;
2351                                 } else {
2352                                         chan0 = EA_SINGLECHANNEL_XOR_BANK_MODE;
2353                                 }
2354                         }
2355                         if (chan1_populated) {
2356                                 if (chan1_dualsided) {
2357                                         chan1 = EA_SINGLECHANNEL_XOR_BANK_RANK_MODE;
2358                                 } else {
2359                                         chan1 = EA_SINGLECHANNEL_XOR_BANK_MODE;
2360                                 }
2361                         }
2362                 } else {
2363                         /* Interleaved has always both channels populated */
2364                         if (chan0_dualsided) {
2365                                 chan0 = EA_DUALCHANNEL_XOR_BANK_RANK_MODE;
2366                         } else {
2367                                 chan0 = EA_DUALCHANNEL_XOR_BANK_MODE;
2368                         }
2369
2370                         if (chan1_dualsided) {
2371                                 chan1 = EA_DUALCHANNEL_XOR_BANK_RANK_MODE;
2372                         } else {
2373                                 chan1 = EA_DUALCHANNEL_XOR_BANK_MODE;
2374                         }
2375                 }
2376         } else {
2377                 if (!sysinfo->interleaved) {
2378                         /* Single Channel & Dual Channel Assymetric */
2379                         if (chan0_populated) {
2380                                 if (chan0_dualsided) {
2381                                         chan0 = EA_SINGLECHANNEL_BANK_RANK_MODE;
2382                                 } else {
2383                                         chan0 = EA_SINGLECHANNEL_BANK_MODE;
2384                                 }
2385                         }
2386                         if (chan1_populated) {
2387                                 if (chan1_dualsided) {
2388                                         chan1 = EA_SINGLECHANNEL_BANK_RANK_MODE;
2389                                 } else {
2390                                         chan1 = EA_SINGLECHANNEL_BANK_MODE;
2391                                 }
2392                         }
2393                 } else {
2394                         /* Interleaved has always both channels populated */
2395                         if (chan0_dualsided) {
2396                                 chan0 = EA_DUALCHANNEL_BANK_RANK_MODE;
2397                         } else {
2398                                 chan0 = EA_DUALCHANNEL_BANK_MODE;
2399                         }
2400
2401                         if (chan1_dualsided) {
2402                                 chan1 = EA_DUALCHANNEL_BANK_RANK_MODE;
2403                         } else {
2404                                 chan1 = EA_DUALCHANNEL_BANK_MODE;
2405                         }
2406                 }
2407         }
2408
2409         MCHBAR32(C0DRC1) &= 0x00ffffff;
2410         MCHBAR32(C0DRC1) |= chan0;
2411         MCHBAR32(C1DRC1) &= 0x00ffffff;
2412         MCHBAR32(C1DRC1) |= chan1;
2413 }
2414
2415 static void sdram_post_jedec_initialization(struct sys_info *sysinfo)
2416 {
2417         u32 reg32;
2418
2419         /* Enable Channel XORing for Dual Channel Interleave */
2420         if (sysinfo->interleaved) {
2421
2422                 reg32 = MCHBAR32(DCC);
2423 #if CONFIG_CHANNEL_XOR_RANDOMIZATION
2424                 reg32 &= ~(1 << 10);
2425                 reg32 |= (1 << 9);
2426 #else
2427                 reg32 &= ~(1 << 9);
2428 #endif
2429                 MCHBAR32(DCC) = reg32;
2430         }
2431
2432         /* DRAM mode optimizations */
2433         sdram_enhanced_addressing_mode(sysinfo);
2434
2435         reg32 = MCHBAR32(FSBPMC3);
2436         reg32 &= ~(1 << 1);
2437         MCHBAR32(FSBPMC3) = reg32;
2438
2439         reg32 = MCHBAR32(SBTEST);
2440         reg32 &= ~(1 << 2);
2441         MCHBAR32(SBTEST) = reg32;
2442
2443         reg32 = MCHBAR32(SBOCC);
2444         reg32 &= 0xffbdb6ff;
2445         reg32 |= (0xbdb6 << 8) | (1 << 0);
2446         MCHBAR32(SBOCC) = reg32;
2447 }
2448
2449 static void sdram_power_management(struct sys_info *sysinfo)
2450 {
2451         u8 reg8;
2452         u16 reg16;
2453         u32 reg32;
2454         int integrated_graphics = 1;
2455         int i;
2456
2457         reg32 = MCHBAR32(C0DRT2);
2458         reg32 &= 0xffffff00;
2459         /* Idle timer = 8 clocks, CKE idle timer = 16 clocks */
2460         reg32 |= (1 << 5) | (1 << 4);
2461         MCHBAR32(C0DRT2) = reg32;
2462
2463         reg32 = MCHBAR32(C1DRT2);
2464         reg32 &= 0xffffff00;
2465         /* Idle timer = 8 clocks, CKE idle timer = 16 clocks */
2466         reg32 |= (1 << 5) | (1 << 4);
2467         MCHBAR32(C1DRT2) = reg32;
2468
2469         reg32 = MCHBAR32(C0DRC1);
2470
2471         reg32 |= (1 << 12) | (1 << 11);
2472         MCHBAR32(C0DRC1) = reg32;
2473
2474         reg32 = MCHBAR32(C1DRC1);
2475
2476         reg32 |= (1 << 12) | (1 << 11);
2477         MCHBAR32(C1DRC1) = reg32;
2478
2479         if (i945_silicon_revision()>1) {
2480                 /* FIXME bits 5 and 0 only if PCIe graphics is disabled */
2481                 u16 peg_bits = (1 << 5) | (1 << 0);
2482
2483                 MCHBAR16(UPMC1) = 0x1010 | peg_bits;
2484         } else {
2485                 /* FIXME bits 5 and 0 only if PCIe graphics is disabled */
2486                 u16 peg_bits = (1 << 5) | (1 << 0);
2487
2488                 /* Rev 0 and 1 */
2489                 MCHBAR16(UPMC1) = 0x0010 | peg_bits;
2490         }
2491
2492         reg16 = MCHBAR16(UPMC2);
2493         reg16 &= 0xfc00;
2494         reg16 |= 0x0100;
2495         MCHBAR16(UPMC2) = reg16;
2496
2497         MCHBAR32(UPMC3) = 0x000f06ff;
2498
2499         for (i=0; i<5; i++) {
2500                 MCHBAR32(UPMC3) &= ~(1 << 16);
2501                 MCHBAR32(UPMC3) |= (1 << 16);
2502         }
2503
2504         MCHBAR32(GIPMC1) = 0x8000000c;
2505
2506         reg16 = MCHBAR16(CPCTL);
2507         reg16 &= ~(7 << 11);
2508         if (i945_silicon_revision()>2) {
2509                 reg16 |= (6 << 11);
2510         } else {
2511                 reg16 |= (4 << 11);
2512         }
2513         MCHBAR16(CPCTL) = reg16;
2514
2515 #if 0
2516         if ((MCHBAR32(ECO) & (1 << 16)) != 0) {
2517 #else
2518         if (i945_silicon_revision() != 0) {
2519 #endif
2520                 switch (sysinfo->fsb_frequency) {
2521                 case 667: MCHBAR32(HGIPMC2) = 0x0d590d59; break;
2522                 case 533: MCHBAR32(HGIPMC2) = 0x155b155b; break;
2523                 }
2524         } else {
2525                 switch (sysinfo->fsb_frequency) {
2526                 case 667: MCHBAR32(HGIPMC2) = 0x09c409c4; break;
2527                 case 533: MCHBAR32(HGIPMC2) = 0x0fa00fa0; break;
2528                 }
2529         }
2530
2531         MCHBAR32(FSBPMC1) = 0x8000000c;
2532
2533         reg32 = MCHBAR32(C2C3TT);
2534         reg32 &= 0xffff0000;
2535         switch (sysinfo->fsb_frequency) {
2536         case 667: reg32 |= 0x0600; break;
2537         case 533: reg32 |= 0x0480; break;
2538         }
2539         MCHBAR32(C2C3TT) = reg32;
2540
2541         reg32 = MCHBAR32(C3C4TT);
2542         reg32 &= 0xffff0000;
2543         switch (sysinfo->fsb_frequency) {
2544         case 667: reg32 |= 0x0b80; break;
2545         case 533: reg32 |= 0x0980; break;
2546         }
2547         MCHBAR32(C3C4TT) = reg32;
2548
2549         if (i945_silicon_revision() == 0) {
2550                 MCHBAR32(ECO) &= ~(1 << 16);
2551         } else {
2552                 MCHBAR32(ECO) |= (1 << 16);
2553         }
2554
2555 #if 0
2556
2557         if (i945_silicon_revision() == 0) {
2558                 MCHBAR32(FSBPMC3) &= ~(1 << 29);
2559         } else {
2560                 MCHBAR32(FSBPMC3) |= (1 << 29);
2561         }
2562 #endif
2563         MCHBAR32(FSBPMC3) &= ~(1 << 29);
2564
2565         MCHBAR32(FSBPMC3) |= (1 << 21);
2566
2567         MCHBAR32(FSBPMC3) &= ~(1 << 19);
2568
2569         MCHBAR32(FSBPMC3) &= ~(1 << 13);
2570
2571         reg32 = MCHBAR32(FSBPMC4);
2572         reg32 &= ~(3 << 24);
2573         reg32 |= ( 2 << 24);
2574         MCHBAR32(FSBPMC4) = reg32;
2575
2576         MCHBAR32(FSBPMC4) |= (1 << 21);
2577
2578         MCHBAR32(FSBPMC4) |= (1 << 5);
2579
2580         if ((i945_silicon_revision() < 2) /* || cpuid() = 0x6e8 */ ) {
2581                 /* stepping 0 and 1 or CPUID 6e8 */
2582                 MCHBAR32(FSBPMC4) &= ~(1 << 4);
2583         } else {
2584                 MCHBAR32(FSBPMC4) |= (1 << 4);
2585         }
2586
2587         reg8 = pci_read_config8(PCI_DEV(0,0x0,0), 0xfc);
2588         reg8 |= (1 << 4);
2589         pci_write_config8(PCI_DEV(0, 0x0, 0), 0xfc, reg8);
2590
2591         reg8 = pci_read_config8(PCI_DEV(0,0x2,0), 0xc1);
2592         reg8 |= (1 << 2);
2593         pci_write_config8(PCI_DEV(0, 0x2, 0), 0xc1, reg8);
2594
2595 #ifdef C2_SELF_REFRESH_DISABLE
2596
2597         if (integrated_graphics) {
2598                 printk(BIOS_DEBUG, "C2 self-refresh with IGD\n");
2599                 MCHBAR16(MIPMC4) = 0x0468;
2600                 MCHBAR16(MIPMC5) = 0x046c;
2601                 MCHBAR16(MIPMC6) = 0x046c;
2602         } else {
2603                 MCHBAR16(MIPMC4) = 0x6468;
2604                 MCHBAR16(MIPMC5) = 0x646c;
2605                 MCHBAR16(MIPMC6) = 0x646c;
2606         }
2607 #else
2608         if (integrated_graphics) {
2609                 MCHBAR16(MIPMC4) = 0x04f8;
2610                 MCHBAR16(MIPMC5) = 0x04fc;
2611                 MCHBAR16(MIPMC6) = 0x04fc;
2612         } else {
2613                 MCHBAR16(MIPMC4) = 0x64f8;
2614                 MCHBAR16(MIPMC5) = 0x64fc;
2615                 MCHBAR16(MIPMC6) = 0x64fc;
2616         }
2617
2618 #endif
2619
2620         reg32 = MCHBAR32(PMCFG);
2621         reg32 &= ~(3 << 17);
2622         reg32 |= (2 << 17);
2623         MCHBAR32(PMCFG) = reg32;
2624
2625         MCHBAR32(PMCFG) |= (1 << 4);
2626
2627         reg32 = MCHBAR32(0xc30);
2628         reg32 &= 0xffffff00;
2629         reg32 |= 0x01;
2630         MCHBAR32(0xc30) = reg32;
2631
2632         MCHBAR32(0xb18) &= ~(1 << 21);
2633 }
2634
2635 static void sdram_thermal_management(void)
2636 {
2637
2638         MCHBAR8(TCO1) = 0x00;
2639         MCHBAR8(TCO0) = 0x00;
2640
2641         /* The Thermal Sensors for DIMMs at 0x50, 0x52 are at I2C addr
2642          * 0x30/0x32.
2643          */
2644
2645         /* TODO This is not implemented yet. Volunteers? */
2646 }
2647
2648 static void sdram_save_receive_enable(void)
2649 {
2650         int i;
2651         u32 reg32;
2652         u8 values[4];
2653
2654         /* The following values are stored to an unused CMOS
2655          * area and restored instead of recalculated in case
2656          * of an S3 resume.
2657          *
2658          * C0WL0REOST [7:0]             -> 8 bit
2659          * C1WL0REOST [7:0]             -> 8 bit
2660          * RCVENMT    [11:8] [3:0]      -> 8 bit
2661          * C0DRT1     [27:24]           -> 4 bit
2662          * C1DRT1     [27:24]           -> 4 bit
2663          */
2664
2665         values[0] = MCHBAR8(C0WL0REOST);
2666         values[1] = MCHBAR8(C1WL0REOST);
2667
2668         reg32 = MCHBAR32(RCVENMT);
2669         values[2] = (u8)((reg32 >> (8 - 4)) & 0xf0) | (reg32 & 0x0f);
2670
2671         reg32 = MCHBAR32(C0DRT1);
2672         values[3] = (reg32 >> 24) & 0x0f;
2673         reg32 = MCHBAR32(C1DRT1);
2674         values[3] |= (reg32 >> (24 - 4)) & 0xf0;
2675
2676         /* coreboot only uses bytes 0 - 127 for its CMOS values so far
2677          * so we grab bytes 128 - 131 to save the receive enable values
2678          */
2679
2680         for (i=0; i<4; i++)
2681                 cmos_write(values[i], 128 + i);
2682 }
2683
2684 static void sdram_recover_receive_enable(void)
2685 {
2686         int i;
2687         u32 reg32;
2688         u8 values[4];
2689
2690         for (i=0; i<4; i++)
2691                 values[i] = cmos_read(128 + i);
2692
2693         MCHBAR8(C0WL0REOST) = values[0];
2694         MCHBAR8(C1WL0REOST) = values[1];
2695
2696         reg32 = MCHBAR32(RCVENMT);
2697         reg32 &= ~((0x0f << 8) | (0x0f << 0));
2698         reg32 |= ((u32)(values[2] & 0xf0) << (8 - 4)) | (values[2] & 0x0f);
2699         MCHBAR32(RCVENMT) = reg32;
2700
2701         reg32 = MCHBAR32(C0DRT1) & ~(0x0f << 24);
2702         reg32 |= (u32)(values[3] & 0x0f) << 24;
2703         MCHBAR32(C0DRT1) = reg32;
2704
2705         reg32 = MCHBAR32(C1DRT1) & ~(0x0f << 24);
2706         reg32 |= (u32)(values[3] & 0xf0) << (24 - 4);
2707         MCHBAR32(C1DRT1) = reg32;
2708 }
2709
2710 #include "rcven.c"
2711
2712 static void sdram_program_receive_enable(struct sys_info *sysinfo)
2713 {
2714         MCHBAR32(REPC) |= (1 << 0);
2715
2716         /* enable upper CMOS */
2717         RCBA32(0x3400) = (1 << 2);
2718
2719         /* Program Receive Enable Timings */
2720         if (sysinfo->boot_path == BOOT_PATH_RESUME) {
2721                 sdram_recover_receive_enable();
2722         } else {
2723                 receive_enable_adjust(sysinfo);
2724                 sdram_save_receive_enable();
2725         }
2726
2727         MCHBAR32(C0DRC1) |= (1 << 6);
2728         MCHBAR32(C1DRC1) |= (1 << 6);
2729         MCHBAR32(C0DRC1) &= ~(1 << 6);
2730         MCHBAR32(C1DRC1) &= ~(1 << 6);
2731
2732         MCHBAR32(MIPMC3) |= (0x0f << 0);
2733 }
2734
2735 /**
2736  * @brief Enable On-Die Termination for DDR2.
2737  *
2738  */
2739
2740 static void sdram_on_die_termination(struct sys_info *sysinfo)
2741 {
2742         static const u32 odt[] = {
2743                 0x00024911, 0xe0010000,
2744                 0x00049211, 0xe0020000,
2745                 0x0006db11, 0xe0030000,
2746         };
2747
2748         u32 reg32;
2749         int cas;
2750
2751         reg32 = MCHBAR32(ODTC);
2752         reg32 &= ~(3 << 16);
2753         reg32 |= (1 << 14) | (1 << 6) | (2 << 16);
2754         MCHBAR32(ODTC) = reg32;
2755
2756         if ( !(sysinfo->dimm[0] != SYSINFO_DIMM_NOT_POPULATED &&
2757                         sysinfo->dimm[1] != SYSINFO_DIMM_NOT_POPULATED) ) {
2758                 printk(BIOS_DEBUG, "one dimm per channel config.. \n");
2759
2760                 reg32 = MCHBAR32(C0ODT);
2761                 reg32 &= ~(7 << 28);
2762                 MCHBAR32(C0ODT) = reg32;
2763                 reg32 = MCHBAR32(C1ODT);
2764                 reg32 &= ~(7 << 28);
2765                 MCHBAR32(C1ODT) = reg32;
2766         }
2767
2768         cas = sysinfo->cas;
2769
2770         reg32 = MCHBAR32(C0ODT) & 0xfff00000;
2771         reg32 |= odt[(cas-3) * 2];
2772         MCHBAR32(C0ODT) = reg32;
2773
2774         reg32 = MCHBAR32(C1ODT) & 0xfff00000;
2775         reg32 |= odt[(cas-3) * 2];
2776         MCHBAR32(C1ODT) = reg32;
2777
2778         reg32 = MCHBAR32(C0ODT + 4) & 0x1fc8ffff;
2779         reg32 |= odt[((cas-3) * 2) + 1];
2780         MCHBAR32(C0ODT + 4) = reg32;
2781
2782         reg32 = MCHBAR32(C1ODT + 4) & 0x1fc8ffff;
2783         reg32 |= odt[((cas-3) * 2) + 1];
2784         MCHBAR32(C1ODT + 4) = reg32;
2785 }
2786
2787 /**
2788  * @brief Enable clocks to populated sockets
2789  */
2790
2791 static void sdram_enable_memory_clocks(struct sys_info *sysinfo)
2792 {
2793         u8 clocks[2] = { 0, 0 };
2794
2795 #if CONFIG_I945GM
2796 #define CLOCKS_WIDTH 2
2797 #endif
2798 #if CONFIG_I945GC
2799 #define CLOCKS_WIDTH 3
2800 #endif
2801         if (sysinfo->dimm[0] != SYSINFO_DIMM_NOT_POPULATED)
2802                 clocks[0] |= (1 << CLOCKS_WIDTH)-1;
2803
2804         if (sysinfo->dimm[1] != SYSINFO_DIMM_NOT_POPULATED)
2805                 clocks[0] |= ((1 << CLOCKS_WIDTH)-1) << CLOCKS_WIDTH;
2806
2807         if (sysinfo->dimm[2] != SYSINFO_DIMM_NOT_POPULATED)
2808                 clocks[1] |= (1 << CLOCKS_WIDTH)-1;
2809
2810         if (sysinfo->dimm[3] != SYSINFO_DIMM_NOT_POPULATED)
2811                 clocks[1] |= ((1 << CLOCKS_WIDTH)-1) << CLOCKS_WIDTH;
2812
2813 #if CONFIG_OVERRIDE_CLOCK_DISABLE
2814         /* Usually system firmware turns off system memory clock signals
2815          * to unused SO-DIMM slots to reduce EMI and power consumption.
2816          * However, the Kontron 986LCD-M does not like unused clock
2817          * signals to be disabled.
2818          */
2819
2820         clocks[0] = 0xf; /* force all clock gate pairs to enable */
2821         clocks[1] = 0xf; /* force all clock gate pairs to enable */
2822 #endif
2823
2824         MCHBAR8(C0DCLKDIS) = clocks[0];
2825         MCHBAR8(C1DCLKDIS) = clocks[1];
2826 }
2827
2828 #define RTT_ODT_NONE    0
2829 #define RTT_ODT_50_OHM  ( (1 << 9) | (1 << 5) )
2830 #define RTT_ODT_75_OHM  (1 << 5)
2831 #define RTT_ODT_150_OHM (1 << 9)
2832
2833 #define EMRS_OCD_DEFAULT        ( (1 << 12) | (1 << 11) | (1 << 10) )
2834
2835 #define MRS_CAS_3       (3 << 7)
2836 #define MRS_CAS_4       (4 << 7)
2837 #define MRS_CAS_5       (5 << 7)
2838
2839 #define MRS_TWR_3       (2 << 12)
2840 #define MRS_TWR_4       (3 << 12)
2841 #define MRS_TWR_5       (4 << 12)
2842
2843 #define MRS_BT          (1 << 6)
2844
2845 #define MRS_BL4         (2 << 3)
2846 #define MRS_BL8         (3 << 3)
2847
2848 static void sdram_jedec_enable(struct sys_info *sysinfo)
2849 {
2850         int i, nonzero;
2851         u32 bankaddr = 0, tmpaddr, mrsaddr = 0;
2852
2853         for (i = 0, nonzero = -1; i < 8; i++) {
2854                 if (sysinfo->banksize[i]  == 0) {
2855                         continue;
2856                 }
2857
2858                 printk(BIOS_DEBUG, "jedec enable sequence: bank %d\n", i);
2859                 switch (i) {
2860                 case 0:
2861                         /* Start at address 0 */
2862                         bankaddr = 0;
2863                         break;
2864                 case 4:
2865                         if (sysinfo->interleaved) {
2866                                 bankaddr = 0x40;
2867                                 break;
2868                         }
2869                 default:
2870                         if (nonzero != -1) {
2871                                 printk(BIOS_DEBUG, "bankaddr from bank size of rank %d\n", nonzero);
2872                                 bankaddr += sysinfo->banksize[nonzero] <<
2873                                         (sysinfo->interleaved ? 26 : 25);
2874                                 break;
2875                         }
2876                         /* No populated bank hit before. Start at address 0 */
2877                         bankaddr = 0;
2878                 }
2879
2880                 /* We have a bank with a non-zero size.. Remember it
2881                  * for the next offset we have to calculate
2882                  */
2883                 nonzero = i;
2884
2885                 /* Get CAS latency set up */
2886                 switch (sysinfo->cas) {
2887                 case 5: mrsaddr = MRS_CAS_5; break;
2888                 case 4: mrsaddr = MRS_CAS_4; break;
2889                 case 3: mrsaddr = MRS_CAS_3; break;
2890                 default: die("Jedec Error (CAS).\n");
2891                 }
2892
2893                 /* Get tWR set */
2894                 switch (sysinfo->twr) {
2895                 case 5: mrsaddr |= MRS_TWR_5; break;
2896                 case 4: mrsaddr |= MRS_TWR_4; break;
2897                 case 3: mrsaddr |= MRS_TWR_3; break;
2898                 default: die("Jedec Error (tWR).\n");
2899                 }
2900
2901                 /* Set "Burst Type" */
2902                 mrsaddr |= MRS_BT;
2903
2904                 /* Interleaved */
2905                 if (sysinfo->interleaved) {
2906                         mrsaddr = mrsaddr << 1;
2907                 }
2908
2909                 /* Only burst length 8 supported */
2910                 mrsaddr |= MRS_BL8;
2911
2912                 /* Apply NOP */
2913                 PRINTK_DEBUG("Apply NOP\n");
2914                 do_ram_command(RAM_COMMAND_NOP);
2915                 ram_read32(bankaddr);
2916
2917                 /* Precharge all banks */
2918                 PRINTK_DEBUG("All Banks Precharge\n");
2919                 do_ram_command(RAM_COMMAND_PRECHARGE);
2920                 ram_read32(bankaddr);
2921
2922                 /* Extended Mode Register Set (2) */
2923                 PRINTK_DEBUG("Extended Mode Register Set(2)\n");
2924                 do_ram_command(RAM_COMMAND_EMRS | RAM_EMRS_2);
2925                 ram_read32(bankaddr);
2926
2927                 /* Extended Mode Register Set (3) */
2928                 PRINTK_DEBUG("Extended Mode Register Set(3)\n");
2929                 do_ram_command(RAM_COMMAND_EMRS | RAM_EMRS_3);
2930                 ram_read32(bankaddr);
2931
2932                 /* Extended Mode Register Set */
2933                 PRINTK_DEBUG("Extended Mode Register Set\n");
2934                 do_ram_command(RAM_COMMAND_EMRS | RAM_EMRS_1);
2935                 tmpaddr = bankaddr;
2936                 if (!sdram_capabilities_dual_channel()) {
2937                         tmpaddr |= RTT_ODT_75_OHM;
2938                 } else if (sysinfo->interleaved) {
2939                         tmpaddr |= (RTT_ODT_150_OHM << 1);
2940                 } else {
2941                         tmpaddr |= RTT_ODT_150_OHM;
2942                 }
2943                 ram_read32(tmpaddr);
2944
2945                 /* Mode Register Set: Reset DLLs */
2946                 PRINTK_DEBUG("MRS: Reset DLLs\n");
2947                 do_ram_command(RAM_COMMAND_MRS);
2948                 tmpaddr = bankaddr;
2949                 tmpaddr |= mrsaddr;
2950                 /* Set DLL reset bit */
2951                 if (sysinfo->interleaved)
2952                         tmpaddr |= (1 << 12);
2953                 else
2954                         tmpaddr |= (1 << 11);
2955                 ram_read32(tmpaddr);
2956
2957                 /* Precharge all banks */
2958                 PRINTK_DEBUG("All Banks Precharge\n");
2959                 do_ram_command(RAM_COMMAND_PRECHARGE);
2960                 ram_read32(bankaddr);
2961
2962                 /* CAS before RAS Refresh */
2963                 PRINTK_DEBUG("CAS before RAS\n");
2964                 do_ram_command(RAM_COMMAND_CBR);
2965
2966                 /* CBR wants two READs */
2967                 ram_read32(bankaddr);
2968                 ram_read32(bankaddr);
2969
2970                 /* Mode Register Set: Enable DLLs */
2971                 PRINTK_DEBUG("MRS: Enable DLLs\n");
2972                 do_ram_command(RAM_COMMAND_MRS);
2973
2974                 tmpaddr = bankaddr;
2975                 tmpaddr |= mrsaddr;
2976                 ram_read32(tmpaddr);
2977
2978                 /* Extended Mode Register Set */
2979                 PRINTK_DEBUG("Extended Mode Register Set: ODT/OCD\n");
2980                 do_ram_command(RAM_COMMAND_EMRS | RAM_EMRS_1);
2981
2982                 tmpaddr = bankaddr;
2983                 if (!sdram_capabilities_dual_channel()) {
2984
2985                         tmpaddr |= RTT_ODT_75_OHM | EMRS_OCD_DEFAULT;
2986                 } else if (sysinfo->interleaved) {
2987                         tmpaddr |= ((RTT_ODT_150_OHM | EMRS_OCD_DEFAULT) << 1);
2988                 } else {
2989                         tmpaddr |= RTT_ODT_150_OHM | EMRS_OCD_DEFAULT;
2990                 }
2991                 ram_read32(tmpaddr);
2992
2993                 /* Extended Mode Register Set */
2994                 PRINTK_DEBUG("Extended Mode Register Set: OCD Exit\n");
2995                 do_ram_command(RAM_COMMAND_EMRS | RAM_EMRS_1);
2996
2997                 tmpaddr = bankaddr;
2998                 if (!sdram_capabilities_dual_channel()) {
2999                         tmpaddr |= RTT_ODT_75_OHM;
3000                 } else if (sysinfo->interleaved) {
3001                         tmpaddr |= (RTT_ODT_150_OHM << 1);
3002                 } else {
3003                         tmpaddr |= RTT_ODT_150_OHM;
3004                 }
3005                 ram_read32(tmpaddr);
3006         }
3007 }
3008
3009 static void sdram_init_complete(void)
3010 {
3011         PRINTK_DEBUG("Normal Operation\n");
3012         do_ram_command(RAM_COMMAND_NORMAL);
3013 }
3014
3015 static void sdram_setup_processor_side(void)
3016 {
3017         if (i945_silicon_revision() == 0)
3018                 MCHBAR32(FSBPMC3) |= (1 << 2);
3019
3020         MCHBAR8(0xb00) |= 1;
3021
3022         if (i945_silicon_revision() == 0)
3023                 MCHBAR32(SLPCTL) |= (1 << 8);
3024 }
3025
3026 /**
3027  * @param boot_path: 0 = normal, 1 = reset, 2 = resume from s3
3028  */
3029 void sdram_initialize(int boot_path)
3030 {
3031         struct sys_info sysinfo;
3032         u8 reg8, cas_mask;
3033
3034         printk(BIOS_DEBUG, "Setting up RAM controller.\n");
3035
3036         memset(&sysinfo, 0, sizeof(sysinfo));
3037
3038         sysinfo.boot_path = boot_path;
3039
3040         /* Look at the type of DIMMs and verify all DIMMs are x8 or x16 width */
3041         sdram_get_dram_configuration(&sysinfo);
3042
3043         /* If error, do cold boot */
3044         sdram_detect_errors(&sysinfo);
3045
3046         /* Check whether we have stacked DIMMs */
3047         sdram_verify_package_type(&sysinfo);
3048
3049         /* Determine common CAS */
3050         cas_mask = sdram_possible_cas_latencies(&sysinfo);
3051
3052         /* Choose Common Frequency */
3053         sdram_detect_cas_latency_and_ram_speed(&sysinfo, cas_mask);
3054
3055         /* Determine smallest common tRAS */
3056         sdram_detect_smallest_tRAS(&sysinfo);
3057
3058         /* Determine tRP */
3059         sdram_detect_smallest_tRP(&sysinfo);
3060
3061         /* Determine tRCD */
3062         sdram_detect_smallest_tRCD(&sysinfo);
3063
3064         /* Determine smallest refresh period */
3065         sdram_detect_smallest_refresh(&sysinfo);
3066
3067         /* Verify all DIMMs support burst length 8 */
3068         sdram_verify_burst_length(&sysinfo);
3069
3070         /* determine tWR */
3071         sdram_detect_smallest_tWR(&sysinfo);
3072
3073         /* Determine DIMM size parameters (rows, columns banks) */
3074         sdram_detect_dimm_size(&sysinfo);
3075
3076         /* determine tRFC */
3077         sdram_detect_smallest_tRFC(&sysinfo);
3078
3079         /* Program PLL settings */
3080         sdram_program_pll_settings(&sysinfo);
3081
3082         /* Program Graphics Frequency */
3083         sdram_program_graphics_frequency(&sysinfo);
3084
3085         /* Program System Memory Frequency */
3086         sdram_program_memory_frequency(&sysinfo);
3087
3088         /* Determine Mode of Operation (Interleaved etc) */
3089         sdram_set_channel_mode(&sysinfo);
3090
3091         /* Program Clock Crossing values */
3092         sdram_program_clock_crossing();
3093
3094         /* Disable fast dispatch */
3095         sdram_disable_fast_dispatch();
3096
3097         /* Enable WIODLL Power Down in ACPI states */
3098         MCHBAR32(C0DMC) |= (1 << 24);
3099         MCHBAR32(C1DMC) |= (1 << 24);
3100
3101         /* Program DRAM Row Boundary/Attribute Registers */
3102
3103         /* program row size DRB and set TOLUD */
3104         sdram_program_row_boundaries(&sysinfo);
3105
3106         /* program page size DRA */
3107         sdram_set_row_attributes(&sysinfo);
3108
3109         /* Program CxBNKARC */
3110         sdram_set_bank_architecture(&sysinfo);
3111
3112         /* Program DRAM Timing and Control registers based on SPD */
3113         sdram_set_timing_and_control(&sysinfo);
3114
3115         /* On-Die Termination Adjustment */
3116         sdram_on_die_termination(&sysinfo);
3117
3118         /* Pre Jedec Initialization */
3119         sdram_pre_jedec_initialization();
3120
3121         /* Perform System Memory IO Initialization */
3122         sdram_initialize_system_memory_io(&sysinfo);
3123
3124         /* Perform System Memory IO Buffer Enable */
3125         sdram_enable_system_memory_io(&sysinfo);
3126
3127         /* Enable System Memory Clocks */
3128         sdram_enable_memory_clocks(&sysinfo);
3129
3130         if (boot_path == BOOT_PATH_NORMAL) {
3131                 /* Jedec Initialization sequence */
3132                 sdram_jedec_enable(&sysinfo);
3133         }
3134
3135         /* Program Power Management Registers */
3136         sdram_power_management(&sysinfo);
3137
3138         /* Post Jedec Init */
3139         sdram_post_jedec_initialization(&sysinfo);
3140
3141         /* Program DRAM Throttling */
3142         sdram_thermal_management();
3143
3144         /* Normal Operations */
3145         sdram_init_complete();
3146
3147         /* Program Receive Enable Timings */
3148         sdram_program_receive_enable(&sysinfo);
3149
3150         /* Enable Periodic RCOMP */
3151         sdram_enable_rcomp();
3152
3153         /* Tell ICH7 that we're done */
3154         reg8 = pci_read_config8(PCI_DEV(0,0x1f,0), 0xa2);
3155         reg8 &= ~(1 << 7);
3156         pci_write_config8(PCI_DEV(0, 0x1f, 0), 0xa2, reg8);
3157
3158         printk(BIOS_DEBUG, "RAM initialization finished.\n");
3159
3160         sdram_setup_processor_side();
3161 }
3162
3163 unsigned long get_top_of_ram(void)
3164 {
3165         /* This will not work if TSEG is in place! */
3166         u32 tom = pci_read_config32(PCI_DEV(0,2,0), 0x5c);
3167
3168         return (unsigned long) tom;
3169 }
3170