1689 lines
40 KiB
C
Executable File
1689 lines
40 KiB
C
Executable File
/**
|
|
* SPRD ep device driver in host side for Spreadtrum SoCs
|
|
*
|
|
* Copyright (C) 2018 Spreadtrum Co., Ltd.
|
|
*
|
|
* This program is free software: you can redistribute it and/or modify
|
|
* it under the terms of the GNU General Public License version 2 of
|
|
* the License as published by the Free Software Foundation.
|
|
*
|
|
* This program is used to control ep device driver in host side for
|
|
* Spreadtrum SoCs.
|
|
*/
|
|
|
|
#include <linux/delay.h>
|
|
#include <linux/debugfs.h>
|
|
#include <linux/fs.h>
|
|
#include <linux/interrupt.h>
|
|
#include <linux/irq.h>
|
|
#include <linux/module.h>
|
|
#include <linux/spinlock.h>
|
|
#include <linux/bitops.h>
|
|
#include <linux/pci.h>
|
|
#include <linux/pci_regs.h>
|
|
#include <linux/pci_ids.h>
|
|
#include <linux/sched.h>
|
|
#include <linux/types.h>
|
|
#include <linux/workqueue.h>
|
|
#include <linux/version.h>
|
|
|
|
#include <linux/iommu.h>
|
|
#include <linux/iova.h>
|
|
#include <linux/bitops.h>
|
|
|
|
#include "../include/pcie-rc-sprd.h"
|
|
#include "../include/sprd_pcie_ep_device.h"
|
|
#include "../include/sipc_big_to_little.h"
|
|
#include "../include/sprd_pcie_resource.h"
|
|
#include "../include/sipc.h"
|
|
#include "../include/sipa.h"
|
|
#include "../sipa/sipa_core.h"
|
|
|
|
#define VERSION_NUMBER "V1.1.8"
|
|
#define QUECTEL_SPRD_PCIE_VERSION "Quectel_Linux&Android_SPRD_PCIE_Driver_"VERSION_NUMBER
|
|
|
|
#define DRV_MODULE_NAME "sprd-pcie-ep-device"
|
|
#define CONFIG_SPRD_IPA_PCIE_WORKROUND
|
|
|
|
enum dev_pci_barno {
|
|
BAR_0 = 0,
|
|
BAR_1,
|
|
BAR_2,
|
|
BAR_3,
|
|
BAR_4,
|
|
BAR_5,
|
|
BAR_CNT
|
|
};
|
|
|
|
#define MAX_SUPPORT_IRQ 32
|
|
#define IPA_HW_IRQ_CNT 4
|
|
#define IPA_HW_IRQ_BASE 16
|
|
#define IPA_HW_IRQ_BASE_DEFECT 0
|
|
|
|
#define REQUEST_BASE_IRQ (IPA_HW_IRQ_BASE + IPA_HW_IRQ_CNT)
|
|
#define REQUEST_BASE_IRQ_DEFECT 16
|
|
|
|
#ifdef CONFIG_SPRD_IPA_PCIE_WORKROUND
|
|
/* the bar0 and the bar1 are used for ipa */
|
|
#define IPA_MEM_BAR BAR_0
|
|
#define IPA_REG_BAR BAR_1
|
|
#define IPA_TFT_BAR BAR_3
|
|
#define BAR_MIN BAR_2
|
|
#else
|
|
#define BAR_MIN BAR_0
|
|
#endif
|
|
|
|
/* the bar4 and the bar5 are specail bars */
|
|
#define BAR_MAX BAR_4
|
|
|
|
#define PCI_VENDOR_ID_SPRD 0x16c3
|
|
#define PCI_DEVICE_ID_SPRD_ORCA 0xabcd
|
|
#define PCI_CLASS_ID_SPRD_ORCA 0x80d00
|
|
|
|
/* Parameters for the waiting for iATU enabled routine */
|
|
#define LINK_WAIT_MAX_IATU_RETRIES 5
|
|
#define LINK_WAIT_IATU_MIN 9000
|
|
#define LINK_WAIT_IATU_MAX 10000
|
|
|
|
/* ep config bar bar4 , can config ep iatu reg and door bell reg */
|
|
#define EP_CFG_BAR BAR_4
|
|
#define DOOR_BELL_BASE 0x00000
|
|
#define IATU_REG_BASE 0x10000
|
|
|
|
#define DOOR_BELL_ENABLE 0x10
|
|
#define DOOR_BELL_STATUS 0x14
|
|
|
|
/* used 0x18 & 0x1c to save the smem base & size. */
|
|
#define DOOR_BELL_SMEMBASE 0x18
|
|
#define DOOR_BELL_SMEMSIZE 0x1C
|
|
|
|
/* one bit can indicate one irq , if stauts[i] & enable[i] , irq = i */
|
|
#define DOOR_BELL_IRQ_VALUE(irq) BIT((irq))
|
|
#define DOOR_BELL_IRQ_CNT 32
|
|
#define IATU_MAX_REGION 8
|
|
#define PCIE_ATU_VIEWPORT 0x900
|
|
#define PCIE_ATU_CR1 0x904
|
|
#define PCIE_ATU_CR2 0x908
|
|
#define PCIE_ATU_LOWER_BASE 0x90c
|
|
#define PCIE_ATU_UPPER_BASE 0x910
|
|
#define PCIE_ATU_LIMIT 0x914
|
|
#define PCIE_ATU_LOWER_TARGET 0x918
|
|
#define PCIE_ATU_UPPER_TARGET 0x91c
|
|
|
|
#define PCIE_ATU_REGION_INBOUND BIT(31)
|
|
#define PCIE_ATU_ENABLE BIT(31)
|
|
#define PCIE_ATU_BAR_MODE_ENABLE BIT(30)
|
|
#define PCIE_ATU_TYPE_MEM 0x0
|
|
|
|
#define PCIE_ATU_UNR_REGION_CTRL1 0x00
|
|
#define PCIE_ATU_UNR_REGION_CTRL2 0x04
|
|
#define PCIE_ATU_UNR_LOWER_BASE 0x08
|
|
#define PCIE_ATU_UNR_UPPER_BASE 0x0c
|
|
#define PCIE_ATU_UNR_LIMIT 0x10
|
|
#define PCIE_ATU_UNR_LOWER_TARGET 0x14
|
|
#define PCIE_ATU_UNR_UPPER_TARGET 0x18
|
|
|
|
/* bar4 + 0x10000 has map to ep base + 0x18000 ((0x3 << 15)) */
|
|
#define PCIE_ATU_IB_REGION(region) (((region) << 9) | (0x1 << 8))
|
|
#define PCIE_ATU_OB_REGION(region) ((region) << 9)
|
|
|
|
#define PCIE_SAVE_REGION_NUM (IATU_MAX_REGION * 2)
|
|
#define PCIE_SAVE_REG_NUM 8
|
|
|
|
#define PCIE_LEGACY_CLEAR_BASE 0x29000000
|
|
#define PCIE_LEGACY_CLEAR_REG 0x2c
|
|
#define PCIE_LEGACY_CLEAR_MASK BIT(0)
|
|
#define BIT_SET_OFFSET 0x1000
|
|
#define BIT_CLR_OFFSET 0x2000
|
|
|
|
struct sprd_ep_dev_notify {
|
|
void (*notify)(int event, void *data);
|
|
void *data;
|
|
};
|
|
|
|
struct sprd_pci_ep_dev {
|
|
struct pci_dev *pdev;
|
|
void __iomem *cfg_base; /* ep config bar base in rc */
|
|
spinlock_t irq_lock; /* irq spinlock */
|
|
spinlock_t bar_lock; /* bar spinlock */
|
|
spinlock_t set_irq_lock; /* set irq spinlock */
|
|
spinlock_t set_bar_lock; /* set bar spinlock */
|
|
unsigned long bar_res;
|
|
|
|
u32 base_irq;
|
|
u32 ipa_base_irq;
|
|
u32 bak_irq_status;
|
|
u8 iatu_unroll_enabled;
|
|
u8 ep;
|
|
u8 irq_cnt;
|
|
bool no_msi;
|
|
bool need_backup;
|
|
void __iomem *legacy_addr;
|
|
|
|
struct resource *bar[BAR_CNT];
|
|
void __iomem *bar_vir[BAR_MAX];
|
|
void __iomem *cpu_vir[BAR_MAX];
|
|
dma_addr_t src_addr[BAR_MAX];
|
|
dma_addr_t target_addr[BAR_MAX];
|
|
size_t map_size[BAR_MAX];
|
|
int event;
|
|
struct work_struct notify_work;
|
|
};
|
|
|
|
struct sprd_pci_ep_dev_save {
|
|
bool save_succ;
|
|
unsigned long bar_res;
|
|
void __iomem *bar_vir[BAR_MAX];
|
|
void __iomem *cpu_vir[BAR_MAX];
|
|
dma_addr_t src_addr[BAR_MAX];
|
|
dma_addr_t target_addr[BAR_MAX];
|
|
size_t map_size[BAR_MAX];
|
|
u32 doorbell_enable;
|
|
u32 doorbell_status;
|
|
void __iomem *cfg_base;
|
|
u32 save_reg[PCIE_SAVE_REGION_NUM][PCIE_SAVE_REG_NUM];
|
|
};
|
|
|
|
static void __iomem *g_irq_addr[PCIE_EP_NR];
|
|
static struct sprd_pci_ep_dev_save g_ep_save[PCIE_EP_NR];
|
|
static struct sprd_pci_ep_dev *g_ep_dev[PCIE_EP_NR];
|
|
static irq_handler_t ep_dev_handler[PCIE_EP_NR][PCIE_MSI_MAX_IRQ];
|
|
static void *ep_dev_handler_data[PCIE_EP_NR][PCIE_MSI_MAX_IRQ];
|
|
static struct sprd_ep_dev_notify g_ep_dev_notify[PCIE_EP_NR];
|
|
|
|
static int sprd_ep_dev_get_bar(int ep);
|
|
static int sprd_ep_dev_put_bar(int ep, int bar);
|
|
static int sprd_ep_dev_adjust_region(struct sprd_pci_ep_dev *ep_dev,
|
|
int bar, dma_addr_t *cpu_addr_ptr,
|
|
size_t *size_ptr, dma_addr_t *offset_ptr);
|
|
static int sprd_ep_dev_just_map_bar(struct sprd_pci_ep_dev *ep_dev, int bar,
|
|
dma_addr_t cpu_addr, size_t size);
|
|
static int sprd_ep_dev_just_unmap_bar(struct sprd_pci_ep_dev *ep_dev, int bar);
|
|
static void __iomem *sprd_ep_dev_map_bar(int ep, int bar,
|
|
dma_addr_t cpu_addr,
|
|
size_t size);
|
|
static int sprd_ep_dev_unmap_bar(int ep, int bar);
|
|
static void sprd_pci_ep_dev_backup(struct sprd_pci_ep_dev *ep_dev);
|
|
|
|
/* //Quectel Remove
|
|
static void sprd_pcie_iommu_init(struct device *dev)
|
|
{
|
|
struct iommu_domain *domain = NULL;
|
|
unsigned long pg_size;
|
|
dma_addr_t start, end, addr;
|
|
u32 base, size;
|
|
int ret = 0;
|
|
|
|
domain = iommu_get_domain_for_dev(dev);
|
|
if(!domain) {
|
|
dev_info(dev, "sprd_pcie_iommu_init domian null");
|
|
return;
|
|
}
|
|
|
|
pg_size = 1UL << __ffs(domain->pgsize_bitmap);
|
|
smem_get_area(SIPC_ID_MINIAP, &base, &size);
|
|
start = ALIGN(base, pg_size);
|
|
end = ALIGN(base + size, pg_size);
|
|
|
|
for (addr = start; addr < end; addr += pg_size) {
|
|
phys_addr_t phys_addr;
|
|
|
|
phys_addr = iommu_iova_to_phys(domain, addr);
|
|
if (phys_addr) {
|
|
dev_info(dev, "sprd_pcie_iommu_init iova:%d have been used", (u32)addr);
|
|
continue;
|
|
}
|
|
ret = iommu_map(domain, addr, addr, pg_size, IOMMU_READ | IOMMU_WRITE);
|
|
if (ret) {
|
|
dev_info(dev, "sprd_pcie_iommu_init iommu_map failed");
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
*/
|
|
|
|
|
|
#if (LINUX_VERSION_CODE < KERNEL_VERSION( 3,11,0 ))
|
|
int pci_enable_msi_range(struct pci_dev *dev, int minvec, int maxvec)
|
|
{
|
|
int nvec = maxvec;
|
|
int rc;
|
|
|
|
if (maxvec < minvec)
|
|
return -ERANGE;
|
|
|
|
do {
|
|
rc = pci_enable_msi_block(dev, nvec);
|
|
if (rc < 0) {
|
|
return rc;
|
|
} else if (rc > 0) {
|
|
if (rc < minvec)
|
|
return -ENOSPC;
|
|
nvec = rc;
|
|
}
|
|
} while (rc);
|
|
|
|
return nvec;
|
|
}
|
|
#endif
|
|
|
|
int sprd_ep_dev_register_notify(int ep,
|
|
void (*notify)(int event, void *data),
|
|
void *data)
|
|
{
|
|
struct sprd_ep_dev_notify *dev_notify;
|
|
|
|
if (ep >= PCIE_EP_NR)
|
|
return -EINVAL;
|
|
|
|
dev_notify = &g_ep_dev_notify[ep];
|
|
dev_notify->notify = notify;
|
|
dev_notify->data = data;
|
|
|
|
return 0;
|
|
}
|
|
EXPORT_SYMBOL_GPL(sprd_ep_dev_register_notify);
|
|
|
|
int sprd_ep_dev_unregister_notify(int ep)
|
|
{
|
|
struct sprd_ep_dev_notify *notify;
|
|
|
|
if (ep >= PCIE_EP_NR)
|
|
return -EINVAL;
|
|
|
|
notify = &g_ep_dev_notify[ep];
|
|
notify->notify = NULL;
|
|
notify->data = NULL;
|
|
|
|
return 0;
|
|
}
|
|
EXPORT_SYMBOL_GPL(sprd_ep_dev_unregister_notify);
|
|
|
|
int sprd_ep_dev_set_irq_addr(int ep, void __iomem *irq_addr)
|
|
{
|
|
if (ep >= PCIE_EP_NR)
|
|
return -EINVAL;
|
|
|
|
g_irq_addr[ep] = irq_addr;
|
|
|
|
return 0;
|
|
}
|
|
EXPORT_SYMBOL_GPL(sprd_ep_dev_set_irq_addr);
|
|
|
|
int sprd_ep_dev_register_irq_handler(int ep, int irq,
|
|
irq_handler_t handler, void *data)
|
|
{
|
|
struct sprd_pci_ep_dev *ep_dev;
|
|
|
|
if (ep >= PCIE_EP_NR || irq >= PCIE_MSI_MAX_IRQ)
|
|
return -EINVAL;
|
|
|
|
ep_dev_handler[ep][irq] = handler;
|
|
ep_dev_handler_data[ep][irq] = data;
|
|
ep_dev = g_ep_dev[ep];
|
|
|
|
if (handler && ep_dev &&
|
|
(BIT(irq) & ep_dev->bak_irq_status)) {
|
|
ep_dev->bak_irq_status &= ~BIT(irq);
|
|
handler(irq, data);
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
EXPORT_SYMBOL_GPL(sprd_ep_dev_register_irq_handler);
|
|
|
|
int sprd_ep_dev_unregister_irq_handler(int ep, int irq)
|
|
{
|
|
if (ep < PCIE_EP_NR && irq < PCIE_MSI_MAX_IRQ) {
|
|
ep_dev_handler[ep][irq] = NULL;
|
|
ep_dev_handler_data[ep][irq] = NULL;
|
|
return 0;
|
|
}
|
|
|
|
return -EINVAL;
|
|
}
|
|
EXPORT_SYMBOL_GPL(sprd_ep_dev_unregister_irq_handler);
|
|
|
|
int sprd_ep_dev_register_irq_handler_ex(int ep,
|
|
int from_irq,
|
|
int to_irq,
|
|
irq_handler_t handler,
|
|
void *data)
|
|
{
|
|
int i, ret;
|
|
|
|
for (i = from_irq; i < to_irq + 1; i++) {
|
|
ret = sprd_ep_dev_register_irq_handler(ep,
|
|
i, handler, data);
|
|
if (ret)
|
|
return ret;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
int sprd_ep_dev_unregister_irq_handler_ex(int ep,
|
|
int from_irq,
|
|
int to_irq)
|
|
{
|
|
int i, ret;
|
|
|
|
for (i = from_irq; i < to_irq + 1; i++) {
|
|
ret = sprd_ep_dev_unregister_irq_handler(ep, i);
|
|
if (ret)
|
|
return ret;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
void __iomem *sprd_ep_map_memory(int ep,
|
|
phys_addr_t cpu_addr,
|
|
size_t size)
|
|
{
|
|
int bar;
|
|
void __iomem *bar_addr;
|
|
|
|
bar = sprd_ep_dev_get_bar(ep);
|
|
if (bar < 0) {
|
|
pr_err("%s: get bar err = %d\n", __func__, bar);
|
|
return NULL;
|
|
}else{
|
|
pr_err("%s: get bar = %d\n", __func__, bar);
|
|
}
|
|
|
|
bar_addr = sprd_ep_dev_map_bar(ep, bar, cpu_addr, size);
|
|
if (!bar_addr) {
|
|
pr_err("%s: map bar = %d err!\n", __func__, bar);
|
|
sprd_ep_dev_put_bar(ep, bar);
|
|
return NULL;
|
|
}
|
|
|
|
return bar_addr;
|
|
}
|
|
EXPORT_SYMBOL_GPL(sprd_ep_map_memory);
|
|
|
|
void sprd_ep_unmap_memory(int ep, const void __iomem *bar_addr)
|
|
{
|
|
int bar;
|
|
struct sprd_pci_ep_dev *ep_dev;
|
|
|
|
if (ep >= PCIE_EP_NR || !g_ep_dev[ep])
|
|
return;
|
|
|
|
ep_dev = g_ep_dev[ep];
|
|
|
|
for (bar = 0; bar < BAR_MAX; bar++) {
|
|
if (bar_addr == ep_dev->cpu_vir[bar]) {
|
|
sprd_ep_dev_unmap_bar(ep, bar);
|
|
sprd_ep_dev_put_bar(ep, bar);
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
EXPORT_SYMBOL_GPL(sprd_ep_unmap_memory);
|
|
|
|
#ifdef CONFIG_SPRD_SIPA
|
|
phys_addr_t sprd_ep_ipa_map(int type, phys_addr_t target_addr, size_t size)
|
|
{
|
|
int bar, ep = PCIE_EP_MODEM;
|
|
dma_addr_t offset;
|
|
struct sprd_pci_ep_dev *ep_dev;
|
|
struct pci_dev *pdev;
|
|
struct device *dev;
|
|
struct resource *res;
|
|
|
|
ep_dev = g_ep_dev[ep];
|
|
if (!ep_dev)
|
|
return 0;
|
|
|
|
ep_dev = g_ep_dev[ep];
|
|
pdev = ep_dev->pdev;
|
|
dev = &pdev->dev;
|
|
#ifdef CONFIG_SPRD_IPA_PCIE_WORKROUND
|
|
if(type == PCIE_IPA_TYPE_REG){
|
|
bar = IPA_REG_BAR;
|
|
}else if(type == PCIE_IPA_TYPE_TFT_REG){
|
|
bar = IPA_TFT_BAR;
|
|
}else{
|
|
bar = IPA_MEM_BAR;
|
|
}
|
|
pr_info("%s: get bar = %d\n", __func__, bar);
|
|
#else
|
|
bar = sprd_ep_dev_get_bar(ep);
|
|
if (bar < 0) {
|
|
dev_err(dev, "ep: ipa map, get bar err = %d\n", bar);
|
|
return 0;
|
|
}
|
|
#endif
|
|
res = &pdev->resource[bar];
|
|
|
|
dev_info(dev, "ep: ipa map type=%d, addr=0x%lx, size=0x%lx\n",
|
|
type,
|
|
(unsigned long)target_addr,
|
|
(unsigned long)size);
|
|
|
|
/* 1st, adjust the map region */
|
|
if (sprd_ep_dev_adjust_region(ep_dev, bar, &target_addr,
|
|
&size, &offset))
|
|
return 0;
|
|
|
|
/* than, map bar */
|
|
if (sprd_ep_dev_just_map_bar(ep_dev, bar, target_addr, size))
|
|
return 0;
|
|
|
|
/* save for unmap */
|
|
ep_dev->src_addr[bar] = res->start + offset;
|
|
ep_dev->target_addr[bar] = target_addr;
|
|
ep_dev->map_size[bar] = size;
|
|
|
|
/* return the cpu phy address */
|
|
return res->start + offset;
|
|
}
|
|
|
|
int sprd_ep_ipa_unmap(int type, const phys_addr_t cpu_addr)
|
|
{
|
|
int bar, ep = PCIE_EP_MODEM;
|
|
bool find_bar = false;
|
|
struct sprd_pci_ep_dev *ep_dev;
|
|
struct pci_dev *pdev;
|
|
struct resource *res;
|
|
|
|
ep_dev = g_ep_dev[ep];
|
|
if (!ep_dev)
|
|
return -EINVAL;
|
|
|
|
pdev = ep_dev->pdev;
|
|
res = &pdev->resource[bar];
|
|
|
|
dev_info(&pdev->dev, "ep: ipa unmap cpu_addr=0x%lx\n",
|
|
(unsigned long)cpu_addr);
|
|
|
|
#ifdef CONFIG_SPRD_IPA_PCIE_WORKROUND
|
|
if(type == PCIE_IPA_TYPE_REG){
|
|
bar = IPA_REG_BAR;
|
|
}else if(type == PCIE_IPA_TYPE_TFT_REG){
|
|
bar = IPA_TFT_BAR;
|
|
}else{
|
|
bar = IPA_MEM_BAR;
|
|
}
|
|
|
|
if (ep_dev->src_addr[bar] == cpu_addr)
|
|
find_bar = true;
|
|
#else
|
|
for (bar = 0; bar < BAR_MAX; bar++) {
|
|
if (cpu_addr == ep_dev->src_addr[bar]) {
|
|
find_bar = true;
|
|
break;
|
|
}
|
|
}
|
|
#endif
|
|
|
|
if (!find_bar) {
|
|
dev_err(&pdev->dev, "ep: ipa unmap can't find bar!");
|
|
return -EINVAL;
|
|
}
|
|
|
|
ep_dev->target_addr[bar] = 0;
|
|
ep_dev->target_addr[bar] = 0;
|
|
ep_dev->map_size[bar] = 0;
|
|
return sprd_ep_dev_just_unmap_bar(ep_dev, bar);
|
|
}
|
|
#endif
|
|
|
|
int sprd_ep_dev_raise_irq(int ep, int irq)
|
|
{
|
|
struct pci_dev *pdev;
|
|
struct device *dev;
|
|
struct sprd_pci_ep_dev *ep_dev;
|
|
void __iomem *base;
|
|
u32 value;
|
|
|
|
if (ep >= PCIE_EP_NR || !g_ep_dev[ep])
|
|
return -ENODEV;
|
|
|
|
ep_dev = g_ep_dev[ep];
|
|
pdev = ep_dev->pdev;
|
|
dev = &pdev->dev;
|
|
|
|
dev_dbg(dev, "ep: raise, ep=%d, irq=%d\n", ep, irq);
|
|
|
|
if (irq >= DOOR_BELL_IRQ_CNT) {
|
|
dev_err(&pdev->dev, "raise err, irq=%d\n", irq);
|
|
return -EINVAL;
|
|
}
|
|
|
|
spin_lock(&ep_dev->set_irq_lock);
|
|
base = ep_dev->cfg_base + DOOR_BELL_BASE;
|
|
value = readl_relaxed(base + DOOR_BELL_STATUS);
|
|
writel_relaxed(value | DOOR_BELL_IRQ_VALUE(irq),
|
|
base + DOOR_BELL_STATUS);
|
|
spin_unlock(&ep_dev->set_irq_lock);
|
|
|
|
return 0;
|
|
}
|
|
EXPORT_SYMBOL_GPL(sprd_ep_dev_raise_irq);
|
|
|
|
int sprd_ep_dev_clear_doolbell_irq(int ep, int irq)
|
|
{
|
|
struct pci_dev *pdev;
|
|
struct device *dev;
|
|
struct sprd_pci_ep_dev *ep_dev;
|
|
void __iomem *base;
|
|
u32 value;
|
|
|
|
if (ep >= PCIE_EP_NR || !g_ep_dev[ep])
|
|
return -ENODEV;
|
|
|
|
ep_dev = g_ep_dev[ep];
|
|
pdev = ep_dev->pdev;
|
|
dev = &pdev->dev;
|
|
|
|
dev_dbg(dev, "ep: clear doorbell, ep=%d, irq=%d\n", ep, irq);
|
|
|
|
if (irq >= DOOR_BELL_IRQ_CNT)
|
|
return -EINVAL;
|
|
|
|
spin_lock(&ep_dev->set_irq_lock);
|
|
base = ep_dev->cfg_base + DOOR_BELL_BASE;
|
|
value = readl_relaxed(base + DOOR_BELL_STATUS);
|
|
if (value & DOOR_BELL_IRQ_VALUE(irq))
|
|
writel_relaxed(value & (~DOOR_BELL_IRQ_VALUE(irq)),
|
|
base + DOOR_BELL_STATUS);
|
|
spin_unlock(&ep_dev->set_irq_lock);
|
|
|
|
return 0;
|
|
}
|
|
|
|
int sprd_ep_dev_set_backup(int ep)
|
|
{
|
|
struct sprd_pci_ep_dev *ep_dev;
|
|
|
|
if (ep >= PCIE_EP_NR || !g_ep_dev[ep])
|
|
return -ENODEV;
|
|
|
|
ep_dev = g_ep_dev[ep];
|
|
ep_dev->need_backup = true;
|
|
|
|
/* backup once immediately. */
|
|
sprd_pci_ep_dev_backup(ep_dev);
|
|
|
|
return 0;
|
|
}
|
|
|
|
int sprd_ep_dev_clear_backup(int ep)
|
|
{
|
|
struct sprd_pci_ep_dev *ep_dev;
|
|
|
|
if (ep >= PCIE_EP_NR || !g_ep_dev[ep])
|
|
return -ENODEV;
|
|
|
|
ep_dev = g_ep_dev[ep];
|
|
ep_dev->need_backup = false;
|
|
|
|
return 0;
|
|
}
|
|
|
|
int sprd_ep_dev_pass_smem(int ep, u32 base, u32 size)
|
|
{
|
|
struct sprd_pci_ep_dev *ep_dev;
|
|
void __iomem *reg_base;
|
|
struct pci_dev *pdev;
|
|
struct device *dev;
|
|
|
|
if (ep >= PCIE_EP_NR || !g_ep_dev[ep])
|
|
return -ENODEV;
|
|
|
|
ep_dev = g_ep_dev[ep];
|
|
pdev = ep_dev->pdev;
|
|
dev = &pdev->dev;
|
|
dev_info(&pdev->dev,
|
|
"pass_smem, base=0x%x,size=0x%x\n",
|
|
base, size);
|
|
|
|
reg_base = ep_dev->cfg_base + DOOR_BELL_BASE;
|
|
|
|
writel_relaxed(base, reg_base + DOOR_BELL_SMEMBASE);
|
|
writel_relaxed(size, reg_base + DOOR_BELL_SMEMSIZE);
|
|
|
|
return 0;
|
|
}
|
|
|
|
static inline u32 sprd_pci_ep_iatu_readl(struct sprd_pci_ep_dev *ep_dev,
|
|
u32 offset)
|
|
{
|
|
return readl_relaxed(ep_dev->cfg_base + IATU_REG_BASE + offset);
|
|
}
|
|
|
|
static inline void sprd_pci_ep_iatu_writel(struct sprd_pci_ep_dev *ep_dev,
|
|
u32 offset, u32 value)
|
|
{
|
|
writel_relaxed(value, ep_dev->cfg_base + IATU_REG_BASE + offset);
|
|
}
|
|
|
|
static int sprd_ep_dev_get_bar(int ep)
|
|
{
|
|
int bar;
|
|
int ret = -EBUSY;
|
|
struct sprd_pci_ep_dev *ep_dev;
|
|
|
|
if (ep >= PCIE_EP_NR || !g_ep_dev[ep])
|
|
return -ENODEV;
|
|
|
|
ep_dev = g_ep_dev[ep];
|
|
spin_lock(&ep_dev->bar_lock);
|
|
for (bar = BAR_MIN; bar < BAR_MAX; bar++) {
|
|
if (ep_dev->bar[bar] && !test_bit(bar, &ep_dev->bar_res)) {
|
|
set_bit(bar, &ep_dev->bar_res);
|
|
ret = bar;
|
|
break;
|
|
}
|
|
}
|
|
spin_unlock(&ep_dev->bar_lock);
|
|
|
|
return ret;
|
|
}
|
|
|
|
static int sprd_ep_dev_put_bar(int ep, int bar)
|
|
{
|
|
int ret = -ENODEV;
|
|
struct sprd_pci_ep_dev *ep_dev;
|
|
|
|
if (ep >= PCIE_EP_NR || !g_ep_dev[ep])
|
|
return -ENODEV;
|
|
|
|
ep_dev = g_ep_dev[ep];
|
|
spin_lock(&ep_dev->bar_lock);
|
|
if (test_and_clear_bit(bar, &ep_dev->bar_res))
|
|
ret = bar;
|
|
spin_unlock(&ep_dev->bar_lock);
|
|
|
|
return ret;
|
|
}
|
|
|
|
static int sprd_ep_dev_unr_set_bar(struct sprd_pci_ep_dev *ep_dev,
|
|
int bar,
|
|
dma_addr_t cpu_addr, size_t size)
|
|
{
|
|
u32 retries, val;
|
|
struct pci_dev *pdev = ep_dev->pdev;
|
|
|
|
spin_lock(&ep_dev->set_bar_lock);
|
|
|
|
/* bar n use region n to map, map to bar match mode */
|
|
sprd_pci_ep_iatu_writel(ep_dev,
|
|
PCIE_ATU_IB_REGION(bar) +
|
|
PCIE_ATU_UNR_LOWER_TARGET,
|
|
lower_32_bits(cpu_addr));
|
|
sprd_pci_ep_iatu_writel(ep_dev,
|
|
PCIE_ATU_IB_REGION(bar) +
|
|
PCIE_ATU_UNR_UPPER_TARGET,
|
|
upper_32_bits(cpu_addr));
|
|
|
|
sprd_pci_ep_iatu_writel(ep_dev,
|
|
PCIE_ATU_IB_REGION(bar) +
|
|
PCIE_ATU_UNR_REGION_CTRL1,
|
|
PCIE_ATU_TYPE_MEM);
|
|
sprd_pci_ep_iatu_writel(ep_dev,
|
|
PCIE_ATU_IB_REGION(bar) +
|
|
PCIE_ATU_UNR_REGION_CTRL2,
|
|
PCIE_ATU_ENABLE |
|
|
PCIE_ATU_BAR_MODE_ENABLE | (bar << 8));
|
|
|
|
spin_unlock(&ep_dev->set_bar_lock);
|
|
|
|
/*
|
|
* Make sure ATU enable takes effect before any subsequent config
|
|
* and I/O accesses.
|
|
*/
|
|
for (retries = 0; retries < LINK_WAIT_MAX_IATU_RETRIES; retries++) {
|
|
val = sprd_pci_ep_iatu_readl(ep_dev,
|
|
PCIE_ATU_IB_REGION(bar) +
|
|
PCIE_ATU_UNR_REGION_CTRL2);
|
|
if (val & PCIE_ATU_ENABLE)
|
|
return 0;
|
|
|
|
dev_dbg(&pdev->dev,
|
|
"ep: unr set bar[%d], var = 0x%x\n",
|
|
bar,
|
|
val);
|
|
/* wait a moment for polling ep atu enable bit */
|
|
usleep_range(LINK_WAIT_IATU_MIN, LINK_WAIT_IATU_MAX);
|
|
}
|
|
|
|
return -EINVAL;
|
|
}
|
|
|
|
static int sprd_ep_dev_unr_clear_bar(struct sprd_pci_ep_dev *ep_dev, int bar)
|
|
{
|
|
struct pci_dev *pdev = ep_dev->pdev;
|
|
|
|
dev_dbg(&pdev->dev, "ep: unr clear map bar=%d\n", bar);
|
|
|
|
spin_lock(&ep_dev->set_bar_lock);
|
|
|
|
sprd_pci_ep_iatu_writel(ep_dev,
|
|
PCIE_ATU_IB_REGION(bar) +
|
|
PCIE_ATU_UNR_REGION_CTRL2,
|
|
(u32)(~PCIE_ATU_ENABLE));
|
|
spin_unlock(&ep_dev->set_bar_lock);
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int sprd_ep_dev_adjust_region(struct sprd_pci_ep_dev *ep_dev, int bar,
|
|
dma_addr_t *cpu_addr_ptr,
|
|
size_t *size_ptr,
|
|
dma_addr_t *offset_ptr)
|
|
{
|
|
dma_addr_t cpu_addr, base, offset;
|
|
resource_size_t bar_size, size;
|
|
struct pci_dev *pdev = ep_dev->pdev;
|
|
struct resource *res = &pdev->resource[bar];
|
|
|
|
size = (resource_size_t)*size_ptr;
|
|
cpu_addr = *cpu_addr_ptr;
|
|
bar_size = resource_size(res);
|
|
|
|
/* size must align with page */
|
|
size = PAGE_ALIGN(size);
|
|
|
|
/* base must be divisible by bar size for bar match mode */
|
|
base = cpu_addr / bar_size * bar_size;
|
|
offset = cpu_addr - base;
|
|
size += PAGE_ALIGN(offset);
|
|
|
|
/* size must < bar size */
|
|
if (size > bar_size) {
|
|
dev_err(&pdev->dev,
|
|
"bar[%d]:size=0x%lx > 0x%lx\n",
|
|
bar,
|
|
(unsigned long)size,
|
|
(unsigned long)bar_size);
|
|
return -EINVAL;
|
|
}
|
|
|
|
dev_dbg(&pdev->dev,
|
|
"bar[%d]: base=0x%lx,size=0x%lx,offset=0x%lx\n",
|
|
bar, (unsigned long)base,
|
|
(unsigned long)size,
|
|
(unsigned long)offset);
|
|
|
|
*size_ptr = (size_t)size;
|
|
*offset_ptr = offset;
|
|
*cpu_addr_ptr = base;
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int sprd_ep_dev_just_map_bar(struct sprd_pci_ep_dev *ep_dev, int bar,
|
|
dma_addr_t cpu_addr, size_t size)
|
|
{
|
|
u32 retries, val;
|
|
struct pci_dev *pdev;
|
|
struct device *dev;
|
|
|
|
pdev = ep_dev->pdev;
|
|
dev = &pdev->dev;
|
|
|
|
dev_dbg(dev, "ep: map bar=%d, addr=0x%lx, size=0x%lx\n",
|
|
bar,
|
|
(unsigned long)cpu_addr,
|
|
(unsigned long)size);
|
|
|
|
if (ep_dev->iatu_unroll_enabled)
|
|
return sprd_ep_dev_unr_set_bar(ep_dev, bar, cpu_addr, size);
|
|
|
|
spin_lock(&ep_dev->set_bar_lock);
|
|
|
|
/* bar n use region n to map, map to bar match mode */
|
|
sprd_pci_ep_iatu_writel(ep_dev,
|
|
PCIE_ATU_VIEWPORT,
|
|
PCIE_ATU_REGION_INBOUND | bar);
|
|
sprd_pci_ep_iatu_writel(ep_dev, PCIE_ATU_LOWER_TARGET,
|
|
lower_32_bits(cpu_addr));
|
|
sprd_pci_ep_iatu_writel(ep_dev, PCIE_ATU_UPPER_TARGET,
|
|
upper_32_bits(cpu_addr));
|
|
sprd_pci_ep_iatu_writel(ep_dev,
|
|
PCIE_ATU_CR1,
|
|
PCIE_ATU_TYPE_MEM);
|
|
sprd_pci_ep_iatu_writel(ep_dev,
|
|
PCIE_ATU_CR2,
|
|
PCIE_ATU_ENABLE |
|
|
PCIE_ATU_BAR_MODE_ENABLE | (bar << 8));
|
|
|
|
spin_unlock(&ep_dev->set_bar_lock);
|
|
|
|
/*
|
|
* Make sure ATU enable takes effect
|
|
* before any subsequent config and I/O accesses.
|
|
*/
|
|
for (retries = 0;
|
|
retries < LINK_WAIT_MAX_IATU_RETRIES;
|
|
retries++) {
|
|
val = sprd_pci_ep_iatu_readl(ep_dev, PCIE_ATU_CR2);
|
|
if (val & PCIE_ATU_ENABLE)
|
|
return 0;
|
|
|
|
/* wait a moment for polling ep atu enable bit */
|
|
usleep_range(LINK_WAIT_IATU_MIN, LINK_WAIT_IATU_MAX);
|
|
}
|
|
|
|
return -EINVAL;
|
|
}
|
|
|
|
static int sprd_ep_dev_just_unmap_bar(struct sprd_pci_ep_dev *ep_dev, int bar)
|
|
{
|
|
struct pci_dev *pdev;
|
|
struct device *dev;
|
|
|
|
pdev = ep_dev->pdev;
|
|
dev = &pdev->dev;
|
|
|
|
dev_dbg(dev, "ep: unmap bar = %d\n", bar);
|
|
|
|
if (ep_dev->iatu_unroll_enabled)
|
|
return sprd_ep_dev_unr_clear_bar(ep_dev, bar);
|
|
|
|
spin_lock(&ep_dev->set_bar_lock);
|
|
|
|
sprd_pci_ep_iatu_writel(ep_dev, PCIE_ATU_VIEWPORT,
|
|
PCIE_ATU_REGION_INBOUND | bar);
|
|
sprd_pci_ep_iatu_writel(ep_dev, PCIE_ATU_CR2,
|
|
(u32)(~PCIE_ATU_ENABLE));
|
|
|
|
spin_unlock(&ep_dev->set_bar_lock);
|
|
|
|
return 0;
|
|
}
|
|
|
|
static void __iomem *sprd_ep_dev_map_bar(int ep, int bar,
|
|
dma_addr_t cpu_addr, size_t size)
|
|
{
|
|
resource_size_t offset;
|
|
struct pci_dev *pdev;
|
|
struct device *dev;
|
|
struct sprd_pci_ep_dev *ep_dev;
|
|
void __iomem *bar_vir;
|
|
struct resource *res;
|
|
|
|
if (ep >= PCIE_EP_NR || !g_ep_dev[ep])
|
|
return NULL;
|
|
|
|
ep_dev = g_ep_dev[ep];
|
|
pdev = ep_dev->pdev;
|
|
dev = &pdev->dev;
|
|
|
|
/* bar is be used */
|
|
if (ep_dev->bar_vir[bar]) {
|
|
dev_err(dev, "ep: bar[%d] is used!", bar);
|
|
return NULL;
|
|
}
|
|
|
|
/* 1st, adjust the map region */
|
|
if (sprd_ep_dev_adjust_region(ep_dev, bar, &cpu_addr, &size, &offset))
|
|
return NULL;
|
|
|
|
/* than, ioremap, if map failed, no need to set bar */
|
|
res = &pdev->resource[bar];
|
|
#ifndef ioremap_nocache
|
|
#define ioremap_nocache ioremap
|
|
#endif
|
|
bar_vir = ioremap_nocache(res->start, size);
|
|
if (!bar_vir) {
|
|
dev_err(dev, "ep: map error, bar=%d, addr=0x%lx, size=0x%lx\n",
|
|
bar,
|
|
(unsigned long)cpu_addr,
|
|
(unsigned long)size);
|
|
return NULL;
|
|
}
|
|
|
|
if (sprd_ep_dev_just_map_bar(ep_dev, bar, cpu_addr, size)) {
|
|
dev_err(dev, "ep: map bar =%d\n", bar);
|
|
iounmap(ep_dev->bar_vir[bar]);
|
|
return NULL;
|
|
}
|
|
|
|
ep_dev->bar_vir[bar] = (void __iomem *)bar_vir;
|
|
ep_dev->cpu_vir[bar] = (void __iomem *)(bar_vir + offset);
|
|
ep_dev->src_addr[bar] = res->start + offset;
|
|
ep_dev->target_addr[bar] = cpu_addr;
|
|
ep_dev->map_size[bar] = size;
|
|
|
|
return ep_dev->cpu_vir[bar];
|
|
}
|
|
|
|
static int sprd_ep_dev_unmap_bar(int ep, int bar)
|
|
{
|
|
struct pci_dev *pdev;
|
|
struct device *dev;
|
|
struct sprd_pci_ep_dev *ep_dev;
|
|
|
|
if (ep >= PCIE_EP_NR || !g_ep_dev[ep])
|
|
return -ENODEV;
|
|
|
|
ep_dev = g_ep_dev[ep];
|
|
pdev = ep_dev->pdev;
|
|
dev = &pdev->dev;
|
|
|
|
dev_info(dev, "ep: unmap bar = %d\n", bar);
|
|
|
|
if (!ep_dev->bar_vir[bar])
|
|
return -ENODEV;
|
|
|
|
sprd_ep_dev_just_unmap_bar(ep_dev, bar);
|
|
|
|
iounmap(ep_dev->bar_vir[bar]);
|
|
ep_dev->bar_vir[bar] = NULL;
|
|
ep_dev->cpu_vir[bar] = NULL;
|
|
ep_dev->src_addr[bar] = 0;
|
|
ep_dev->target_addr[bar] = 0;
|
|
ep_dev->map_size[bar] = 0;
|
|
|
|
return 0;
|
|
}
|
|
|
|
static void sprd_pci_ep_dev_clear_legacy_irq(struct sprd_pci_ep_dev *ep_dev)
|
|
{
|
|
if (!ep_dev->legacy_addr)
|
|
return;
|
|
|
|
writel_relaxed(PCIE_LEGACY_CLEAR_MASK,
|
|
ep_dev->legacy_addr + BIT_CLR_OFFSET + PCIE_LEGACY_CLEAR_REG);
|
|
}
|
|
|
|
static irqreturn_t sprd_pci_ep_dev_irqhandler(int irq, void *dev_ptr)
|
|
{
|
|
struct sprd_pci_ep_dev *ep_dev = dev_ptr;
|
|
struct pci_dev *pdev = ep_dev->pdev;
|
|
struct device *dev = &pdev->dev;
|
|
irq_handler_t handler;
|
|
u32 i, j, value;
|
|
int ipa_irq;
|
|
|
|
if (ep_dev->no_msi) {
|
|
/* clear irq */
|
|
sprd_pci_ep_dev_clear_legacy_irq(ep_dev);
|
|
|
|
/* workaroud for IPA */
|
|
handler = ep_dev_handler[ep_dev->ep][PCIE_MSI_IPA];
|
|
if (handler)
|
|
handler(irq, ep_dev_handler_data[ep_dev->ep][PCIE_MSI_IPA]);
|
|
|
|
|
|
value = BL_READL(g_irq_addr[ep_dev->ep]);
|
|
dev_dbg(dev, "ep: irq handler. irq = 0x%x, base=%d\n", value, ep_dev->base_irq);
|
|
for (i = 0; i < 32; i++) {
|
|
if (value & BIT(i)) {
|
|
/* clear iqr bit*/
|
|
value = BL_READL(g_irq_addr[ep_dev->ep]);
|
|
value &= ~(BIT(i));
|
|
BL_WRITEL(value,g_irq_addr[ep_dev->ep]);
|
|
j = i - ep_dev->base_irq;
|
|
if (j >= PCIE_MSI_MAX_IRQ)
|
|
continue;
|
|
|
|
handler = ep_dev_handler[ep_dev->ep][j];
|
|
if (handler)
|
|
handler(irq, ep_dev_handler_data[ep_dev->ep][j]);
|
|
else
|
|
ep_dev->bak_irq_status |= BIT(j);
|
|
}
|
|
}
|
|
return IRQ_HANDLED;
|
|
}
|
|
|
|
dev_dbg(dev, "ep: irq handler. irq = %d\n", irq);
|
|
/* for ipa hw irq. */
|
|
ipa_irq = irq - (pdev->irq + ep_dev->ipa_base_irq);
|
|
if (ipa_irq >= 0 && ipa_irq < IPA_HW_IRQ_CNT) {
|
|
handler = ep_dev_handler[ep_dev->ep][PCIE_MSI_IPA];
|
|
if (handler)
|
|
handler(ipa_irq, ep_dev_handler_data[ep_dev->ep][PCIE_MSI_IPA]);
|
|
else
|
|
ep_dev->bak_irq_status |= BIT(PCIE_MSI_IPA);
|
|
|
|
return IRQ_HANDLED;
|
|
}
|
|
|
|
|
|
irq -= (pdev->irq + ep_dev->base_irq);
|
|
if (irq >= PCIE_MSI_MAX_IRQ || irq < 0) {
|
|
dev_err(dev, "ep: error, irq = %d", irq);
|
|
return IRQ_HANDLED;
|
|
}
|
|
|
|
handler = ep_dev_handler[ep_dev->ep][irq];
|
|
if (handler)
|
|
handler(irq, ep_dev_handler_data[ep_dev->ep][irq]);
|
|
else
|
|
ep_dev->bak_irq_status |= BIT(irq);
|
|
|
|
return IRQ_HANDLED;
|
|
}
|
|
|
|
static void sprd_pci_ep_save_reg(struct sprd_pci_ep_dev *ep_dev)
|
|
{
|
|
int i, j;
|
|
u32 (*save_reg)[PCIE_SAVE_REG_NUM];
|
|
static struct sprd_pci_ep_dev_save *ep_save;
|
|
|
|
ep_save = &g_ep_save[ep_dev->ep];
|
|
save_reg = ep_save->save_reg;
|
|
|
|
for (i = 0; i < PCIE_SAVE_REGION_NUM; i += 2) {
|
|
for (j = 0; j < PCIE_SAVE_REG_NUM; j++) {
|
|
save_reg[i][j] =
|
|
sprd_pci_ep_iatu_readl(ep_dev,
|
|
PCIE_ATU_OB_REGION(i) +
|
|
j * sizeof(u32));
|
|
save_reg[i + 1][j] =
|
|
sprd_pci_ep_iatu_readl(ep_dev,
|
|
PCIE_ATU_IB_REGION(i) +
|
|
j * sizeof(u32));
|
|
}
|
|
}
|
|
|
|
ep_save->doorbell_enable = sprd_pci_ep_iatu_readl(ep_dev,
|
|
DOOR_BELL_BASE +
|
|
DOOR_BELL_ENABLE);
|
|
ep_save->doorbell_status = sprd_pci_ep_iatu_readl(ep_dev,
|
|
DOOR_BELL_BASE +
|
|
DOOR_BELL_STATUS);
|
|
ep_save->cfg_base = ep_dev->cfg_base;
|
|
|
|
ep_save->save_succ = true;
|
|
}
|
|
|
|
static void sprd_pci_ep_dev_backup(struct sprd_pci_ep_dev *ep_dev)
|
|
{
|
|
struct pci_dev *pdev = ep_dev->pdev;
|
|
struct device *dev = &pdev->dev;
|
|
struct sprd_pci_ep_dev_save *ep_save;
|
|
int i;
|
|
|
|
ep_save = &g_ep_save[ep_dev->ep];
|
|
|
|
/* save some member */
|
|
ep_save->bar_res = ep_dev->bar_res;
|
|
for (i = 0; i < BAR_MAX; i++) {
|
|
if (!ep_dev->src_addr[i])
|
|
continue;
|
|
|
|
dev_info(dev, "ep: backup bar=%d, addr=0x%lx, size=0x%lx\n",
|
|
i,
|
|
(unsigned long)ep_save->target_addr[i],
|
|
(unsigned long)ep_save->map_size[i]);
|
|
|
|
ep_save->bar_vir[i] = ep_dev->bar_vir[i];
|
|
ep_save->cpu_vir[i] = ep_dev->cpu_vir[i];
|
|
ep_save->src_addr[i] = ep_dev->src_addr[i];
|
|
ep_save->target_addr[i] = ep_dev->target_addr[i];
|
|
ep_save->map_size[i] = ep_dev->map_size[i];
|
|
}
|
|
|
|
/* save ep reg */
|
|
sprd_pci_ep_save_reg(ep_dev);
|
|
}
|
|
|
|
static void sprd_pci_ep_dev_restore(struct sprd_pci_ep_dev *ep_dev)
|
|
{
|
|
struct pci_dev *pdev = ep_dev->pdev;
|
|
struct device *dev = &pdev->dev;
|
|
struct sprd_pci_ep_dev_save *ep_save;
|
|
int i;
|
|
|
|
ep_save = &g_ep_save[ep_dev->ep];
|
|
|
|
/* save some member */
|
|
ep_dev->bar_res = ep_save->bar_res;
|
|
for (i = 0; i < BAR_MAX; i++) {
|
|
if (!ep_save->src_addr[i])
|
|
continue;
|
|
|
|
ep_dev->bar_vir[i] = ep_save->bar_vir[i];
|
|
ep_dev->cpu_vir[i] = ep_save->cpu_vir[i];
|
|
ep_dev->src_addr[i] = ep_save->src_addr[i];
|
|
ep_dev->target_addr[i] = ep_save->target_addr[i];
|
|
ep_dev->map_size[i] = ep_save->map_size[i];
|
|
|
|
dev_info(dev, "ep: restore bar=%d, addr=0x%lx, size=0x%lx\n",
|
|
i,
|
|
(unsigned long)ep_dev->target_addr[i],
|
|
(unsigned long)ep_dev->map_size[i]);
|
|
|
|
if (sprd_ep_dev_just_map_bar(ep_dev,
|
|
i,
|
|
ep_dev->target_addr[i],
|
|
ep_dev->map_size[i]))
|
|
dev_err(dev, "ep: restore map err i = %d.\n", i);
|
|
}
|
|
}
|
|
|
|
static void sprd_pci_ep_notify_fn(struct work_struct *work)
|
|
{
|
|
struct sprd_ep_dev_notify *notify;
|
|
struct sprd_pci_ep_dev *ep_dev = container_of(work,
|
|
struct sprd_pci_ep_dev,
|
|
notify_work);
|
|
|
|
notify = &g_ep_dev_notify[ep_dev->ep];
|
|
if (notify->notify)
|
|
notify->notify(ep_dev->event, notify->data);
|
|
}
|
|
|
|
static int sprd_pci_ep_dev_probe(struct pci_dev *pdev,
|
|
const struct pci_device_id *ent)
|
|
{
|
|
int i, err, irq_cnt = 0;
|
|
u32 val;
|
|
enum dev_pci_barno bar;
|
|
struct device *dev = &pdev->dev;
|
|
struct sprd_pci_ep_dev *ep_dev;
|
|
struct resource *res;
|
|
|
|
dev_info(dev, "ep: probe\n");
|
|
|
|
//sprd_pcie_iommu_init(dev); //Quectel Remove
|
|
|
|
ep_dev = devm_kzalloc(dev, sizeof(*ep_dev), GFP_KERNEL);
|
|
spin_lock_init(&ep_dev->set_bar_lock);
|
|
spin_lock_init(&ep_dev->bar_lock);
|
|
spin_lock_init(&ep_dev->set_irq_lock);
|
|
|
|
if (!ep_dev)
|
|
return -ENOMEM;
|
|
|
|
ep_dev->pdev = pdev;
|
|
|
|
if (ent->device == PCI_DEVICE_ID_SPRD_ORCA)
|
|
ep_dev->ep = PCIE_EP_MODEM;
|
|
else {
|
|
dev_err(dev, "ep: Cannot support ep device = 0x%x\n",
|
|
ent->device);
|
|
return -EINVAL;
|
|
}
|
|
|
|
err = pci_enable_device(pdev);
|
|
if (err) {
|
|
dev_err(dev, "ep: Cannot enable PCI device\n");
|
|
return err;
|
|
}
|
|
|
|
err = pci_request_regions(pdev, DRV_MODULE_NAME);
|
|
if (err) {
|
|
dev_err(dev, "ep: Cannot obtain PCI resources\n");
|
|
goto err_disable_pdev;
|
|
}
|
|
|
|
pci_set_master(pdev);
|
|
if(dma_set_mask(dev, DMA_BIT_MASK(32))) {
|
|
dev_warn(dev, "ep: No suitable DMA available\n");
|
|
}
|
|
|
|
#if defined(CONFIG_PCI_IRQ_MSI) && defined(CONFIG_PCI_MSI)
|
|
irq_cnt = pci_alloc_irq_vectors(pdev,
|
|
1,
|
|
MAX_SUPPORT_IRQ,
|
|
PCI_IRQ_MSI);
|
|
//irq_cnt = pci_enable_msi_range(pdev, 1, MAX_SUPPORT_IRQ);
|
|
dev_info(dev, "pci alloc msi irq cnt = %d\n", irq_cnt);
|
|
#endif
|
|
|
|
if (sprd_pcie_is_defective_chip()) {
|
|
ep_dev->base_irq = REQUEST_BASE_IRQ_DEFECT;
|
|
ep_dev->ipa_base_irq = IPA_HW_IRQ_BASE_DEFECT;
|
|
} else {
|
|
ep_dev->base_irq = REQUEST_BASE_IRQ;
|
|
ep_dev->ipa_base_irq = IPA_HW_IRQ_BASE;
|
|
}
|
|
|
|
for (bar = BAR_0; bar <= BAR_5; bar++) {
|
|
res = &pdev->resource[bar];
|
|
dev_info(dev, "ep: BAR[%d] %pR\n", bar, res);
|
|
/* only save mem bar */
|
|
if (resource_type(res) == IORESOURCE_MEM)
|
|
ep_dev->bar[bar] = res;
|
|
}
|
|
|
|
ep_dev->cfg_base = pci_ioremap_bar(pdev, EP_CFG_BAR);
|
|
if (!ep_dev->cfg_base) {
|
|
dev_err(dev, "ep: failed to map cfg bar.\n");
|
|
err = -ENOMEM;
|
|
goto err_disable_msi;
|
|
}
|
|
|
|
/* clear all 32 bit door bell */
|
|
writel_relaxed(0x0,
|
|
ep_dev->cfg_base + DOOR_BELL_BASE + DOOR_BELL_STATUS);
|
|
|
|
pci_set_drvdata(pdev, ep_dev);
|
|
pci_read_config_dword(ep_dev->pdev, PCIE_ATU_VIEWPORT, &val);
|
|
/*
|
|
* this atu view port reg is 0xffffffff means that the ep device
|
|
* doesn't support atu view port, we need unroll iatu registers
|
|
*/
|
|
dev_info(dev, "ep: atu_view_port val = 0x%x", val);
|
|
ep_dev->iatu_unroll_enabled = val == 0xffffffff;
|
|
|
|
/* default , PCIE_EP_PROBE */
|
|
ep_dev->event = PCIE_EP_PROBE;
|
|
g_ep_dev[ep_dev->ep] = ep_dev;
|
|
|
|
if (!ep_dev->bar[BAR_1] || !ep_dev->bar[BAR_3]) {
|
|
/* only 2 bar, set PCIE_EP_PROBE_BEFORE_SPLIT_BAR */
|
|
ep_dev->event = PCIE_EP_PROBE_BEFORE_SPLIT_BAR;
|
|
dev_info(dev, "ep:bar not ready, wait the next probe!");
|
|
}
|
|
|
|
/* restore all the config */
|
|
if (ep_dev->event == PCIE_EP_PROBE)
|
|
sprd_pci_ep_dev_restore(ep_dev);
|
|
|
|
/* start notify work */
|
|
INIT_WORK(&ep_dev->notify_work, sprd_pci_ep_notify_fn);
|
|
schedule_work(&ep_dev->notify_work);
|
|
|
|
if (irq_cnt < MAX_SUPPORT_IRQ) {
|
|
err = irq_cnt < 0 ? irq_cnt : -EINVAL;
|
|
ep_dev->no_msi = true;
|
|
dev_info(dev, "ep: failed to get MSI, err=%d, irq=%d\n", err, pdev->irq);
|
|
|
|
ep_dev->legacy_addr = sprd_ep_map_memory(ep_dev->ep, PCIE_LEGACY_CLEAR_BASE, 0x4000);
|
|
/* request legacy irq */
|
|
err = devm_request_irq(dev, pdev->irq, sprd_pci_ep_dev_irqhandler,
|
|
IRQF_SHARED, DRV_MODULE_NAME, ep_dev);
|
|
if (err)
|
|
dev_warn(dev,
|
|
"ep: failed to request legacy %d\n",
|
|
pdev->irq);
|
|
|
|
} else {
|
|
ep_dev->irq_cnt = irq_cnt;
|
|
dev_info(dev, "ep: request IRQ = %d, cnt =%d\n",
|
|
pdev->irq,
|
|
ep_dev->irq_cnt);
|
|
|
|
/* request msi irq */
|
|
for (i = ep_dev->base_irq;
|
|
i < ep_dev->base_irq + PCIE_MSI_MAX_IRQ;
|
|
i++) {
|
|
err = devm_request_irq(dev, pdev->irq + i,
|
|
sprd_pci_ep_dev_irqhandler,
|
|
IRQF_SHARED, DRV_MODULE_NAME, ep_dev);
|
|
if (err)
|
|
dev_warn(dev,
|
|
"ep: failed to request IRQ %d for MSI %d\n",
|
|
pdev->irq + i, i + 1);
|
|
}
|
|
|
|
#ifdef CONFIG_SPRD_IPA_PCIE_WORKROUND
|
|
for (i = ep_dev->ipa_base_irq;
|
|
i < ep_dev->ipa_base_irq + IPA_HW_IRQ_CNT;
|
|
i++) {
|
|
err = devm_request_irq(dev, pdev->irq + i,
|
|
sprd_pci_ep_dev_irqhandler,
|
|
IRQF_SHARED, DRV_MODULE_NAME,
|
|
ep_dev);
|
|
if (!err)
|
|
sprd_pcie_teardown_msi_irq(pdev->irq + i);
|
|
}
|
|
#endif
|
|
}
|
|
|
|
#ifndef SPRD_PCIE_USE_DTS
|
|
sipa_module_init(dev);
|
|
sipa_eth_init();
|
|
sipa_dummy_init();
|
|
#endif
|
|
|
|
return 0;
|
|
|
|
err_disable_msi:
|
|
pci_disable_msi(pdev);
|
|
pci_release_regions(pdev);
|
|
|
|
err_disable_pdev:
|
|
pci_disable_device(pdev);
|
|
|
|
return err;
|
|
}
|
|
|
|
static void sprd_pci_ep_dev_remove(struct pci_dev *pdev)
|
|
{
|
|
u32 i;
|
|
struct sprd_ep_dev_notify *notify;
|
|
struct sprd_pci_ep_dev *ep_dev = pci_get_drvdata(pdev);
|
|
struct sipa_core *ctrl = sipa_get_ctrl_pointer();
|
|
dev_info(&pdev->dev, "ep: remove\n");
|
|
ctrl->remote_ready = false;
|
|
spipe_device_down();
|
|
spool_device_down();
|
|
|
|
cancel_work_sync(&ep_dev->notify_work);
|
|
|
|
/* first notify PCIE_EP_REMOVE */
|
|
notify = &g_ep_dev_notify[ep_dev->ep];
|
|
if (notify->notify)
|
|
notify->notify(PCIE_EP_REMOVE, notify->data);
|
|
|
|
/* back up some config before remove */
|
|
if (ep_dev->need_backup)
|
|
sprd_pci_ep_dev_backup(ep_dev);
|
|
|
|
if (ep_dev->no_msi) {
|
|
devm_free_irq(&pdev->dev, pdev->irq, ep_dev);
|
|
} else {
|
|
for (i = ep_dev->base_irq; i < ep_dev->base_irq + PCIE_MSI_MAX_IRQ; i++)
|
|
devm_free_irq(&pdev->dev, pdev->irq + i, ep_dev);
|
|
|
|
#ifdef CONFIG_SPRD_IPA_PCIE_WORKROUND
|
|
for (i = ep_dev->ipa_base_irq;
|
|
i < ep_dev->ipa_base_irq + IPA_HW_IRQ_CNT;
|
|
i++)
|
|
devm_free_irq(&pdev->dev, pdev->irq + i, ep_dev);
|
|
#endif
|
|
}
|
|
|
|
pci_disable_msi(pdev);
|
|
|
|
#ifndef SPRD_PCIE_USE_DTS
|
|
sipa_dummy_exit();
|
|
sipa_eth_exit();
|
|
sipa_module_exit();
|
|
#endif
|
|
|
|
if (ep_dev->legacy_addr) {
|
|
sprd_ep_unmap_memory(ep_dev->ep, ep_dev->legacy_addr);
|
|
ep_dev->legacy_addr = NULL;
|
|
dev_info(&ep_dev->pdev->dev, "set ep_dev->legacy_addr = %lx\n", (long)ep_dev->legacy_addr);
|
|
}
|
|
|
|
if (ep_dev->cfg_base) {
|
|
iounmap(ep_dev->cfg_base);
|
|
ep_dev->cfg_base = NULL;
|
|
}
|
|
|
|
pci_release_regions(pdev);
|
|
pci_disable_device(pdev);
|
|
|
|
g_ep_dev[ep_dev->ep] = NULL;
|
|
ep_dev->bar_res = 0;
|
|
}
|
|
|
|
static const struct pci_device_id sprd_pci_ep_dev_tbl[] = {
|
|
{ PCI_DEVICE(PCI_VENDOR_ID_SPRD, PCI_DEVICE_ID_SPRD_ORCA) },
|
|
{ }
|
|
};
|
|
MODULE_DEVICE_TABLE(pci, sprd_pci_ep_dev_tbl);
|
|
|
|
#ifdef CONFIG_PM_SLEEP
|
|
static int sprd_pci_ep_suspend(struct device *dev)
|
|
{
|
|
struct pci_dev *pdev = to_pci_dev(dev);
|
|
int rc;
|
|
|
|
dev_info(dev, "suspend\n");
|
|
|
|
/* Exec pci PCI_D3cold one time */
|
|
if (pdev->current_state != PCI_D0) {
|
|
dev_info(dev, "done for pm %d\n", pdev->current_state);
|
|
return 0;
|
|
}
|
|
|
|
/*
|
|
* TODO: The HAL will ask the shared memory layer whether D3 is allowed.
|
|
*/
|
|
|
|
/* Save the PCI configuration space of a device before suspending. */
|
|
rc = pci_save_state(pdev);
|
|
if (rc) {
|
|
dev_err(dev, "pci_save_state error=%d\n", rc);
|
|
return rc;
|
|
}
|
|
|
|
/* Set the power state of a PCI device.
|
|
* Transition a device to a new power state, using the device's PCI PM
|
|
* registers.
|
|
*/
|
|
rc = pci_set_power_state(pdev, PCI_D3cold);
|
|
if (rc) {
|
|
dev_err(dev, "pci_set_power_state error=%d\n", rc);
|
|
return rc;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
static int sprd_pci_ep_resume(struct device *dev)
|
|
{
|
|
struct pci_dev *pdev = to_pci_dev(dev);
|
|
int rc;
|
|
|
|
dev_info(dev, "resume\n");
|
|
|
|
/* Set the power state of a PCI device. */
|
|
rc = pci_set_power_state(pdev, PCI_D0);
|
|
if (rc) {
|
|
dev_err(dev, "pci_set_power_state error=%d\n", rc);
|
|
return rc;
|
|
}
|
|
|
|
/* Restore the saved state of a PCI device. */
|
|
pci_restore_state(pdev);
|
|
|
|
/* TODO: The HAL shall inform that the device is active. */
|
|
return 0;
|
|
}
|
|
#endif /* CONFIG_PM_SLEEP */
|
|
|
|
static const struct dev_pm_ops sprd_pci_ep_pm = {
|
|
SET_SYSTEM_SLEEP_PM_OPS(sprd_pci_ep_suspend,
|
|
sprd_pci_ep_resume)
|
|
};
|
|
|
|
static struct pci_driver sprd_pci_ep_dev_driver = {
|
|
.name = DRV_MODULE_NAME,
|
|
.id_table = sprd_pci_ep_dev_tbl,
|
|
.probe = sprd_pci_ep_dev_probe,
|
|
.remove = sprd_pci_ep_dev_remove,
|
|
.driver = {
|
|
.pm = &sprd_pci_ep_pm,
|
|
}
|
|
};
|
|
//module_pci_driver(sprd_pci_ep_dev_driver);
|
|
|
|
#if defined(CONFIG_DEBUG_FS)
|
|
static void sprd_pci_ep_dev_save_show(struct seq_file *m,
|
|
struct sprd_pci_ep_dev_save *ep_save,
|
|
int ep)
|
|
{
|
|
u32 i;
|
|
|
|
seq_printf(m, "ep-save-%d configs:\n", ep);
|
|
seq_printf(m, "bar_res = 0x%lx\n", ep_save->bar_res);
|
|
|
|
for (i = 0; i < BAR_MAX; i++) {
|
|
seq_printf(m, "src_addr[%d] = 0x%lx\n",
|
|
i,
|
|
(unsigned long)ep_save->src_addr[i]);
|
|
seq_printf(m, "target_addr[%d] = 0x%lx\n",
|
|
i,
|
|
(unsigned long)ep_save->target_addr[i]);
|
|
seq_printf(m, "map_size[%d] = 0x%lx\n",
|
|
i,
|
|
(unsigned long)ep_save->map_size[i]);
|
|
}
|
|
}
|
|
|
|
static void sprd_pci_ep_dev_config_show(struct seq_file *m,
|
|
struct sprd_pci_ep_dev *ep_dev)
|
|
{
|
|
u32 i;
|
|
void __iomem *base;
|
|
|
|
seq_printf(m, "ep-%d configs:\n", ep_dev->ep);
|
|
|
|
/* doorbell regs */
|
|
seq_puts(m, "door bell regs:\n");
|
|
base = ep_dev->cfg_base + DOOR_BELL_BASE;
|
|
|
|
seq_printf(m, "irq_enable = 0x%08x\n irq_status = 0x%08x\n",
|
|
readl_relaxed(base + DOOR_BELL_ENABLE),
|
|
readl_relaxed(base + DOOR_BELL_STATUS));
|
|
|
|
/* iatu reg regs */
|
|
seq_puts(m, "iatu regs reg:\n");
|
|
for (i = 0; i < IATU_MAX_REGION * 2; i++) {
|
|
base = ep_dev->cfg_base + IATU_REG_BASE + i * 100;
|
|
seq_printf(m, "IATU[%d]:\n", i);
|
|
seq_printf(m, "0x%p: 0x%08x 0x%08x 0x%08x 0x%08x\n",
|
|
base,
|
|
readl_relaxed(base + 0x0),
|
|
readl_relaxed(base + 0x4),
|
|
readl_relaxed(base + 0x8),
|
|
readl_relaxed(base + 0xc));
|
|
base += 0x10;
|
|
seq_printf(m, "0x%p: 0x%08x 0x%08x 0x%08x 0x%08x\n",
|
|
base,
|
|
readl_relaxed(base + 0x0),
|
|
readl_relaxed(base + 0x4),
|
|
readl_relaxed(base + 0x8),
|
|
readl_relaxed(base + 0x10));
|
|
}
|
|
}
|
|
|
|
static void sprd_pci_ep_dev_backup_show(struct seq_file *m,
|
|
struct sprd_pci_ep_dev_save *ep_save,
|
|
int ep)
|
|
{
|
|
int i;
|
|
u32 (*save_reg)[PCIE_SAVE_REG_NUM];
|
|
void __iomem *base;
|
|
|
|
save_reg = ep_save->save_reg;
|
|
|
|
seq_printf(m, "ep-%d backup configs:\n", ep);
|
|
|
|
/* doorbell regs */
|
|
seq_puts(m, "door bell regs:\n");
|
|
seq_printf(m, "irq_enable = 0x%08x\n irq_status = 0x%08x\n",
|
|
ep_save->doorbell_enable,
|
|
ep_save->doorbell_status);
|
|
|
|
/* iatu reg regs */
|
|
seq_puts(m, "iatu regs reg:\n");
|
|
for (i = 0; i < PCIE_SAVE_REGION_NUM; i++) {
|
|
seq_printf(m, "IATU[%d]:\n", i);
|
|
base = ep_save->cfg_base + IATU_REG_BASE + i * 100;
|
|
|
|
seq_printf(m, "0x%p: 0x%08x 0x%08x 0x%08x 0x%08x\n",
|
|
base,
|
|
save_reg[i][0],
|
|
save_reg[i][1],
|
|
save_reg[i][2],
|
|
save_reg[i][3]);
|
|
base += 0x10;
|
|
seq_printf(m, "0x%p: 0x%08x 0x%08x 0x%08x 0x%08x\n",
|
|
base,
|
|
save_reg[i][4],
|
|
save_reg[i][5],
|
|
save_reg[i][6],
|
|
save_reg[i][7]);
|
|
}
|
|
}
|
|
|
|
static int sprd_pci_ep_dev_show(struct seq_file *m, void *unused)
|
|
{
|
|
u32 i;
|
|
struct sprd_pci_ep_dev *ep_dev;
|
|
struct sprd_pci_ep_dev_save *ep_save;
|
|
|
|
for (i = 0; i < PCIE_EP_NR; i++) {
|
|
/* ep_save configus */
|
|
ep_save = &g_ep_save[i];
|
|
ep_dev = g_ep_dev[i];
|
|
|
|
if (!ep_dev && !ep_save->save_succ)
|
|
continue;
|
|
|
|
if (ep_save)
|
|
sprd_pci_ep_dev_save_show(m, ep_save, i);
|
|
|
|
if (ep_dev)
|
|
sprd_pci_ep_dev_config_show(m, ep_dev);
|
|
else
|
|
sprd_pci_ep_dev_backup_show(m, ep_save, i);
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int sprd_pci_ep_dev_open(struct inode *inode, struct file *file)
|
|
{
|
|
return single_open(file, sprd_pci_ep_dev_show, NULL);
|
|
}
|
|
|
|
static const struct file_operations sprd_pci_ep_dev_fops = {
|
|
.owner = THIS_MODULE,
|
|
.open = sprd_pci_ep_dev_open,
|
|
.read = seq_read,
|
|
.llseek = seq_lseek,
|
|
.release = single_release,
|
|
};
|
|
|
|
static struct dentry *g_ep_debugfs_root;
|
|
|
|
static int sprd_pci_ep_dev_init_debugfs(void)
|
|
{
|
|
struct dentry *g_ep_debugfs_root = debugfs_create_dir("ep_dev", NULL);
|
|
|
|
if (!g_ep_debugfs_root)
|
|
return -ENXIO;
|
|
|
|
debugfs_create_file("ep", 0444,
|
|
g_ep_debugfs_root,
|
|
NULL, &sprd_pci_ep_dev_fops);
|
|
return 0;
|
|
}
|
|
|
|
static void sprd_pci_ep_dev_remove_debugfs(void)
|
|
{
|
|
debugfs_remove_recursive(g_ep_debugfs_root);
|
|
}
|
|
#endif
|
|
|
|
static int __init sprd_pci_ep_dev_init(void)
|
|
{
|
|
pr_info("%s %s\n", __func__, QUECTEL_SPRD_PCIE_VERSION);
|
|
#if defined(CONFIG_DEBUG_FS)
|
|
sprd_pci_ep_dev_init_debugfs();
|
|
sipc_init_debugfs();
|
|
#endif
|
|
sipc_init();
|
|
spipe_init();
|
|
spool_init();
|
|
modem_power_manager_init();
|
|
return pci_register_driver(&sprd_pci_ep_dev_driver);
|
|
}
|
|
|
|
static void __exit sprd_pci_ep_dev_exit(void)
|
|
{
|
|
spipe_exit();
|
|
spool_exit();
|
|
modem_power_manager_exit();
|
|
pci_unregister_driver(&sprd_pci_ep_dev_driver);
|
|
#if defined(CONFIG_DEBUG_FS)
|
|
sprd_pci_ep_dev_remove_debugfs();
|
|
#endif
|
|
sipc_exit();
|
|
}
|
|
|
|
module_init(sprd_pci_ep_dev_init);
|
|
module_exit(sprd_pci_ep_dev_exit);
|
|
|
|
|
|
MODULE_DESCRIPTION("SPRD PCI EP DEVICE HOST DRIVER");
|
|
MODULE_AUTHOR("Wenping Zhou <wenping.zhou@unisoc.com>");
|
|
MODULE_LICENSE("GPL v2");
|
|
MODULE_VERSION(QUECTEL_SPRD_PCIE_VERSION);
|