gitops/mt76
1
0
Fork 0
mirror of https://github.com/outbackdingo/mt76.git synced 2026-10-24 05:16:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

wifi: mt76: Introduce the NPU generic layer

Browse Source 
Add the NPU generic layer in mt76 module. NPU will be used to enable
traffic forward offloading between the MT76 NIC and the Airoha ethernet one
available on the Airoha EN7581 SoC using Netfilter Flowtable APIs.

Signed-off-by: Lorenzo Bianconi <[email protected]>
Link: https://patch.msgid.link/[email protected]
Signed-off-by: Felix Fietkau <[email protected]>
This commit is contained in:
Lorenzo Bianconi
2025-10-30 11:54:43 +00:00
committed by Felix Fietkau
parent a3b6c67c97
commit 141a815570
7 changed files with 1027 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Makefile
+1
View File
@@ -13,6 +13,7 @@ mt76-y := \
mmio.o util.o trace.o dma.o mac80211.o debugfs.o eeprom.o \
tx.o agg-rx.o mcu.o wed.o scan.o channel.o
mt76-$(CONFIG_MT76_NPU) += npu.o
mt76-$(CONFIG_PCI) += pci.o
mt76-$(CONFIG_NL80211_TESTMODE) += testmode.o
airoha_offload.h
+317
View File
@@ -0,0 +1,317 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (c) 2025 AIROHA Inc
* Author: Lorenzo Bianconi <[email protected]>
*/
#ifndef AIROHA_OFFLOAD_H
#define AIROHA_OFFLOAD_H
#include <linux/skbuff.h>
#include <linux/spinlock.h>
#include <linux/workqueue.h>
enum {
PPE_CPU_REASON_HIT_UNBIND_RATE_REACHED = 0x0f,
};
struct airoha_ppe_dev {
struct {
int (*setup_tc_block_cb)(struct airoha_ppe_dev *dev,
void *type_data);
void (*check_skb)(struct airoha_ppe_dev *dev,
struct sk_buff *skb, u16 hash,
bool rx_wlan);
} ops;
void *priv;
};
#if (IS_BUILTIN(CONFIG_NET_AIROHA) || IS_MODULE(CONFIG_NET_AIROHA))
struct airoha_ppe_dev *airoha_ppe_get_dev(struct device *dev);
void airoha_ppe_put_dev(struct airoha_ppe_dev *dev);
static inline int airoha_ppe_dev_setup_tc_block_cb(struct airoha_ppe_dev *dev,
void *type_data)
{
return dev->ops.setup_tc_block_cb(dev, type_data);
}
static inline void airoha_ppe_dev_check_skb(struct airoha_ppe_dev *dev,
struct sk_buff *skb,
u16 hash, bool rx_wlan)
{
dev->ops.check_skb(dev, skb, hash, rx_wlan);
}
#else
static inline struct airoha_ppe_dev *airoha_ppe_get_dev(struct device *dev)
{
return NULL;
}
static inline void airoha_ppe_put_dev(struct airoha_ppe_dev *dev)
{
}
static inline int airoha_ppe_setup_tc_block_cb(struct airoha_ppe_dev *dev,
void *type_data)
{
return -EOPNOTSUPP;
}
static inline void airoha_ppe_dev_check_skb(struct airoha_ppe_dev *dev,
struct sk_buff *skb, u16 hash,
bool rx_wlan)
{
}
#endif
#define NPU_NUM_CORES 8
#define NPU_NUM_IRQ 6
#define NPU_RX0_DESC_NUM 512
#define NPU_RX1_DESC_NUM 512
/* CTRL */
#define NPU_RX_DMA_DESC_LAST_MASK BIT(29)
#define NPU_RX_DMA_DESC_LEN_MASK GENMASK(28, 15)
#define NPU_RX_DMA_DESC_CUR_LEN_MASK GENMASK(14, 1)
#define NPU_RX_DMA_DESC_DONE_MASK BIT(0)
/* INFO */
#define NPU_RX_DMA_PKT_COUNT_MASK GENMASK(31, 28)
#define NPU_RX_DMA_PKT_ID_MASK GENMASK(28, 26)
#define NPU_RX_DMA_SRC_PORT_MASK GENMASK(25, 21)
#define NPU_RX_DMA_CRSN_MASK GENMASK(20, 16)
#define NPU_RX_DMA_FOE_ID_MASK GENMASK(15, 0)
/* DATA */
#define NPU_RX_DMA_SID_MASK GENMASK(31, 16)
#define NPU_RX_DMA_FRAG_TYPE_MASK GENMASK(15, 14)
#define NPU_RX_DMA_PRIORITY_MASK GENMASK(13, 10)
#define NPU_RX_DMA_RADIO_ID_MASK GENMASK(9, 6)
#define NPU_RX_DMA_VAP_ID_MASK GENMASK(5, 2)
#define NPU_RX_DMA_FRAME_TYPE_MASK GENMASK(1, 0)
struct airoha_npu_rx_dma_desc {
u32 ctrl;
u32 info;
u32 data;
u32 addr;
u64 rsv;
} __packed;
/* CTRL */
#define NPU_TX_DMA_DESC_SCHED_MASK BIT(31)
#define NPU_TX_DMA_DESC_LEN_MASK GENMASK(30, 18)
#define NPU_TX_DMA_DESC_VEND_LEN_MASK GENMASK(17, 1)
#define NPU_TX_DMA_DESC_DONE_MASK BIT(0)
#define NPU_TXWI_LEN 192
struct airoha_npu_tx_dma_desc {
u32 ctrl;
u32 addr;
u64 rsv;
u8 txwi[NPU_TXWI_LEN];
} __packed;
enum airoha_npu_wlan_set_cmd {
WLAN_FUNC_SET_WAIT_PCIE_ADDR,
WLAN_FUNC_SET_WAIT_DESC,
WLAN_FUNC_SET_WAIT_NPU_INIT_DONE,
WLAN_FUNC_SET_WAIT_TRAN_TO_CPU,
WLAN_FUNC_SET_WAIT_BA_WIN_SIZE,
WLAN_FUNC_SET_WAIT_DRIVER_MODEL,
WLAN_FUNC_SET_WAIT_DEL_STA,
WLAN_FUNC_SET_WAIT_DRAM_BA_NODE_ADDR,
WLAN_FUNC_SET_WAIT_PKT_BUF_ADDR,
WLAN_FUNC_SET_WAIT_IS_TEST_NOBA,
WLAN_FUNC_SET_WAIT_FLUSHONE_TIMEOUT,
WLAN_FUNC_SET_WAIT_FLUSHALL_TIMEOUT,
WLAN_FUNC_SET_WAIT_IS_FORCE_TO_CPU,
WLAN_FUNC_SET_WAIT_PCIE_STATE,
WLAN_FUNC_SET_WAIT_PCIE_PORT_TYPE,
WLAN_FUNC_SET_WAIT_ERROR_RETRY_TIMES,
WLAN_FUNC_SET_WAIT_BAR_INFO,
WLAN_FUNC_SET_WAIT_FAST_FLAG,
WLAN_FUNC_SET_WAIT_NPU_BAND0_ONCPU,
WLAN_FUNC_SET_WAIT_TX_RING_PCIE_ADDR,
WLAN_FUNC_SET_WAIT_TX_DESC_HW_BASE,
WLAN_FUNC_SET_WAIT_TX_BUF_SPACE_HW_BASE,
WLAN_FUNC_SET_WAIT_RX_RING_FOR_TXDONE_HW_BASE,
WLAN_FUNC_SET_WAIT_TX_PKT_BUF_ADDR,
WLAN_FUNC_SET_WAIT_INODE_TXRX_REG_ADDR,
WLAN_FUNC_SET_WAIT_INODE_DEBUG_FLAG,
WLAN_FUNC_SET_WAIT_INODE_HW_CFG_INFO,
WLAN_FUNC_SET_WAIT_INODE_STOP_ACTION,
WLAN_FUNC_SET_WAIT_INODE_PCIE_SWAP,
WLAN_FUNC_SET_WAIT_RATELIMIT_CTRL,
WLAN_FUNC_SET_WAIT_HWNAT_INIT,
WLAN_FUNC_SET_WAIT_ARHT_CHIP_INFO,
WLAN_FUNC_SET_WAIT_TX_BUF_CHECK_ADDR,
WLAN_FUNC_SET_WAIT_TOKEN_ID_SIZE,
};
enum airoha_npu_wlan_get_cmd {
WLAN_FUNC_GET_WAIT_NPU_INFO,
WLAN_FUNC_GET_WAIT_LAST_RATE,
WLAN_FUNC_GET_WAIT_COUNTER,
WLAN_FUNC_GET_WAIT_DBG_COUNTER,
WLAN_FUNC_GET_WAIT_RXDESC_BASE,
WLAN_FUNC_GET_WAIT_WCID_DBG_COUNTER,
WLAN_FUNC_GET_WAIT_DMA_ADDR,
WLAN_FUNC_GET_WAIT_RING_SIZE,
WLAN_FUNC_GET_WAIT_NPU_SUPPORT_MAP,
WLAN_FUNC_GET_WAIT_MDC_LOCK_ADDRESS,
WLAN_FUNC_GET_WAIT_NPU_VERSION,
};
struct airoha_npu {
#if (IS_BUILTIN(CONFIG_NET_AIROHA_NPU) || IS_MODULE(CONFIG_NET_AIROHA_NPU))
struct device *dev;
struct regmap *regmap;
struct airoha_npu_core {
struct airoha_npu *npu;
/* protect concurrent npu memory accesses */
spinlock_t lock;
struct work_struct wdt_work;
} cores[NPU_NUM_CORES];
int irqs[NPU_NUM_IRQ];
struct airoha_foe_stats __iomem *stats;
struct {
int (*ppe_init)(struct airoha_npu *npu);
int (*ppe_deinit)(struct airoha_npu *npu);
int (*ppe_init_stats)(struct airoha_npu *npu,
dma_addr_t addr, u32 num_stats_entries);
int (*ppe_flush_sram_entries)(struct airoha_npu *npu,
dma_addr_t foe_addr,
int sram_num_entries);
int (*ppe_foe_commit_entry)(struct airoha_npu *npu,
dma_addr_t foe_addr,
u32 entry_size, u32 hash,
bool ppe2);
int (*wlan_init_reserved_memory)(struct airoha_npu *npu);
int (*wlan_send_msg)(struct airoha_npu *npu, int ifindex,
enum airoha_npu_wlan_set_cmd func_id,
void *data, int data_len, gfp_t gfp);
int (*wlan_get_msg)(struct airoha_npu *npu, int ifindex,
enum airoha_npu_wlan_get_cmd func_id,
void *data, int data_len, gfp_t gfp);
u32 (*wlan_get_queue_addr)(struct airoha_npu *npu, int qid,
bool xmit);
void (*wlan_set_irq_status)(struct airoha_npu *npu, u32 val);
u32 (*wlan_get_irq_status)(struct airoha_npu *npu, int q);
void (*wlan_enable_irq)(struct airoha_npu *npu, int q);
void (*wlan_disable_irq)(struct airoha_npu *npu, int q);
} ops;
#endif
};
#if (IS_BUILTIN(CONFIG_NET_AIROHA_NPU) || IS_MODULE(CONFIG_NET_AIROHA_NPU))
struct airoha_npu *airoha_npu_get(struct device *dev);
void airoha_npu_put(struct airoha_npu *npu);
static inline int airoha_npu_wlan_init_reserved_memory(struct airoha_npu *npu)
{
return npu->ops.wlan_init_reserved_memory(npu);
}
static inline int airoha_npu_wlan_send_msg(struct airoha_npu *npu,
int ifindex,
enum airoha_npu_wlan_set_cmd cmd,
void *data, int data_len, gfp_t gfp)
{
return npu->ops.wlan_send_msg(npu, ifindex, cmd, data, data_len, gfp);
}
static inline int airoha_npu_wlan_get_msg(struct airoha_npu *npu, int ifindex,
enum airoha_npu_wlan_get_cmd cmd,
void *data, int data_len, gfp_t gfp)
{
return npu->ops.wlan_get_msg(npu, ifindex, cmd, data, data_len, gfp);
}
static inline u32 airoha_npu_wlan_get_queue_addr(struct airoha_npu *npu,
int qid, bool xmit)
{
return npu->ops.wlan_get_queue_addr(npu, qid, xmit);
}
static inline void airoha_npu_wlan_set_irq_status(struct airoha_npu *npu,
u32 val)
{
npu->ops.wlan_set_irq_status(npu, val);
}
static inline u32 airoha_npu_wlan_get_irq_status(struct airoha_npu *npu, int q)
{
return npu->ops.wlan_get_irq_status(npu, q);
}
static inline void airoha_npu_wlan_enable_irq(struct airoha_npu *npu, int q)
{
npu->ops.wlan_enable_irq(npu, q);
}
static inline void airoha_npu_wlan_disable_irq(struct airoha_npu *npu, int q)
{
npu->ops.wlan_disable_irq(npu, q);
}
#else
static inline struct airoha_npu *airoha_npu_get(struct device *dev)
{
return NULL;
}
static inline void airoha_npu_put(struct airoha_npu *npu)
{
}
static inline int airoha_npu_wlan_init_reserved_memory(struct airoha_npu *npu)
{
return -EOPNOTSUPP;
}
static inline int airoha_npu_wlan_send_msg(struct airoha_npu *npu,
int ifindex,
enum airoha_npu_wlan_set_cmd cmd,
void *data, int data_len, gfp_t gfp)
{
return -EOPNOTSUPP;
}
static inline int airoha_npu_wlan_get_msg(struct airoha_npu *npu, int ifindex,
enum airoha_npu_wlan_get_cmd cmd,
void *data, int data_len, gfp_t gfp)
{
return -EOPNOTSUPP;
}
static inline u32 airoha_npu_wlan_get_queue_addr(struct airoha_npu *npu,
int qid, bool xmit)
{
return 0;
}
static inline void airoha_npu_wlan_set_irq_status(struct airoha_npu *npu,
u32 val)
{
}
static inline u32 airoha_npu_wlan_get_irq_status(struct airoha_npu *npu,
int q)
{
return 0;
}
static inline void airoha_npu_wlan_enable_irq(struct airoha_npu *npu, int q)
{
}
static inline void airoha_npu_wlan_disable_irq(struct airoha_npu *npu, int q)
{
}
#endif
#endif /* AIROHA_OFFLOAD_H */
dma.c
+35 -6
View File
@@ -189,10 +189,15 @@ static void
mt76_dma_sync_idx(struct mt76_dev *dev, struct mt76_queue *q)
{
Q_WRITE(q, desc_base, q->desc_dma);
if (q->flags & MT_QFLAG_WED_RRO_EN)
if ((q->flags & MT_QFLAG_WED_RRO_EN) && !mt76_npu_device_active(dev))
Q_WRITE(q, ring_size, MT_DMA_RRO_EN | q->ndesc);
else
Q_WRITE(q, ring_size, q->ndesc);
if (mt76_queue_is_npu_tx(q)) {
writel(q->desc_dma, &q->regs->desc_base);
writel(q->ndesc, &q->regs->ring_size);
}
q->head = Q_READ(q, dma_idx);
q->tail = q->head;
}
@@ -204,7 +209,7 @@ void mt76_dma_queue_reset(struct mt76_dev *dev, struct mt76_queue *q,
return;
if (!mt76_queue_is_wed_rro_ind(q) &&
!mt76_queue_is_wed_rro_rxdmad_c(q)) {
!mt76_queue_is_wed_rro_rxdmad_c(q) && !mt76_queue_is_npu(q)) {
int i;
/* clear descriptors */
@@ -415,6 +420,7 @@ mt76_dma_tx_cleanup(struct mt76_dev *dev, struct mt76_queue *q, bool flush)
while (q->queued > 0 && q->tail != last) {
mt76_dma_tx_cleanup_idx(dev, q, q->tail, &entry);
mt76_npu_txdesc_cleanup(q, q->tail);
mt76_queue_tx_complete(dev, q, &entry);
if (entry.txwi) {
@@ -649,6 +655,10 @@ mt76_dma_tx_queue_skb(struct mt76_phy *phy, struct mt76_queue *q,
if (test_bit(MT76_RESET, &phy->state))
goto free_skb;
/* TODO: Take into account unlinear skbs */
if (mt76_npu_device_active(dev) && skb_linearize(skb))
goto free_skb;
t = mt76_get_txwi(dev);
if (!t)
goto free_skb;
@@ -696,6 +706,9 @@ mt76_dma_tx_queue_skb(struct mt76_phy *phy, struct mt76_queue *q,
if (ret < 0)
goto unmap;
if (mt76_npu_device_active(dev))
return mt76_npu_dma_add_buf(phy, q, skb, &tx_info.buf[1], txwi);
return mt76_dma_add_buf(dev, q, tx_info.buf, tx_info.nbuf,
tx_info.info, tx_info.skb, t);
@@ -796,8 +809,15 @@ mt76_dma_alloc_queue(struct mt76_dev *dev, struct mt76_queue *q,
q->hw_idx = idx;
q->dev = dev;
size = mt76_queue_is_wed_rro_ind(q) ? sizeof(struct mt76_wed_rro_desc)
: sizeof(struct mt76_desc);
if (mt76_queue_is_wed_rro_ind(q))
size = sizeof(struct mt76_wed_rro_desc);
else if (mt76_queue_is_npu_tx(q))
size = sizeof(struct airoha_npu_tx_dma_desc);
else if (mt76_queue_is_npu_rx(q))
size = sizeof(struct airoha_npu_rx_dma_desc);
else
size = sizeof(struct mt76_desc);
q->desc = dmam_alloc_coherent(dev->dma_dev, q->ndesc * size,
&q->desc_dma, GFP_KERNEL);
if (!q->desc)
@@ -813,6 +833,7 @@ mt76_dma_alloc_queue(struct mt76_dev *dev, struct mt76_queue *q,
if (ret)
return ret;
mt76_npu_queue_setup(dev, q);
ret = mt76_wed_dma_setup(dev, q, false);
if (ret)
return ret;
@@ -840,6 +861,11 @@ mt76_dma_rx_cleanup(struct mt76_dev *dev, struct mt76_queue *q)
if (!q->ndesc)
return;
if (mt76_queue_is_npu(q)) {
mt76_npu_queue_cleanup(dev, q);
return;
}
do {
spin_lock_bh(&q->lock);
buf = mt76_dma_dequeue(dev, q, true, NULL, NULL, &more, NULL);
@@ -870,7 +896,7 @@ mt76_dma_rx_reset(struct mt76_dev *dev, enum mt76_rxq_id qid)
return;
if (!mt76_queue_is_wed_rro_ind(q) &&
!mt76_queue_is_wed_rro_rxdmad_c(q)) {
!mt76_queue_is_wed_rro_rxdmad_c(q) && !mt76_queue_is_npu(q)) {
int i;
for (i = 0; i < q->ndesc; i++)
@@ -890,7 +916,10 @@ mt76_dma_rx_reset(struct mt76_dev *dev, enum mt76_rxq_id qid)
return;
mt76_dma_sync_idx(dev, q);
mt76_dma_rx_fill_buf(dev, q, false);
if (mt76_queue_is_npu(q))
mt76_npu_fill_rx_queue(dev, q);
else
mt76_dma_rx_fill(dev, q, false);
}
static void
dma.h
+36
View File
@@ -70,6 +70,42 @@
writel(_val, &(_q)->regs->_field); \
} while (0)
#elif IS_ENABLED(CONFIG_MT76_NPU)
#define Q_READ(_q, _field) ({ \
u32 _offset = offsetof(struct mt76_queue_regs, _field); \
u32 _val; \
if ((_q)->flags & MT_QFLAG_NPU) { \
struct airoha_npu *npu; \
\
rcu_read_lock(); \
npu = rcu_dereference(q->dev->mmio.npu); \
if (npu) \
regmap_read(npu->regmap, \
((_q)->wed_regs + _offset), &_val); \
rcu_read_unlock(); \
} else { \
_val = readl(&(_q)->regs->_field); \
} \
_val; \
})
#define Q_WRITE(_q, _field, _val) do { \
u32 _offset = offsetof(struct mt76_queue_regs, _field); \
if ((_q)->flags & MT_QFLAG_NPU) { \
struct airoha_npu *npu; \
\
rcu_read_lock(); \
npu = rcu_dereference(q->dev->mmio.npu); \
if (npu) \
regmap_write(npu->regmap, \
((_q)->wed_regs + _offset), _val); \
rcu_read_unlock(); \
} else { \
writel(_val, &(_q)->regs->_field); \
} \
} while (0)
#else
#define Q_READ(_q, _field) readl(&(_q)->regs->_field)
mac80211.c
+5 -1
View File
@@ -625,6 +625,8 @@ int mt76_create_page_pool(struct mt76_dev *dev, struct mt76_queue *q)
case MT_RXQ_MAIN:
case MT_RXQ_BAND1:
case MT_RXQ_BAND2:
case MT_RXQ_NPU0:
case MT_RXQ_NPU1:
pp_params.pool_size = 256;
break;
default:
@@ -806,6 +808,7 @@ void mt76_free_device(struct mt76_dev *dev)
destroy_workqueue(dev->wq);
dev->wq = NULL;
}
mt76_npu_deinit(dev);
ieee80211_free_hw(dev->hw);
}
EXPORT_SYMBOL_GPL(mt76_free_device);
@@ -1545,7 +1548,8 @@ void mt76_rx_poll_complete(struct mt76_dev *dev, enum mt76_rxq_id q,
while ((skb = __skb_dequeue(&dev->rx_skb[q])) != NULL) {
mt76_check_sta(dev, skb);
if (mtk_wed_device_active(&dev->mmio.wed))
if (mtk_wed_device_active(&dev->mmio.wed) ||
mt76_npu_device_active(dev))
__skb_queue_tail(&frames, skb);
else
mt76_rx_aggr_reorder(skb, &frames);
mt76.h
+139
View File
@@ -13,6 +13,11 @@
#include <linux/leds.h>
#include <linux/usb.h>
#include <linux/average.h>
#if (IS_BUILTIN(CONFIG_NET_AIROHA_NPU) || IS_MODULE(CONFIG_NET_AIROHA_NPU))
#include <linux/soc/airoha/airoha_offload.h>
#else
#include "airoha_offload.h"
#endif
#include <linux/soc/mediatek/mtk_wed.h>
#include <net/netlink.h>
#include <net/mac80211.h>
@@ -42,6 +47,7 @@
#define MT_QFLAG_WED_RRO BIT(6)
#define MT_QFLAG_WED_RRO_EN BIT(7)
#define MT_QFLAG_EMI_EN BIT(8)
#define MT_QFLAG_NPU BIT(9)
#define __MT_WED_Q(_type, _n) (MT_QFLAG_WED | \
FIELD_PREP(MT_QFLAG_WED_TYPE, _type) | \
@@ -56,6 +62,12 @@
#define MT_WED_RRO_Q_IND __MT_WED_RRO_Q(MT76_WED_RRO_Q_IND, 0)
#define MT_WED_RRO_Q_RXDMAD_C __MT_WED_RRO_Q(MT76_WED_RRO_Q_RXDMAD_C, 0)
#define __MT_NPU_Q(_type, _n) (MT_QFLAG_NPU | \
FIELD_PREP(MT_QFLAG_WED_TYPE, _type) | \
FIELD_PREP(MT_QFLAG_WED_RING, _n))
#define MT_NPU_Q_TX(_n) __MT_NPU_Q(MT76_WED_Q_TX, _n)
#define MT_NPU_Q_RX(_n) __MT_NPU_Q(MT76_WED_Q_RX, _n)
struct mt76_dev;
struct mt76_phy;
struct mt76_wcid;
@@ -147,6 +159,8 @@ enum mt76_rxq_id {
MT_RXQ_TXFREE_BAND2,
MT_RXQ_RRO_IND,
MT_RXQ_RRO_RXDMAD_C,
MT_RXQ_NPU0,
MT_RXQ_NPU1,
__MT_RXQ_MAX
};
@@ -715,6 +729,11 @@ struct mt76_mmio {
struct mtk_wed_device wed_hif2;
struct completion wed_reset;
struct completion wed_reset_complete;
struct airoha_ppe_dev __rcu *ppe_dev;
struct airoha_npu __rcu *npu;
phys_addr_t phy_addr;
int npu_type;
};
struct mt76_rx_status {
@@ -1624,6 +1643,109 @@ int mt76_testmode_dump(struct ieee80211_hw *hw, struct sk_buff *skb,
int mt76_testmode_set_state(struct mt76_phy *phy, enum mt76_testmode_state state);
int mt76_testmode_alloc_skb(struct mt76_phy *phy, u32 len);
#ifdef CONFIG_MT76_NPU
void mt76_npu_check_ppe(struct mt76_dev *dev, struct sk_buff *skb,
u32 info);
int mt76_npu_dma_add_buf(struct mt76_phy *phy, struct mt76_queue *q,
struct sk_buff *skb, struct mt76_queue_buf *buf,
void *txwi_ptr);
int mt76_npu_rx_queue_init(struct mt76_dev *dev, struct mt76_queue *q);
int mt76_npu_fill_rx_queue(struct mt76_dev *dev, struct mt76_queue *q);
void mt76_npu_queue_cleanup(struct mt76_dev *dev, struct mt76_queue *q);
void mt76_npu_disable_irqs(struct mt76_dev *dev);
int mt76_npu_init(struct mt76_dev *dev, phys_addr_t phy_addr, int type);
void mt76_npu_deinit(struct mt76_dev *dev);
void mt76_npu_queue_setup(struct mt76_dev *dev, struct mt76_queue *q);
void mt76_npu_txdesc_cleanup(struct mt76_queue *q, int index);
int mt76_npu_net_setup_tc(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
struct net_device *dev, enum tc_setup_type type,
void *type_data);
#else
static inline void mt76_npu_check_ppe(struct mt76_dev *dev,
struct sk_buff *skb, u32 info)
{
}
static inline int mt76_npu_dma_add_buf(struct mt76_phy *phy,
struct mt76_queue *q,
struct sk_buff *skb,
struct mt76_queue_buf *buf,
void *txwi_ptr)
{
return -EOPNOTSUPP;
}
static inline int mt76_npu_fill_rx_queue(struct mt76_dev *dev,
struct mt76_queue *q)
{
return 0;
}
static inline void mt76_npu_queue_cleanup(struct mt76_dev *dev,
struct mt76_queue *q)
{
}
static inline void mt76_npu_disable_irqs(struct mt76_dev *dev)
{
}
static inline int mt76_npu_init(struct mt76_dev *dev, phys_addr_t phy_addr,
int type)
{
return 0;
}
static inline void mt76_npu_deinit(struct mt76_dev *dev)
{
}
static inline void mt76_npu_queue_setup(struct mt76_dev *dev,
struct mt76_queue *q)
{
}
static inline void mt76_npu_txdesc_cleanup(struct mt76_queue *q,
int index)
{
}
static inline int mt76_npu_net_setup_tc(struct ieee80211_hw *hw,
struct ieee80211_vif *vif,
struct net_device *dev,
enum tc_setup_type type,
void *type_data)
{
return -EOPNOTSUPP;
}
#endif /* CONFIG_MT76_NPU */
static inline bool mt76_npu_device_active(struct mt76_dev *dev)
{
return !!rcu_access_pointer(dev->mmio.npu);
}
static inline bool mt76_ppe_device_active(struct mt76_dev *dev)
{
return !!rcu_access_pointer(dev->mmio.ppe_dev);
}
static inline int mt76_npu_send_msg(struct airoha_npu *npu, int ifindex,
enum airoha_npu_wlan_set_cmd cmd,
u32 val, gfp_t gfp)
{
return airoha_npu_wlan_send_msg(npu, ifindex, cmd, &val, sizeof(val),
gfp);
}
static inline int mt76_npu_get_msg(struct airoha_npu *npu, int ifindex,
enum airoha_npu_wlan_get_cmd cmd,
u32 *val, gfp_t gfp)
{
return airoha_npu_wlan_get_msg(npu, ifindex, cmd, val, sizeof(*val),
gfp);
}
static inline void mt76_testmode_reset(struct mt76_phy *phy, bool disable)
{
#ifdef CONFIG_NL80211_TESTMODE
@@ -1866,6 +1988,23 @@ static inline bool mt76_queue_is_emi(struct mt76_queue *q)
return q->flags & MT_QFLAG_EMI_EN;
}
static inline bool mt76_queue_is_npu(struct mt76_queue *q)
{
return q->flags & MT_QFLAG_NPU;
}
static inline bool mt76_queue_is_npu_tx(struct mt76_queue *q)
{
return mt76_queue_is_npu(q) &&
FIELD_GET(MT_QFLAG_WED_TYPE, q->flags) == MT76_WED_Q_TX;
}
static inline bool mt76_queue_is_npu_rx(struct mt76_queue *q)
{
return mt76_queue_is_npu(q) &&
FIELD_GET(MT_QFLAG_WED_TYPE, q->flags) == MT76_WED_Q_RX;
}
struct mt76_txwi_cache *
mt76_token_release(struct mt76_dev *dev, int token, bool *wake);
int mt76_token_consume(struct mt76_dev *dev, struct mt76_txwi_cache **ptxwi);
npu.c
+494
View File
@@ -0,0 +1,494 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (c) 2025 AIROHA Inc
* Author: Lorenzo Bianconi <[email protected]>
*/
#include <linux/kernel.h>
#include <net/flow_offload.h>
#include <net/pkt_cls.h>
#include "mt76.h"
#include "dma.h"
#include "mt76_connac.h"
#define MT76_NPU_RX_BUF_SIZE (1800 + \
SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
int mt76_npu_fill_rx_queue(struct mt76_dev *dev, struct mt76_queue *q)
{
int nframes = 0;
while (q->queued < q->ndesc - 1) {
struct airoha_npu_rx_dma_desc *desc = (void *)q->desc;
struct mt76_queue_entry *e = &q->entry[q->head];
struct page *page;
int offset;
e->buf = mt76_get_page_pool_buf(q, &offset, q->buf_size);
if (!e->buf)
break;
e->dma_len[0] = SKB_WITH_OVERHEAD(q->buf_size);
page = virt_to_head_page(e->buf);
e->dma_addr[0] = page_pool_get_dma_addr(page) + offset;
memset(&desc[q->head], 0, sizeof(*desc));
desc[q->head].addr = e->dma_addr[0];
q->head = (q->head + 1) % q->ndesc;
q->queued++;
nframes++;
}
return nframes;
}
void mt76_npu_queue_cleanup(struct mt76_dev *dev, struct mt76_queue *q)
{
spin_lock_bh(&q->lock);
while (q->queued > 0) {
struct mt76_queue_entry *e = &q->entry[q->tail];
dma_sync_single_for_cpu(dev->dma_dev, e->dma_addr[0],
e->dma_len[0],
page_pool_get_dma_dir(q->page_pool));
mt76_put_page_pool_buf(e->buf, false);
q->tail = (q->tail + 1) % q->ndesc;
q->queued--;
}
spin_unlock_bh(&q->lock);
}
static struct sk_buff *mt76_npu_dequeue(struct mt76_dev *dev,
struct mt76_queue *q,
u32 *info)
{
struct airoha_npu_rx_dma_desc *desc = (void *)q->desc;
int i, nframes, index = q->tail;
struct sk_buff *skb = NULL;
nframes = FIELD_GET(NPU_RX_DMA_PKT_COUNT_MASK, desc[index].info);
nframes = max_t(int, nframes, 1);
for (i = 0; i < nframes; i++) {
struct mt76_queue_entry *e = &q->entry[index];
int len = FIELD_GET(NPU_RX_DMA_DESC_CUR_LEN_MASK,
desc[index].ctrl);
if (!FIELD_GET(NPU_RX_DMA_DESC_DONE_MASK, desc[index].ctrl)) {
dev_kfree_skb(skb);
return NULL;
}
dma_sync_single_for_cpu(dev->dma_dev, e->dma_addr[0],
e->dma_len[0],
page_pool_get_dma_dir(q->page_pool));
if (!skb) {
skb = napi_build_skb(e->buf, q->buf_size);
if (!skb)
return NULL;
__skb_put(skb, len);
skb_reset_mac_header(skb);
skb_mark_for_recycle(skb);
} else {
struct skb_shared_info *shinfo = skb_shinfo(skb);
struct page *page = virt_to_head_page(e->buf);
int nr_frags = shinfo->nr_frags;
if (nr_frags < ARRAY_SIZE(shinfo->frags))
skb_add_rx_frag(skb, nr_frags, page,
e->buf - page_address(page),
len, q->buf_size);
}
*info = desc[index].info;
index = (index + 1) % q->ndesc;
}
q->tail = index;
q->queued -= i;
Q_WRITE(q, dma_idx, q->tail);
return skb;
}
void mt76_npu_check_ppe(struct mt76_dev *dev, struct sk_buff *skb,
u32 info)
{
struct airoha_ppe_dev *ppe_dev;
u16 reason, hash;
if (!mt76_npu_device_active(dev))
return;
rcu_read_lock();
ppe_dev = rcu_dereference(dev->mmio.ppe_dev);
if (!ppe_dev)
goto out;
hash = FIELD_GET(NPU_RX_DMA_FOE_ID_MASK, info);
skb_set_hash(skb, hash, PKT_HASH_TYPE_L4);
reason = FIELD_GET(NPU_RX_DMA_CRSN_MASK, info);
if (reason == PPE_CPU_REASON_HIT_UNBIND_RATE_REACHED) {
skb_set_mac_header(skb, 0);
airoha_ppe_dev_check_skb(ppe_dev, skb, hash, true);
}
out:
rcu_read_unlock();
}
EXPORT_SYMBOL_GPL(mt76_npu_check_ppe);
static int mt76_npu_rx_poll(struct napi_struct *napi, int budget)
{
struct mt76_dev *dev = mt76_priv(napi->dev);
enum mt76_rxq_id qid = napi - dev->napi;
struct airoha_npu *npu;
int done = 0;
rcu_read_lock();
npu = rcu_dereference(dev->mmio.npu);
if (!npu)
goto out;
while (done < budget) {
struct sk_buff *skb;
u32 info = 0;
skb = mt76_npu_dequeue(dev, &dev->q_rx[qid], &info);
if (!skb)
break;
dev->drv->rx_skb(dev, qid, skb, &info);
mt76_rx_poll_complete(dev, qid, napi);
done++;
}
mt76_npu_fill_rx_queue(dev, &dev->q_rx[qid]);
out:
if (done < budget && napi_complete(napi))
dev->drv->rx_poll_complete(dev, qid);
rcu_read_unlock();
return done;
}
static irqreturn_t mt76_npu_irq_handler(int irq, void *q_instance)
{
struct mt76_queue *q = q_instance;
struct mt76_dev *dev = q->dev;
int qid = q - &dev->q_rx[0];
int index = qid - MT_RXQ_NPU0;
struct airoha_npu *npu;
u32 status;
rcu_read_lock();
npu = rcu_dereference(dev->mmio.npu);
if (!npu)
goto out;
status = airoha_npu_wlan_get_irq_status(npu, index);
airoha_npu_wlan_set_irq_status(npu, status);
airoha_npu_wlan_disable_irq(npu, index);
napi_schedule(&dev->napi[qid]);
out:
rcu_read_unlock();
return IRQ_HANDLED;
}
int mt76_npu_dma_add_buf(struct mt76_phy *phy, struct mt76_queue *q,
struct sk_buff *skb, struct mt76_queue_buf *buf,
void *txwi_ptr)
{
u16 txwi_len = min_t(u16, phy->dev->drv->txwi_size, NPU_TXWI_LEN);
struct airoha_npu_tx_dma_desc *desc = (void *)q->desc;
int ret;
/* TODO: Take into account unlinear skbs */
memcpy(desc[q->head].txwi, txwi_ptr, txwi_len);
desc[q->head].addr = buf->addr;
desc[q->head].ctrl = FIELD_PREP(NPU_TX_DMA_DESC_VEND_LEN_MASK, txwi_len) |
FIELD_PREP(NPU_TX_DMA_DESC_LEN_MASK, skb->len) |
NPU_TX_DMA_DESC_DONE_MASK;
ret = q->head;
q->entry[q->head].skip_buf0 = true;
q->entry[q->head].skip_buf1 = true;
q->entry[q->head].txwi = NULL;
q->entry[q->head].skb = NULL;
q->entry[q->head].wcid = 0xffff;
q->head = (q->head + 1) % q->ndesc;
q->queued++;
return ret;
}
void mt76_npu_txdesc_cleanup(struct mt76_queue *q, int index)
{
struct airoha_npu_tx_dma_desc *desc = (void *)q->desc;
if (!mt76_queue_is_npu_tx(q))
return;
desc[index].ctrl &= ~NPU_TX_DMA_DESC_DONE_MASK;
}
void mt76_npu_queue_setup(struct mt76_dev *dev, struct mt76_queue *q)
{
int qid = FIELD_GET(MT_QFLAG_WED_RING, q->flags);
bool xmit = mt76_queue_is_npu_tx(q);
struct airoha_npu *npu;
if (!mt76_queue_is_npu(q))
return;
npu = rcu_dereference_protected(dev->mmio.npu, &dev->mutex);
if (npu)
q->wed_regs = airoha_npu_wlan_get_queue_addr(npu, qid, xmit);
}
int mt76_npu_rx_queue_init(struct mt76_dev *dev, struct mt76_queue *q)
{
int err, irq, qid = q - &dev->q_rx[0];
int size, index = qid - MT_RXQ_NPU0;
struct airoha_npu *npu;
const char *name;
mutex_lock(&dev->mutex);
npu = rcu_dereference_protected(dev->mmio.npu, &dev->mutex);
irq = npu && index < ARRAY_SIZE(npu->irqs) ? npu->irqs[index]
: -EINVAL;
if (irq < 0) {
err = irq;
goto out;
}
q->flags = MT_NPU_Q_RX(index);
size = qid == MT_RXQ_NPU1 ? NPU_RX1_DESC_NUM : NPU_RX0_DESC_NUM;
err = dev->queue_ops->alloc(dev, q, 0, size,
MT76_NPU_RX_BUF_SIZE, 0);
if (err)
goto out;
name = devm_kasprintf(dev->dev, GFP_KERNEL, "mt76-npu.%d", index);
if (!name) {
err = -ENOMEM;
goto out;
}
err = devm_request_irq(dev->dev, irq, mt76_npu_irq_handler,
IRQF_SHARED, name, q);
if (err)
goto out;
netif_napi_add(dev->napi_dev, &dev->napi[qid], mt76_npu_rx_poll);
mt76_npu_fill_rx_queue(dev, q);
napi_enable(&dev->napi[qid]);
out:
mutex_unlock(&dev->mutex);
return err;
}
EXPORT_SYMBOL_GPL(mt76_npu_rx_queue_init);
static int mt76_npu_setup_tc_block_cb(enum tc_setup_type type,
void *type_data, void *cb_priv)
{
struct mt76_phy *phy = cb_priv;
if (type != TC_SETUP_CLSFLOWER)
return -EOPNOTSUPP;
if (!mt76_ppe_device_active(phy->dev))
return -EOPNOTSUPP;
return airoha_ppe_dev_setup_tc_block_cb(phy->dev->mmio.ppe_dev,
type_data);
}
static int mt76_npu_setup_tc_block(struct mt76_phy *phy,
struct net_device *dev,
struct flow_block_offload *f)
{
flow_setup_cb_t *cb = mt76_npu_setup_tc_block_cb;
static LIST_HEAD(block_cb_list);
struct flow_block_cb *block_cb;
if (f->binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS)
return -EOPNOTSUPP;
if (!tc_can_offload(dev))
return -EOPNOTSUPP;
f->driver_block_list = &block_cb_list;
switch (f->command) {
case FLOW_BLOCK_BIND:
block_cb = flow_block_cb_lookup(f->block, cb, dev);
if (block_cb) {
flow_block_cb_incref(block_cb);
return 0;
}
block_cb = flow_block_cb_alloc(cb, dev, phy, NULL);
if (IS_ERR(block_cb))
return PTR_ERR(block_cb);
flow_block_cb_incref(block_cb);
flow_block_cb_add(block_cb, f);
list_add_tail(&block_cb->driver_list, &block_cb_list);
return 0;
case FLOW_BLOCK_UNBIND:
block_cb = flow_block_cb_lookup(f->block, cb, dev);
if (!block_cb)
return -ENOENT;
if (!flow_block_cb_decref(block_cb)) {
flow_block_cb_remove(block_cb, f);
list_del(&block_cb->driver_list);
}
return 0;
default:
return -EOPNOTSUPP;
}
}
int mt76_npu_net_setup_tc(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
struct net_device *dev, enum tc_setup_type type,
void *type_data)
{
struct mt76_phy *phy = hw->priv;
if (!tc_can_offload(dev))
return -EOPNOTSUPP;
if (!mt76_npu_device_active(phy->dev))
return -EOPNOTSUPP;
switch (type) {
case TC_SETUP_BLOCK:
case TC_SETUP_FT:
return mt76_npu_setup_tc_block(phy, dev, type_data);
default:
return -EOPNOTSUPP;
}
}
EXPORT_SYMBOL_GPL(mt76_npu_net_setup_tc);
void mt76_npu_disable_irqs(struct mt76_dev *dev)
{
struct airoha_npu *npu;
int i;
rcu_read_lock();
npu = rcu_dereference(dev->mmio.npu);
if (!npu)
goto unlock;
for (i = MT_RXQ_NPU0; i <= MT_RXQ_NPU1; i++) {
int qid = i - MT_RXQ_NPU0;
u32 status;
status = airoha_npu_wlan_get_irq_status(npu, qid);
airoha_npu_wlan_set_irq_status(npu, status);
airoha_npu_wlan_disable_irq(npu, qid);
}
unlock:
rcu_read_unlock();
}
EXPORT_SYMBOL_GPL(mt76_npu_disable_irqs);
int mt76_npu_init(struct mt76_dev *dev, phys_addr_t phy_addr, int type)
{
struct airoha_ppe_dev *ppe_dev;
struct airoha_npu *npu;
int err = 0;
/* NPU offloading is only supported by MT7992 */
if (!is_mt7992(dev))
return 0;
mutex_lock(&dev->mutex);
npu = airoha_npu_get(dev->dev);
if (IS_ERR(npu)) {
request_module("airoha-npu");
npu = airoha_npu_get(dev->dev);
}
if (IS_ERR(npu)) {
err = PTR_ERR(npu);
goto error_unlock;
}
ppe_dev = airoha_ppe_get_dev(dev->dev);
if (IS_ERR(ppe_dev)) {
request_module("airoha-eth");
ppe_dev = airoha_ppe_get_dev(dev->dev);
}
if (IS_ERR(ppe_dev)) {
err = PTR_ERR(ppe_dev);
goto error_npu_put;
}
err = airoha_npu_wlan_init_reserved_memory(npu);
if (err)
goto error_ppe_put;
dev->dma_dev = npu->dev;
dev->mmio.phy_addr = phy_addr;
dev->mmio.npu_type = type;
/* NPU offloading requires HW-RRO for RX packet reordering. */
dev->hwrro_mode = MT76_HWRRO_V3_1;
rcu_assign_pointer(dev->mmio.npu, npu);
rcu_assign_pointer(dev->mmio.ppe_dev, ppe_dev);
synchronize_rcu();
mutex_unlock(&dev->mutex);
return 0;
error_ppe_put:
airoha_ppe_put_dev(ppe_dev);
error_npu_put:
airoha_npu_put(npu);
error_unlock:
mutex_unlock(&dev->mutex);
return err;
}
EXPORT_SYMBOL_GPL(mt76_npu_init);
void mt76_npu_deinit(struct mt76_dev *dev)
{
struct airoha_ppe_dev *ppe_dev;
struct airoha_npu *npu;
mutex_lock(&dev->mutex);
npu = rcu_replace_pointer(dev->mmio.npu, NULL,
lockdep_is_held(&dev->mutex));
if (npu)
airoha_npu_put(npu);
ppe_dev = rcu_replace_pointer(dev->mmio.ppe_dev, NULL,
lockdep_is_held(&dev->mutex));
if (ppe_dev)
airoha_ppe_put_dev(ppe_dev);
mutex_unlock(&dev->mutex);
mt76_npu_queue_cleanup(dev, &dev->q_rx[MT_RXQ_NPU0]);
mt76_npu_queue_cleanup(dev, &dev->q_rx[MT_RXQ_NPU1]);
}