drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_atcam.c

   1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
   2 /* Copyright (c) 2018 Mellanox Technologies. All rights reserved */
   3
   4 #include <linux/kernel.h>
   5 #include <linux/err.h>
   6 #include <linux/errno.h>
   7 #include <linux/gfp.h>
   8 #include <linux/refcount.h>
   9 #include <linux/rhashtable.h>
  10 #define CREATE_TRACE_POINTS
  11 #include <trace/events/mlxsw.h>
  12
  13 #include "reg.h"
  14 #include "core.h"
  15 #include "spectrum.h"
  16 #include "spectrum_acl_tcam.h"
  17 #include "core_acl_flex_keys.h"
  18
  19 #define MLXSW_SP_ACL_ATCAM_LKEY_ID_BLOCK_CLEAR_START    0
  20 #define MLXSW_SP_ACL_ATCAM_LKEY_ID_BLOCK_CLEAR_END      5
  21
  22 struct mlxsw_sp_acl_atcam_lkey_id_ht_key {
  23         char enc_key[MLXSW_REG_PTCEX_FLEX_KEY_BLOCKS_LEN]; /* MSB blocks */
  24         u8 erp_id;
  25 };
  26
  27 struct mlxsw_sp_acl_atcam_lkey_id {
  28         struct rhash_head ht_node;
  29         struct mlxsw_sp_acl_atcam_lkey_id_ht_key ht_key;
  30         refcount_t refcnt;
  31         u32 id;
  32 };
  33
  34 struct mlxsw_sp_acl_atcam_region_ops {
  35         int (*init)(struct mlxsw_sp_acl_atcam_region *aregion);
  36         void (*fini)(struct mlxsw_sp_acl_atcam_region *aregion);
  37         struct mlxsw_sp_acl_atcam_lkey_id *
  38                 (*lkey_id_get)(struct mlxsw_sp_acl_atcam_region *aregion,
  39                                char *enc_key, u8 erp_id);
  40         void (*lkey_id_put)(struct mlxsw_sp_acl_atcam_region *aregion,
  41                             struct mlxsw_sp_acl_atcam_lkey_id *lkey_id);
  42 };
  43
  44 struct mlxsw_sp_acl_atcam_region_generic {
  45         struct mlxsw_sp_acl_atcam_lkey_id dummy_lkey_id;
  46 };
  47
  48 struct mlxsw_sp_acl_atcam_region_12kb {
  49         struct rhashtable lkey_ht;
  50         unsigned int max_lkey_id;
  51         unsigned long *used_lkey_id;
  52 };
  53
  54 static const struct rhashtable_params mlxsw_sp_acl_atcam_lkey_id_ht_params = {
  55         .key_len = sizeof(struct mlxsw_sp_acl_atcam_lkey_id_ht_key),
  56         .key_offset = offsetof(struct mlxsw_sp_acl_atcam_lkey_id, ht_key),
  57         .head_offset = offsetof(struct mlxsw_sp_acl_atcam_lkey_id, ht_node),
  58 };
  59
  60 static const struct rhashtable_params mlxsw_sp_acl_atcam_entries_ht_params = {
  61         .key_len = sizeof(struct mlxsw_sp_acl_atcam_entry_ht_key),
  62         .key_offset = offsetof(struct mlxsw_sp_acl_atcam_entry, ht_key),
  63         .head_offset = offsetof(struct mlxsw_sp_acl_atcam_entry, ht_node),
  64 };
  65
  66 static bool
  67 mlxsw_sp_acl_atcam_is_centry(const struct mlxsw_sp_acl_atcam_entry *aentry)
  68 {
  69         return mlxsw_sp_acl_erp_mask_is_ctcam(aentry->erp_mask);
  70 }
  71
  72 static int
  73 mlxsw_sp_acl_atcam_region_generic_init(struct mlxsw_sp_acl_atcam_region *aregion)
  74 {
  75         struct mlxsw_sp_acl_atcam_region_generic *region_generic;
  76
  77         region_generic = kzalloc(sizeof(*region_generic), GFP_KERNEL);
  78         if (!region_generic)
  79                 return -ENOMEM;
  80
  81         refcount_set(&region_generic->dummy_lkey_id.refcnt, 1);
  82         aregion->priv = region_generic;
  83
  84         return 0;
  85 }
  86
  87 static void
  88 mlxsw_sp_acl_atcam_region_generic_fini(struct mlxsw_sp_acl_atcam_region *aregion)
  89 {
  90         kfree(aregion->priv);
  91 }
  92
  93 static struct mlxsw_sp_acl_atcam_lkey_id *
  94 mlxsw_sp_acl_atcam_generic_lkey_id_get(struct mlxsw_sp_acl_atcam_region *aregion,
  95                                        char *enc_key, u8 erp_id)
  96 {
  97         struct mlxsw_sp_acl_atcam_region_generic *region_generic;
  98
  99         region_generic = aregion->priv;
 100         return &region_generic->dummy_lkey_id;
 101 }
 102
 103 static void
 104 mlxsw_sp_acl_atcam_generic_lkey_id_put(struct mlxsw_sp_acl_atcam_region *aregion,
 105                                        struct mlxsw_sp_acl_atcam_lkey_id *lkey_id)
 106 {
 107 }
 108
 109 static const struct mlxsw_sp_acl_atcam_region_ops
 110 mlxsw_sp_acl_atcam_region_generic_ops = {
 111         .init           = mlxsw_sp_acl_atcam_region_generic_init,
 112         .fini           = mlxsw_sp_acl_atcam_region_generic_fini,
 113         .lkey_id_get    = mlxsw_sp_acl_atcam_generic_lkey_id_get,
 114         .lkey_id_put    = mlxsw_sp_acl_atcam_generic_lkey_id_put,
 115 };
 116
 117 static int
 118 mlxsw_sp_acl_atcam_region_12kb_init(struct mlxsw_sp_acl_atcam_region *aregion)
 119 {
 120         struct mlxsw_sp *mlxsw_sp = aregion->region->mlxsw_sp;
 121         struct mlxsw_sp_acl_atcam_region_12kb *region_12kb;
 122         size_t alloc_size;
 123         u64 max_lkey_id;
 124         int err;
 125
 126         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_MAX_LARGE_KEY_ID))
 127                 return -EIO;
 128
 129         max_lkey_id = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_MAX_LARGE_KEY_ID);
 130         region_12kb = kzalloc(sizeof(*region_12kb), GFP_KERNEL);
 131         if (!region_12kb)
 132                 return -ENOMEM;
 133
 134         alloc_size = BITS_TO_LONGS(max_lkey_id) * sizeof(unsigned long);
 135         region_12kb->used_lkey_id = kzalloc(alloc_size, GFP_KERNEL);
 136         if (!region_12kb->used_lkey_id) {
 137                 err = -ENOMEM;
 138                 goto err_used_lkey_id_alloc;
 139         }
 140
 141         err = rhashtable_init(&region_12kb->lkey_ht,
 142                               &mlxsw_sp_acl_atcam_lkey_id_ht_params);
 143         if (err)
 144                 goto err_rhashtable_init;
 145
 146         region_12kb->max_lkey_id = max_lkey_id;
 147         aregion->priv = region_12kb;
 148
 149         return 0;
 150
 151 err_rhashtable_init:
 152         kfree(region_12kb->used_lkey_id);
 153 err_used_lkey_id_alloc:
 154         kfree(region_12kb);
 155         return err;
 156 }
 157
 158 static void
 159 mlxsw_sp_acl_atcam_region_12kb_fini(struct mlxsw_sp_acl_atcam_region *aregion)
 160 {
 161         struct mlxsw_sp_acl_atcam_region_12kb *region_12kb = aregion->priv;
 162
 163         rhashtable_destroy(&region_12kb->lkey_ht);
 164         kfree(region_12kb->used_lkey_id);
 165         kfree(region_12kb);
 166 }
 167
 168 static struct mlxsw_sp_acl_atcam_lkey_id *
 169 mlxsw_sp_acl_atcam_lkey_id_create(struct mlxsw_sp_acl_atcam_region *aregion,
 170                                   struct mlxsw_sp_acl_atcam_lkey_id_ht_key *ht_key)
 171 {
 172         struct mlxsw_sp_acl_atcam_region_12kb *region_12kb = aregion->priv;
 173         struct mlxsw_sp_acl_atcam_lkey_id *lkey_id;
 174         u32 id;
 175         int err;
 176
 177         id = find_first_zero_bit(region_12kb->used_lkey_id,
 178                                  region_12kb->max_lkey_id);
 179         if (id < region_12kb->max_lkey_id)
 180                 __set_bit(id, region_12kb->used_lkey_id);
 181         else
 182                 return ERR_PTR(-ENOBUFS);
 183
 184         lkey_id = kzalloc(sizeof(*lkey_id), GFP_KERNEL);
 185         if (!lkey_id) {
 186                 err = -ENOMEM;
 187                 goto err_lkey_id_alloc;
 188         }
 189
 190         lkey_id->id = id;
 191         memcpy(&lkey_id->ht_key, ht_key, sizeof(*ht_key));
 192         refcount_set(&lkey_id->refcnt, 1);
 193
 194         err = rhashtable_insert_fast(&region_12kb->lkey_ht,
 195                                      &lkey_id->ht_node,
 196                                      mlxsw_sp_acl_atcam_lkey_id_ht_params);
 197         if (err)
 198                 goto err_rhashtable_insert;
 199
 200         return lkey_id;
 201
 202 err_rhashtable_insert:
 203         kfree(lkey_id);
 204 err_lkey_id_alloc:
 205         __clear_bit(id, region_12kb->used_lkey_id);
 206         return ERR_PTR(err);
 207 }
 208
 209 static void
 210 mlxsw_sp_acl_atcam_lkey_id_destroy(struct mlxsw_sp_acl_atcam_region *aregion,
 211                                    struct mlxsw_sp_acl_atcam_lkey_id *lkey_id)
 212 {
 213         struct mlxsw_sp_acl_atcam_region_12kb *region_12kb = aregion->priv;
 214         u32 id = lkey_id->id;
 215
 216         rhashtable_remove_fast(&region_12kb->lkey_ht, &lkey_id->ht_node,
 217                                mlxsw_sp_acl_atcam_lkey_id_ht_params);
 218         kfree(lkey_id);
 219         __clear_bit(id, region_12kb->used_lkey_id);
 220 }
 221
 222 static struct mlxsw_sp_acl_atcam_lkey_id *
 223 mlxsw_sp_acl_atcam_12kb_lkey_id_get(struct mlxsw_sp_acl_atcam_region *aregion,
 224                                     char *enc_key, u8 erp_id)
 225 {
 226         struct mlxsw_sp_acl_atcam_region_12kb *region_12kb = aregion->priv;
 227         struct mlxsw_sp_acl_tcam_region *region = aregion->region;
 228         struct mlxsw_sp_acl_atcam_lkey_id_ht_key ht_key = {{ 0 } };
 229         struct mlxsw_sp *mlxsw_sp = region->mlxsw_sp;
 230         struct mlxsw_afk *afk = mlxsw_sp_acl_afk(mlxsw_sp->acl);
 231         struct mlxsw_sp_acl_atcam_lkey_id *lkey_id;
 232
 233         memcpy(ht_key.enc_key, enc_key, sizeof(ht_key.enc_key));
 234         mlxsw_afk_clear(afk, ht_key.enc_key,
 235                         MLXSW_SP_ACL_ATCAM_LKEY_ID_BLOCK_CLEAR_START,
 236                         MLXSW_SP_ACL_ATCAM_LKEY_ID_BLOCK_CLEAR_END);
 237         ht_key.erp_id = erp_id;
 238         lkey_id = rhashtable_lookup_fast(&region_12kb->lkey_ht, &ht_key,
 239                                          mlxsw_sp_acl_atcam_lkey_id_ht_params);
 240         if (lkey_id) {
 241                 refcount_inc(&lkey_id->refcnt);
 242                 return lkey_id;
 243         }
 244
 245         return mlxsw_sp_acl_atcam_lkey_id_create(aregion, &ht_key);
 246 }
 247
 248 static void
 249 mlxsw_sp_acl_atcam_12kb_lkey_id_put(struct mlxsw_sp_acl_atcam_region *aregion,
 250                                     struct mlxsw_sp_acl_atcam_lkey_id *lkey_id)
 251 {
 252         if (refcount_dec_and_test(&lkey_id->refcnt))
 253                 mlxsw_sp_acl_atcam_lkey_id_destroy(aregion, lkey_id);
 254 }
 255
 256 static const struct mlxsw_sp_acl_atcam_region_ops
 257 mlxsw_sp_acl_atcam_region_12kb_ops = {
 258         .init           = mlxsw_sp_acl_atcam_region_12kb_init,
 259         .fini           = mlxsw_sp_acl_atcam_region_12kb_fini,
 260         .lkey_id_get    = mlxsw_sp_acl_atcam_12kb_lkey_id_get,
 261         .lkey_id_put    = mlxsw_sp_acl_atcam_12kb_lkey_id_put,
 262 };
 263
 264 static const struct mlxsw_sp_acl_atcam_region_ops *
 265 mlxsw_sp_acl_atcam_region_ops_arr[] = {
 266         [MLXSW_SP_ACL_ATCAM_REGION_TYPE_2KB]    =
 267                 &mlxsw_sp_acl_atcam_region_generic_ops,
 268         [MLXSW_SP_ACL_ATCAM_REGION_TYPE_4KB]    =
 269                 &mlxsw_sp_acl_atcam_region_generic_ops,
 270         [MLXSW_SP_ACL_ATCAM_REGION_TYPE_8KB]    =
 271                 &mlxsw_sp_acl_atcam_region_generic_ops,
 272         [MLXSW_SP_ACL_ATCAM_REGION_TYPE_12KB]   =
 273                 &mlxsw_sp_acl_atcam_region_12kb_ops,
 274 };
 275
 276 int mlxsw_sp_acl_atcam_region_associate(struct mlxsw_sp *mlxsw_sp,
 277                                         u16 region_id)
 278 {
 279         char perar_pl[MLXSW_REG_PERAR_LEN];
 280         /* For now, just assume that every region has 12 key blocks */
 281         u16 hw_region = region_id * 3;
 282         u64 max_regions;
 283
 284         max_regions = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_MAX_REGIONS);
 285         if (hw_region >= max_regions)
 286                 return -ENOBUFS;
 287
 288         mlxsw_reg_perar_pack(perar_pl, region_id, hw_region);
 289         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(perar), perar_pl);
 290 }
 291
 292 static void
 293 mlxsw_sp_acl_atcam_region_type_init(struct mlxsw_sp_acl_atcam_region *aregion)
 294 {
 295         struct mlxsw_sp_acl_tcam_region *region = aregion->region;
 296         enum mlxsw_sp_acl_atcam_region_type region_type;
 297         unsigned int blocks_count;
 298
 299         /* We already know the blocks count can not exceed the maximum
 300          * blocks count.
 301          */
 302         blocks_count = mlxsw_afk_key_info_blocks_count_get(region->key_info);
 303         if (blocks_count <= 2)
 304                 region_type = MLXSW_SP_ACL_ATCAM_REGION_TYPE_2KB;
 305         else if (blocks_count <= 4)
 306                 region_type = MLXSW_SP_ACL_ATCAM_REGION_TYPE_4KB;
 307         else if (blocks_count <= 8)
 308                 region_type = MLXSW_SP_ACL_ATCAM_REGION_TYPE_8KB;
 309         else
 310                 region_type = MLXSW_SP_ACL_ATCAM_REGION_TYPE_12KB;
 311
 312         aregion->type = region_type;
 313         aregion->ops = mlxsw_sp_acl_atcam_region_ops_arr[region_type];
 314 }
 315
 316 int
 317 mlxsw_sp_acl_atcam_region_init(struct mlxsw_sp *mlxsw_sp,
 318                                struct mlxsw_sp_acl_atcam *atcam,
 319                                struct mlxsw_sp_acl_atcam_region *aregion,
 320                                struct mlxsw_sp_acl_tcam_region *region,
 321                                void *hints_priv,
 322                                const struct mlxsw_sp_acl_ctcam_region_ops *ops)
 323 {
 324         int err;
 325
 326         aregion->region = region;
 327         aregion->atcam = atcam;
 328         mlxsw_sp_acl_atcam_region_type_init(aregion);
 329         INIT_LIST_HEAD(&aregion->entries_list);
 330
 331         err = rhashtable_init(&aregion->entries_ht,
 332                               &mlxsw_sp_acl_atcam_entries_ht_params);
 333         if (err)
 334                 return err;
 335         err = aregion->ops->init(aregion);
 336         if (err)
 337                 goto err_ops_init;
 338         err = mlxsw_sp_acl_erp_region_init(aregion, hints_priv);
 339         if (err)
 340                 goto err_erp_region_init;
 341         err = mlxsw_sp_acl_ctcam_region_init(mlxsw_sp, &aregion->cregion,
 342                                              region, ops);
 343         if (err)
 344                 goto err_ctcam_region_init;
 345
 346         return 0;
 347
 348 err_ctcam_region_init:
 349         mlxsw_sp_acl_erp_region_fini(aregion);
 350 err_erp_region_init:
 351         aregion->ops->fini(aregion);
 352 err_ops_init:
 353         rhashtable_destroy(&aregion->entries_ht);
 354         return err;
 355 }
 356
 357 void mlxsw_sp_acl_atcam_region_fini(struct mlxsw_sp_acl_atcam_region *aregion)
 358 {
 359         mlxsw_sp_acl_ctcam_region_fini(&aregion->cregion);
 360         mlxsw_sp_acl_erp_region_fini(aregion);
 361         aregion->ops->fini(aregion);
 362         rhashtable_destroy(&aregion->entries_ht);
 363         WARN_ON(!list_empty(&aregion->entries_list));
 364 }
 365
 366 void mlxsw_sp_acl_atcam_chunk_init(struct mlxsw_sp_acl_atcam_region *aregion,
 367                                    struct mlxsw_sp_acl_atcam_chunk *achunk,
 368                                    unsigned int priority)
 369 {
 370         mlxsw_sp_acl_ctcam_chunk_init(&aregion->cregion, &achunk->cchunk,
 371                                       priority);
 372 }
 373
 374 void mlxsw_sp_acl_atcam_chunk_fini(struct mlxsw_sp_acl_atcam_chunk *achunk)
 375 {
 376         mlxsw_sp_acl_ctcam_chunk_fini(&achunk->cchunk);
 377 }
 378
 379 static int
 380 mlxsw_sp_acl_atcam_region_entry_insert(struct mlxsw_sp *mlxsw_sp,
 381                                        struct mlxsw_sp_acl_atcam_region *aregion,
 382                                        struct mlxsw_sp_acl_atcam_entry *aentry,
 383                                        struct mlxsw_sp_acl_rule_info *rulei)
 384 {
 385         struct mlxsw_sp_acl_tcam_region *region = aregion->region;
 386         u8 erp_id = mlxsw_sp_acl_erp_mask_erp_id(aentry->erp_mask);
 387         struct mlxsw_sp_acl_atcam_lkey_id *lkey_id;
 388         char ptce3_pl[MLXSW_REG_PTCE3_LEN];
 389         u32 kvdl_index, priority;
 390         int err;
 391
 392         err = mlxsw_sp_acl_tcam_priority_get(mlxsw_sp, rulei, &priority, true);
 393         if (err)
 394                 return err;
 395
 396         lkey_id = aregion->ops->lkey_id_get(aregion, aentry->enc_key, erp_id);
 397         if (IS_ERR(lkey_id))
 398                 return PTR_ERR(lkey_id);
 399         aentry->lkey_id = lkey_id;
 400
 401         kvdl_index = mlxsw_afa_block_first_kvdl_index(rulei->act_block);
 402         mlxsw_reg_ptce3_pack(ptce3_pl, true, MLXSW_REG_PTCE3_OP_WRITE_WRITE,
 403                              priority, region->tcam_region_info,
 404                              aentry->enc_key, erp_id,
 405                              aentry->delta_info.start,
 406                              aentry->delta_info.mask,
 407                              aentry->delta_info.value,
 408                              refcount_read(&lkey_id->refcnt) != 1, lkey_id->id,
 409                              kvdl_index);
 410         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ptce3), ptce3_pl);
 411         if (err)
 412                 goto err_ptce3_write;
 413
 414         return 0;
 415
 416 err_ptce3_write:
 417         aregion->ops->lkey_id_put(aregion, lkey_id);
 418         return err;
 419 }
 420
 421 static void
 422 mlxsw_sp_acl_atcam_region_entry_remove(struct mlxsw_sp *mlxsw_sp,
 423                                        struct mlxsw_sp_acl_atcam_region *aregion,
 424                                        struct mlxsw_sp_acl_atcam_entry *aentry)
 425 {
 426         struct mlxsw_sp_acl_atcam_lkey_id *lkey_id = aentry->lkey_id;
 427         struct mlxsw_sp_acl_tcam_region *region = aregion->region;
 428         u8 erp_id = mlxsw_sp_acl_erp_mask_erp_id(aentry->erp_mask);
 429         char ptce3_pl[MLXSW_REG_PTCE3_LEN];
 430
 431         mlxsw_reg_ptce3_pack(ptce3_pl, false, MLXSW_REG_PTCE3_OP_WRITE_WRITE, 0,
 432                              region->tcam_region_info,
 433                              aentry->enc_key, erp_id,
 434                              aentry->delta_info.start,
 435                              aentry->delta_info.mask,
 436                              aentry->delta_info.value,
 437                              refcount_read(&lkey_id->refcnt) != 1,
 438                              lkey_id->id, 0);
 439         mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ptce3), ptce3_pl);
 440         aregion->ops->lkey_id_put(aregion, lkey_id);
 441 }
 442
 443 static int
 444 mlxsw_sp_acl_atcam_region_entry_action_replace(struct mlxsw_sp *mlxsw_sp,
 445                                                struct mlxsw_sp_acl_atcam_region *aregion,
 446                                                struct mlxsw_sp_acl_atcam_entry *aentry,
 447                                                struct mlxsw_sp_acl_rule_info *rulei)
 448 {
 449         struct mlxsw_sp_acl_atcam_lkey_id *lkey_id = aentry->lkey_id;
 450         u8 erp_id = mlxsw_sp_acl_erp_mask_erp_id(aentry->erp_mask);
 451         struct mlxsw_sp_acl_tcam_region *region = aregion->region;
 452         char ptce3_pl[MLXSW_REG_PTCE3_LEN];
 453         u32 kvdl_index, priority;
 454         int err;
 455
 456         err = mlxsw_sp_acl_tcam_priority_get(mlxsw_sp, rulei, &priority, true);
 457         if (err)
 458                 return err;
 459         kvdl_index = mlxsw_afa_block_first_kvdl_index(rulei->act_block);
 460         mlxsw_reg_ptce3_pack(ptce3_pl, true, MLXSW_REG_PTCE3_OP_WRITE_UPDATE,
 461                              priority, region->tcam_region_info,
 462                              aentry->enc_key, erp_id,
 463                              aentry->delta_info.start,
 464                              aentry->delta_info.mask,
 465                              aentry->delta_info.value,
 466                              refcount_read(&lkey_id->refcnt) != 1, lkey_id->id,
 467                              kvdl_index);
 468         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ptce3), ptce3_pl);
 469 }
 470
 471 static int
 472 __mlxsw_sp_acl_atcam_entry_add(struct mlxsw_sp *mlxsw_sp,
 473                                struct mlxsw_sp_acl_atcam_region *aregion,
 474                                struct mlxsw_sp_acl_atcam_entry *aentry,
 475                                struct mlxsw_sp_acl_rule_info *rulei)
 476 {
 477         struct mlxsw_sp_acl_tcam_region *region = aregion->region;
 478         char mask[MLXSW_REG_PTCEX_FLEX_KEY_BLOCKS_LEN] = { 0 };
 479         struct mlxsw_afk *afk = mlxsw_sp_acl_afk(mlxsw_sp->acl);
 480         const struct mlxsw_sp_acl_erp_delta *delta;
 481         struct mlxsw_sp_acl_erp_mask *erp_mask;
 482         int err;
 483
 484         mlxsw_afk_encode(afk, region->key_info, &rulei->values,
 485                          aentry->ht_key.full_enc_key, mask);
 486
 487         erp_mask = mlxsw_sp_acl_erp_mask_get(aregion, mask, false);
 488         if (IS_ERR(erp_mask))
 489                 return PTR_ERR(erp_mask);
 490         aentry->erp_mask = erp_mask;
 491         aentry->ht_key.erp_id = mlxsw_sp_acl_erp_mask_erp_id(erp_mask);
 492         memcpy(aentry->enc_key, aentry->ht_key.full_enc_key,
 493                sizeof(aentry->enc_key));
 494
 495         /* Compute all needed delta information and clear the delta bits
 496          * from the encrypted key.
 497          */
 498         delta = mlxsw_sp_acl_erp_delta(aentry->erp_mask);
 499         aentry->delta_info.start = mlxsw_sp_acl_erp_delta_start(delta);
 500         aentry->delta_info.mask = mlxsw_sp_acl_erp_delta_mask(delta);
 501         aentry->delta_info.value =
 502                 mlxsw_sp_acl_erp_delta_value(delta,
 503                                              aentry->ht_key.full_enc_key);
 504         mlxsw_sp_acl_erp_delta_clear(delta, aentry->enc_key);
 505
 506         /* Add rule to the list of A-TCAM rules, assuming this
 507          * rule is intended to A-TCAM. In case this rule does
 508          * not fit into A-TCAM it will be removed from the list.
 509          */
 510         list_add(&aentry->list, &aregion->entries_list);
 511
 512         /* We can't insert identical rules into the A-TCAM, so fail and
 513          * let the rule spill into C-TCAM
 514          */
 515         err = rhashtable_lookup_insert_fast(&aregion->entries_ht,
 516                                             &aentry->ht_node,
 517                                             mlxsw_sp_acl_atcam_entries_ht_params);
 518         if (err)
 519                 goto err_rhashtable_insert;
 520
 521         /* Bloom filter must be updated here, before inserting the rule into
 522          * the A-TCAM.
 523          */
 524         err = mlxsw_sp_acl_erp_bf_insert(mlxsw_sp, aregion, erp_mask, aentry);
 525         if (err)
 526                 goto err_bf_insert;
 527
 528         err = mlxsw_sp_acl_atcam_region_entry_insert(mlxsw_sp, aregion, aentry,
 529                                                      rulei);
 530         if (err)
 531                 goto err_rule_insert;
 532
 533         return 0;
 534
 535 err_rule_insert:
 536         mlxsw_sp_acl_erp_bf_remove(mlxsw_sp, aregion, erp_mask, aentry);
 537 err_bf_insert:
 538         rhashtable_remove_fast(&aregion->entries_ht, &aentry->ht_node,
 539                                mlxsw_sp_acl_atcam_entries_ht_params);
 540 err_rhashtable_insert:
 541         list_del(&aentry->list);
 542         mlxsw_sp_acl_erp_mask_put(aregion, erp_mask);
 543         return err;
 544 }
 545
 546 static void
 547 __mlxsw_sp_acl_atcam_entry_del(struct mlxsw_sp *mlxsw_sp,
 548                                struct mlxsw_sp_acl_atcam_region *aregion,
 549                                struct mlxsw_sp_acl_atcam_entry *aentry)
 550 {
 551         mlxsw_sp_acl_atcam_region_entry_remove(mlxsw_sp, aregion, aentry);
 552         mlxsw_sp_acl_erp_bf_remove(mlxsw_sp, aregion, aentry->erp_mask, aentry);
 553         rhashtable_remove_fast(&aregion->entries_ht, &aentry->ht_node,
 554                                mlxsw_sp_acl_atcam_entries_ht_params);
 555         list_del(&aentry->list);
 556         mlxsw_sp_acl_erp_mask_put(aregion, aentry->erp_mask);
 557 }
 558
 559 static int
 560 __mlxsw_sp_acl_atcam_entry_action_replace(struct mlxsw_sp *mlxsw_sp,
 561                                           struct mlxsw_sp_acl_atcam_region *aregion,
 562                                           struct mlxsw_sp_acl_atcam_entry *aentry,
 563                                           struct mlxsw_sp_acl_rule_info *rulei)
 564 {
 565         return mlxsw_sp_acl_atcam_region_entry_action_replace(mlxsw_sp, aregion,
 566                                                               aentry, rulei);
 567 }
 568
 569 int mlxsw_sp_acl_atcam_entry_add(struct mlxsw_sp *mlxsw_sp,
 570                                  struct mlxsw_sp_acl_atcam_region *aregion,
 571                                  struct mlxsw_sp_acl_atcam_chunk *achunk,
 572                                  struct mlxsw_sp_acl_atcam_entry *aentry,
 573                                  struct mlxsw_sp_acl_rule_info *rulei)
 574 {
 575         int err;
 576
 577         err = __mlxsw_sp_acl_atcam_entry_add(mlxsw_sp, aregion, aentry, rulei);
 578         if (!err)
 579                 return 0;
 580
 581         /* It is possible we failed to add the rule to the A-TCAM due to
 582          * exceeded number of masks. Try to spill into C-TCAM.
 583          */
 584         trace_mlxsw_sp_acl_atcam_entry_add_ctcam_spill(mlxsw_sp, aregion);
 585         err = mlxsw_sp_acl_ctcam_entry_add(mlxsw_sp, &aregion->cregion,
 586                                            &achunk->cchunk, &aentry->centry,
 587                                            rulei, true);
 588         if (!err)
 589                 return 0;
 590
 591         return err;
 592 }
 593
 594 void mlxsw_sp_acl_atcam_entry_del(struct mlxsw_sp *mlxsw_sp,
 595                                   struct mlxsw_sp_acl_atcam_region *aregion,
 596                                   struct mlxsw_sp_acl_atcam_chunk *achunk,
 597                                   struct mlxsw_sp_acl_atcam_entry *aentry)
 598 {
 599         if (mlxsw_sp_acl_atcam_is_centry(aentry))
 600                 mlxsw_sp_acl_ctcam_entry_del(mlxsw_sp, &aregion->cregion,
 601                                              &achunk->cchunk, &aentry->centry);
 602         else
 603                 __mlxsw_sp_acl_atcam_entry_del(mlxsw_sp, aregion, aentry);
 604 }
 605
 606 int
 607 mlxsw_sp_acl_atcam_entry_action_replace(struct mlxsw_sp *mlxsw_sp,
 608                                         struct mlxsw_sp_acl_atcam_region *aregion,
 609                                         struct mlxsw_sp_acl_atcam_entry *aentry,
 610                                         struct mlxsw_sp_acl_rule_info *rulei)
 611 {
 612         int err;
 613
 614         if (mlxsw_sp_acl_atcam_is_centry(aentry))
 615                 err = mlxsw_sp_acl_ctcam_entry_action_replace(mlxsw_sp,
 616                                                               &aregion->cregion,
 617                                                               &aentry->centry,
 618                                                               rulei);
 619         else
 620                 err = __mlxsw_sp_acl_atcam_entry_action_replace(mlxsw_sp,
 621                                                                 aregion, aentry,
 622                                                                 rulei);
 623
 624         return err;
 625 }
 626
 627 int mlxsw_sp_acl_atcam_init(struct mlxsw_sp *mlxsw_sp,
 628                             struct mlxsw_sp_acl_atcam *atcam)
 629 {
 630         return mlxsw_sp_acl_erps_init(mlxsw_sp, atcam);
 631 }
 632
 633 void mlxsw_sp_acl_atcam_fini(struct mlxsw_sp *mlxsw_sp,
 634                              struct mlxsw_sp_acl_atcam *atcam)
 635 {
 636         mlxsw_sp_acl_erps_fini(mlxsw_sp, atcam);
 637 }
 638
 639 void *
 640 mlxsw_sp_acl_atcam_rehash_hints_get(struct mlxsw_sp_acl_atcam_region *aregion)
 641 {
 642         return mlxsw_sp_acl_erp_rehash_hints_get(aregion);
 643 }
 644
 645 void mlxsw_sp_acl_atcam_rehash_hints_put(void *hints_priv)
 646 {
 647         mlxsw_sp_acl_erp_rehash_hints_put(hints_priv);
 648 }