net/sunrpc/auth.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * linux/net/sunrpc/auth.c
   4  *
   5  * Generic RPC client authentication API.
   6  *
   7  * Copyright (C) 1996, Olaf Kirch <okir@monad.swb.de>
   8  */
   9
  10 #include <linux/types.h>
  11 #include <linux/sched.h>
  12 #include <linux/cred.h>
  13 #include <linux/module.h>
  14 #include <linux/slab.h>
  15 #include <linux/errno.h>
  16 #include <linux/hash.h>
  17 #include <linux/sunrpc/clnt.h>
  18 #include <linux/sunrpc/gss_api.h>
  19 #include <linux/spinlock.h>
  20
  21 #include <trace/events/sunrpc.h>
  22
  23 #define RPC_CREDCACHE_DEFAULT_HASHBITS  (4)
  24 struct rpc_cred_cache {
  25         struct hlist_head       *hashtable;
  26         unsigned int            hashbits;
  27         spinlock_t              lock;
  28 };
  29
  30 static unsigned int auth_hashbits = RPC_CREDCACHE_DEFAULT_HASHBITS;
  31
  32 static const struct rpc_authops __rcu *auth_flavors[RPC_AUTH_MAXFLAVOR] = {
  33         [RPC_AUTH_NULL] = (const struct rpc_authops __force __rcu *)&authnull_ops,
  34         [RPC_AUTH_UNIX] = (const struct rpc_authops __force __rcu *)&authunix_ops,
  35         [RPC_AUTH_TLS]  = (const struct rpc_authops __force __rcu *)&authtls_ops,
  36 };
  37
  38 static LIST_HEAD(cred_unused);
  39 static unsigned long number_cred_unused;
  40
  41 static struct cred machine_cred = {
  42         .usage = ATOMIC_INIT(1),
  43 };
  44
  45 /*
  46  * Return the machine_cred pointer to be used whenever
  47  * the a generic machine credential is needed.
  48  */
  49 const struct cred *rpc_machine_cred(void)
  50 {
  51         return &machine_cred;
  52 }
  53 EXPORT_SYMBOL_GPL(rpc_machine_cred);
  54
  55 #define MAX_HASHTABLE_BITS (14)
  56 static int param_set_hashtbl_sz(const char *val, const struct kernel_param *kp)
  57 {
  58         unsigned long num;
  59         unsigned int nbits;
  60         int ret;
  61
  62         if (!val)
  63                 goto out_inval;
  64         ret = kstrtoul(val, 0, &num);
  65         if (ret)
  66                 goto out_inval;
  67         nbits = fls(num - 1);
  68         if (nbits > MAX_HASHTABLE_BITS || nbits < 2)
  69                 goto out_inval;
  70         *(unsigned int *)kp->arg = nbits;
  71         return 0;
  72 out_inval:
  73         return -EINVAL;
  74 }
  75
  76 static int param_get_hashtbl_sz(char *buffer, const struct kernel_param *kp)
  77 {
  78         unsigned int nbits;
  79
  80         nbits = *(unsigned int *)kp->arg;
  81         return sprintf(buffer, "%u\n", 1U << nbits);
  82 }
  83
  84 #define param_check_hashtbl_sz(name, p) __param_check(name, p, unsigned int);
  85
  86 static const struct kernel_param_ops param_ops_hashtbl_sz = {
  87         .set = param_set_hashtbl_sz,
  88         .get = param_get_hashtbl_sz,
  89 };
  90
  91 module_param_named(auth_hashtable_size, auth_hashbits, hashtbl_sz, 0644);
  92 MODULE_PARM_DESC(auth_hashtable_size, "RPC credential cache hashtable size");
  93
  94 static unsigned long auth_max_cred_cachesize = ULONG_MAX;
  95 module_param(auth_max_cred_cachesize, ulong, 0644);
  96 MODULE_PARM_DESC(auth_max_cred_cachesize, "RPC credential maximum total cache size");
  97
  98 static u32
  99 pseudoflavor_to_flavor(u32 flavor) {
 100         if (flavor > RPC_AUTH_MAXFLAVOR)
 101                 return RPC_AUTH_GSS;
 102         return flavor;
 103 }
 104
 105 int
 106 rpcauth_register(const struct rpc_authops *ops)
 107 {
 108         const struct rpc_authops *old;
 109         rpc_authflavor_t flavor;
 110
 111         if ((flavor = ops->au_flavor) >= RPC_AUTH_MAXFLAVOR)
 112                 return -EINVAL;
 113         old = cmpxchg((const struct rpc_authops ** __force)&auth_flavors[flavor], NULL, ops);
 114         if (old == NULL || old == ops)
 115                 return 0;
 116         return -EPERM;
 117 }
 118 EXPORT_SYMBOL_GPL(rpcauth_register);
 119
 120 int
 121 rpcauth_unregister(const struct rpc_authops *ops)
 122 {
 123         const struct rpc_authops *old;
 124         rpc_authflavor_t flavor;
 125
 126         if ((flavor = ops->au_flavor) >= RPC_AUTH_MAXFLAVOR)
 127                 return -EINVAL;
 128
 129         old = cmpxchg((const struct rpc_authops ** __force)&auth_flavors[flavor], ops, NULL);
 130         if (old == ops || old == NULL)
 131                 return 0;
 132         return -EPERM;
 133 }
 134 EXPORT_SYMBOL_GPL(rpcauth_unregister);
 135
 136 static const struct rpc_authops *
 137 rpcauth_get_authops(rpc_authflavor_t flavor)
 138 {
 139         const struct rpc_authops *ops;
 140
 141         if (flavor >= RPC_AUTH_MAXFLAVOR)
 142                 return NULL;
 143
 144         rcu_read_lock();
 145         ops = rcu_dereference(auth_flavors[flavor]);
 146         if (ops == NULL) {
 147                 rcu_read_unlock();
 148                 request_module("rpc-auth-%u", flavor);
 149                 rcu_read_lock();
 150                 ops = rcu_dereference(auth_flavors[flavor]);
 151                 if (ops == NULL)
 152                         goto out;
 153         }
 154         if (!try_module_get(ops->owner))
 155                 ops = NULL;
 156 out:
 157         rcu_read_unlock();
 158         return ops;
 159 }
 160
 161 static void
 162 rpcauth_put_authops(const struct rpc_authops *ops)
 163 {
 164         module_put(ops->owner);
 165 }
 166
 167 /**
 168  * rpcauth_get_pseudoflavor - check if security flavor is supported
 169  * @flavor: a security flavor
 170  * @info: a GSS mech OID, quality of protection, and service value
 171  *
 172  * Verifies that an appropriate kernel module is available or already loaded.
 173  * Returns an equivalent pseudoflavor, or RPC_AUTH_MAXFLAVOR if "flavor" is
 174  * not supported locally.
 175  */
 176 rpc_authflavor_t
 177 rpcauth_get_pseudoflavor(rpc_authflavor_t flavor, struct rpcsec_gss_info *info)
 178 {
 179         const struct rpc_authops *ops = rpcauth_get_authops(flavor);
 180         rpc_authflavor_t pseudoflavor;
 181
 182         if (!ops)
 183                 return RPC_AUTH_MAXFLAVOR;
 184         pseudoflavor = flavor;
 185         if (ops->info2flavor != NULL)
 186                 pseudoflavor = ops->info2flavor(info);
 187
 188         rpcauth_put_authops(ops);
 189         return pseudoflavor;
 190 }
 191 EXPORT_SYMBOL_GPL(rpcauth_get_pseudoflavor);
 192
 193 /**
 194  * rpcauth_get_gssinfo - find GSS tuple matching a GSS pseudoflavor
 195  * @pseudoflavor: GSS pseudoflavor to match
 196  * @info: rpcsec_gss_info structure to fill in
 197  *
 198  * Returns zero and fills in "info" if pseudoflavor matches a
 199  * supported mechanism.
 200  */
 201 int
 202 rpcauth_get_gssinfo(rpc_authflavor_t pseudoflavor, struct rpcsec_gss_info *info)
 203 {
 204         rpc_authflavor_t flavor = pseudoflavor_to_flavor(pseudoflavor);
 205         const struct rpc_authops *ops;
 206         int result;
 207
 208         ops = rpcauth_get_authops(flavor);
 209         if (ops == NULL)
 210                 return -ENOENT;
 211
 212         result = -ENOENT;
 213         if (ops->flavor2info != NULL)
 214                 result = ops->flavor2info(pseudoflavor, info);
 215
 216         rpcauth_put_authops(ops);
 217         return result;
 218 }
 219 EXPORT_SYMBOL_GPL(rpcauth_get_gssinfo);
 220
 221 struct rpc_auth *
 222 rpcauth_create(const struct rpc_auth_create_args *args, struct rpc_clnt *clnt)
 223 {
 224         struct rpc_auth *auth = ERR_PTR(-EINVAL);
 225         const struct rpc_authops *ops;
 226         u32 flavor = pseudoflavor_to_flavor(args->pseudoflavor);
 227
 228         ops = rpcauth_get_authops(flavor);
 229         if (ops == NULL)
 230                 goto out;
 231
 232         auth = ops->create(args, clnt);
 233
 234         rpcauth_put_authops(ops);
 235         if (IS_ERR(auth))
 236                 return auth;
 237         if (clnt->cl_auth)
 238                 rpcauth_release(clnt->cl_auth);
 239         clnt->cl_auth = auth;
 240
 241 out:
 242         return auth;
 243 }
 244 EXPORT_SYMBOL_GPL(rpcauth_create);
 245
 246 void
 247 rpcauth_release(struct rpc_auth *auth)
 248 {
 249         if (!refcount_dec_and_test(&auth->au_count))
 250                 return;
 251         auth->au_ops->destroy(auth);
 252 }
 253
 254 static DEFINE_SPINLOCK(rpc_credcache_lock);
 255
 256 /*
 257  * On success, the caller is responsible for freeing the reference
 258  * held by the hashtable
 259  */
 260 static bool
 261 rpcauth_unhash_cred_locked(struct rpc_cred *cred)
 262 {
 263         if (!test_and_clear_bit(RPCAUTH_CRED_HASHED, &cred->cr_flags))
 264                 return false;
 265         hlist_del_rcu(&cred->cr_hash);
 266         return true;
 267 }
 268
 269 static bool
 270 rpcauth_unhash_cred(struct rpc_cred *cred)
 271 {
 272         spinlock_t *cache_lock;
 273         bool ret;
 274
 275         if (!test_bit(RPCAUTH_CRED_HASHED, &cred->cr_flags))
 276                 return false;
 277         cache_lock = &cred->cr_auth->au_credcache->lock;
 278         spin_lock(cache_lock);
 279         ret = rpcauth_unhash_cred_locked(cred);
 280         spin_unlock(cache_lock);
 281         return ret;
 282 }
 283
 284 /*
 285  * Initialize RPC credential cache
 286  */
 287 int
 288 rpcauth_init_credcache(struct rpc_auth *auth)
 289 {
 290         struct rpc_cred_cache *new;
 291         unsigned int hashsize;
 292
 293         new = kmalloc(sizeof(*new), GFP_KERNEL);
 294         if (!new)
 295                 goto out_nocache;
 296         new->hashbits = auth_hashbits;
 297         hashsize = 1U << new->hashbits;
 298         new->hashtable = kcalloc(hashsize, sizeof(new->hashtable[0]), GFP_KERNEL);
 299         if (!new->hashtable)
 300                 goto out_nohashtbl;
 301         spin_lock_init(&new->lock);
 302         auth->au_credcache = new;
 303         return 0;
 304 out_nohashtbl:
 305         kfree(new);
 306 out_nocache:
 307         return -ENOMEM;
 308 }
 309 EXPORT_SYMBOL_GPL(rpcauth_init_credcache);
 310
 311 char *
 312 rpcauth_stringify_acceptor(struct rpc_cred *cred)
 313 {
 314         if (!cred->cr_ops->crstringify_acceptor)
 315                 return NULL;
 316         return cred->cr_ops->crstringify_acceptor(cred);
 317 }
 318 EXPORT_SYMBOL_GPL(rpcauth_stringify_acceptor);
 319
 320 /*
 321  * Destroy a list of credentials
 322  */
 323 static inline
 324 void rpcauth_destroy_credlist(struct list_head *head)
 325 {
 326         struct rpc_cred *cred;
 327
 328         while (!list_empty(head)) {
 329                 cred = list_entry(head->next, struct rpc_cred, cr_lru);
 330                 list_del_init(&cred->cr_lru);
 331                 put_rpccred(cred);
 332         }
 333 }
 334
 335 static void
 336 rpcauth_lru_add_locked(struct rpc_cred *cred)
 337 {
 338         if (!list_empty(&cred->cr_lru))
 339                 return;
 340         number_cred_unused++;
 341         list_add_tail(&cred->cr_lru, &cred_unused);
 342 }
 343
 344 static void
 345 rpcauth_lru_add(struct rpc_cred *cred)
 346 {
 347         if (!list_empty(&cred->cr_lru))
 348                 return;
 349         spin_lock(&rpc_credcache_lock);
 350         rpcauth_lru_add_locked(cred);
 351         spin_unlock(&rpc_credcache_lock);
 352 }
 353
 354 static void
 355 rpcauth_lru_remove_locked(struct rpc_cred *cred)
 356 {
 357         if (list_empty(&cred->cr_lru))
 358                 return;
 359         number_cred_unused--;
 360         list_del_init(&cred->cr_lru);
 361 }
 362
 363 static void
 364 rpcauth_lru_remove(struct rpc_cred *cred)
 365 {
 366         if (list_empty(&cred->cr_lru))
 367                 return;
 368         spin_lock(&rpc_credcache_lock);
 369         rpcauth_lru_remove_locked(cred);
 370         spin_unlock(&rpc_credcache_lock);
 371 }
 372
 373 /*
 374  * Clear the RPC credential cache, and delete those credentials
 375  * that are not referenced.
 376  */
 377 void
 378 rpcauth_clear_credcache(struct rpc_cred_cache *cache)
 379 {
 380         LIST_HEAD(free);
 381         struct hlist_head *head;
 382         struct rpc_cred *cred;
 383         unsigned int hashsize = 1U << cache->hashbits;
 384         int             i;
 385
 386         spin_lock(&rpc_credcache_lock);
 387         spin_lock(&cache->lock);
 388         for (i = 0; i < hashsize; i++) {
 389                 head = &cache->hashtable[i];
 390                 while (!hlist_empty(head)) {
 391                         cred = hlist_entry(head->first, struct rpc_cred, cr_hash);
 392                         rpcauth_unhash_cred_locked(cred);
 393                         /* Note: We now hold a reference to cred */
 394                         rpcauth_lru_remove_locked(cred);
 395                         list_add_tail(&cred->cr_lru, &free);
 396                 }
 397         }
 398         spin_unlock(&cache->lock);
 399         spin_unlock(&rpc_credcache_lock);
 400         rpcauth_destroy_credlist(&free);
 401 }
 402
 403 /*
 404  * Destroy the RPC credential cache
 405  */
 406 void
 407 rpcauth_destroy_credcache(struct rpc_auth *auth)
 408 {
 409         struct rpc_cred_cache *cache = auth->au_credcache;
 410
 411         if (cache) {
 412                 auth->au_credcache = NULL;
 413                 rpcauth_clear_credcache(cache);
 414                 kfree(cache->hashtable);
 415                 kfree(cache);
 416         }
 417 }
 418 EXPORT_SYMBOL_GPL(rpcauth_destroy_credcache);
 419
 420
 421 #define RPC_AUTH_EXPIRY_MORATORIUM (60 * HZ)
 422
 423 /*
 424  * Remove stale credentials. Avoid sleeping inside the loop.
 425  */
 426 static long
 427 rpcauth_prune_expired(struct list_head *free, int nr_to_scan)
 428 {
 429         struct rpc_cred *cred, *next;
 430         unsigned long expired = jiffies - RPC_AUTH_EXPIRY_MORATORIUM;
 431         long freed = 0;
 432
 433         list_for_each_entry_safe(cred, next, &cred_unused, cr_lru) {
 434
 435                 if (nr_to_scan-- == 0)
 436                         break;
 437                 if (refcount_read(&cred->cr_count) > 1) {
 438                         rpcauth_lru_remove_locked(cred);
 439                         continue;
 440                 }
 441                 /*
 442                  * Enforce a 60 second garbage collection moratorium
 443                  * Note that the cred_unused list must be time-ordered.
 444                  */
 445                 if (time_in_range(cred->cr_expire, expired, jiffies))
 446                         continue;
 447                 if (!rpcauth_unhash_cred(cred))
 448                         continue;
 449
 450                 rpcauth_lru_remove_locked(cred);
 451                 freed++;
 452                 list_add_tail(&cred->cr_lru, free);
 453         }
 454         return freed ? freed : SHRINK_STOP;
 455 }
 456
 457 static unsigned long
 458 rpcauth_cache_do_shrink(int nr_to_scan)
 459 {
 460         LIST_HEAD(free);
 461         unsigned long freed;
 462
 463         spin_lock(&rpc_credcache_lock);
 464         freed = rpcauth_prune_expired(&free, nr_to_scan);
 465         spin_unlock(&rpc_credcache_lock);
 466         rpcauth_destroy_credlist(&free);
 467
 468         return freed;
 469 }
 470
 471 /*
 472  * Run memory cache shrinker.
 473  */
 474 static unsigned long
 475 rpcauth_cache_shrink_scan(struct shrinker *shrink, struct shrink_control *sc)
 476
 477 {
 478         if ((sc->gfp_mask & GFP_KERNEL) != GFP_KERNEL)
 479                 return SHRINK_STOP;
 480
 481         /* nothing left, don't come back */
 482         if (list_empty(&cred_unused))
 483                 return SHRINK_STOP;
 484
 485         return rpcauth_cache_do_shrink(sc->nr_to_scan);
 486 }
 487
 488 static unsigned long
 489 rpcauth_cache_shrink_count(struct shrinker *shrink, struct shrink_control *sc)
 490
 491 {
 492         return number_cred_unused * sysctl_vfs_cache_pressure / 100;
 493 }
 494
 495 static void
 496 rpcauth_cache_enforce_limit(void)
 497 {
 498         unsigned long diff;
 499         unsigned int nr_to_scan;
 500
 501         if (number_cred_unused <= auth_max_cred_cachesize)
 502                 return;
 503         diff = number_cred_unused - auth_max_cred_cachesize;
 504         nr_to_scan = 100;
 505         if (diff < nr_to_scan)
 506                 nr_to_scan = diff;
 507         rpcauth_cache_do_shrink(nr_to_scan);
 508 }
 509
 510 /*
 511  * Look up a process' credentials in the authentication cache
 512  */
 513 struct rpc_cred *
 514 rpcauth_lookup_credcache(struct rpc_auth *auth, struct auth_cred * acred,
 515                 int flags, gfp_t gfp)
 516 {
 517         LIST_HEAD(free);
 518         struct rpc_cred_cache *cache = auth->au_credcache;
 519         struct rpc_cred *cred = NULL,
 520                         *entry, *new;
 521         unsigned int nr;
 522
 523         nr = auth->au_ops->hash_cred(acred, cache->hashbits);
 524
 525         rcu_read_lock();
 526         hlist_for_each_entry_rcu(entry, &cache->hashtable[nr], cr_hash) {
 527                 if (!entry->cr_ops->crmatch(acred, entry, flags))
 528                         continue;
 529                 cred = get_rpccred(entry);
 530                 if (cred)
 531                         break;
 532         }
 533         rcu_read_unlock();
 534
 535         if (cred != NULL)
 536                 goto found;
 537
 538         new = auth->au_ops->crcreate(auth, acred, flags, gfp);
 539         if (IS_ERR(new)) {
 540                 cred = new;
 541                 goto out;
 542         }
 543
 544         spin_lock(&cache->lock);
 545         hlist_for_each_entry(entry, &cache->hashtable[nr], cr_hash) {
 546                 if (!entry->cr_ops->crmatch(acred, entry, flags))
 547                         continue;
 548                 cred = get_rpccred(entry);
 549                 if (cred)
 550                         break;
 551         }
 552         if (cred == NULL) {
 553                 cred = new;
 554                 set_bit(RPCAUTH_CRED_HASHED, &cred->cr_flags);
 555                 refcount_inc(&cred->cr_count);
 556                 hlist_add_head_rcu(&cred->cr_hash, &cache->hashtable[nr]);
 557         } else
 558                 list_add_tail(&new->cr_lru, &free);
 559         spin_unlock(&cache->lock);
 560         rpcauth_cache_enforce_limit();
 561 found:
 562         if (test_bit(RPCAUTH_CRED_NEW, &cred->cr_flags) &&
 563             cred->cr_ops->cr_init != NULL &&
 564             !(flags & RPCAUTH_LOOKUP_NEW)) {
 565                 int res = cred->cr_ops->cr_init(auth, cred);
 566                 if (res < 0) {
 567                         put_rpccred(cred);
 568                         cred = ERR_PTR(res);
 569                 }
 570         }
 571         rpcauth_destroy_credlist(&free);
 572 out:
 573         return cred;
 574 }
 575 EXPORT_SYMBOL_GPL(rpcauth_lookup_credcache);
 576
 577 struct rpc_cred *
 578 rpcauth_lookupcred(struct rpc_auth *auth, int flags)
 579 {
 580         struct auth_cred acred;
 581         struct rpc_cred *ret;
 582         const struct cred *cred = current_cred();
 583
 584         memset(&acred, 0, sizeof(acred));
 585         acred.cred = cred;
 586         ret = auth->au_ops->lookup_cred(auth, &acred, flags);
 587         return ret;
 588 }
 589 EXPORT_SYMBOL_GPL(rpcauth_lookupcred);
 590
 591 void
 592 rpcauth_init_cred(struct rpc_cred *cred, const struct auth_cred *acred,
 593                   struct rpc_auth *auth, const struct rpc_credops *ops)
 594 {
 595         INIT_HLIST_NODE(&cred->cr_hash);
 596         INIT_LIST_HEAD(&cred->cr_lru);
 597         refcount_set(&cred->cr_count, 1);
 598         cred->cr_auth = auth;
 599         cred->cr_flags = 0;
 600         cred->cr_ops = ops;
 601         cred->cr_expire = jiffies;
 602         cred->cr_cred = get_cred(acred->cred);
 603 }
 604 EXPORT_SYMBOL_GPL(rpcauth_init_cred);
 605
 606 static struct rpc_cred *
 607 rpcauth_bind_root_cred(struct rpc_task *task, int lookupflags)
 608 {
 609         struct rpc_auth *auth = task->tk_client->cl_auth;
 610         struct auth_cred acred = {
 611                 .cred = get_task_cred(&init_task),
 612         };
 613         struct rpc_cred *ret;
 614
 615         if (RPC_IS_ASYNC(task))
 616                 lookupflags |= RPCAUTH_LOOKUP_ASYNC;
 617         ret = auth->au_ops->lookup_cred(auth, &acred, lookupflags);
 618         put_cred(acred.cred);
 619         return ret;
 620 }
 621
 622 static struct rpc_cred *
 623 rpcauth_bind_machine_cred(struct rpc_task *task, int lookupflags)
 624 {
 625         struct rpc_auth *auth = task->tk_client->cl_auth;
 626         struct auth_cred acred = {
 627                 .principal = task->tk_client->cl_principal,
 628                 .cred = init_task.cred,
 629         };
 630
 631         if (!acred.principal)
 632                 return NULL;
 633         if (RPC_IS_ASYNC(task))
 634                 lookupflags |= RPCAUTH_LOOKUP_ASYNC;
 635         return auth->au_ops->lookup_cred(auth, &acred, lookupflags);
 636 }
 637
 638 static struct rpc_cred *
 639 rpcauth_bind_new_cred(struct rpc_task *task, int lookupflags)
 640 {
 641         struct rpc_auth *auth = task->tk_client->cl_auth;
 642
 643         return rpcauth_lookupcred(auth, lookupflags);
 644 }
 645
 646 static int
 647 rpcauth_bindcred(struct rpc_task *task, const struct cred *cred, int flags)
 648 {
 649         struct rpc_rqst *req = task->tk_rqstp;
 650         struct rpc_cred *new = NULL;
 651         int lookupflags = 0;
 652         struct rpc_auth *auth = task->tk_client->cl_auth;
 653         struct auth_cred acred = {
 654                 .cred = cred,
 655         };
 656
 657         if (flags & RPC_TASK_ASYNC)
 658                 lookupflags |= RPCAUTH_LOOKUP_NEW | RPCAUTH_LOOKUP_ASYNC;
 659         if (task->tk_op_cred)
 660                 /* Task must use exactly this rpc_cred */
 661                 new = get_rpccred(task->tk_op_cred);
 662         else if (cred != NULL && cred != &machine_cred)
 663                 new = auth->au_ops->lookup_cred(auth, &acred, lookupflags);
 664         else if (cred == &machine_cred)
 665                 new = rpcauth_bind_machine_cred(task, lookupflags);
 666
 667         /* If machine cred couldn't be bound, try a root cred */
 668         if (new)
 669                 ;
 670         else if (cred == &machine_cred)
 671                 new = rpcauth_bind_root_cred(task, lookupflags);
 672         else if (flags & RPC_TASK_NULLCREDS)
 673                 new = authnull_ops.lookup_cred(NULL, NULL, 0);
 674         else
 675                 new = rpcauth_bind_new_cred(task, lookupflags);
 676         if (IS_ERR(new))
 677                 return PTR_ERR(new);
 678         put_rpccred(req->rq_cred);
 679         req->rq_cred = new;
 680         return 0;
 681 }
 682
 683 void
 684 put_rpccred(struct rpc_cred *cred)
 685 {
 686         if (cred == NULL)
 687                 return;
 688         rcu_read_lock();
 689         if (refcount_dec_and_test(&cred->cr_count))
 690                 goto destroy;
 691         if (refcount_read(&cred->cr_count) != 1 ||
 692             !test_bit(RPCAUTH_CRED_HASHED, &cred->cr_flags))
 693                 goto out;
 694         if (test_bit(RPCAUTH_CRED_UPTODATE, &cred->cr_flags) != 0) {
 695                 cred->cr_expire = jiffies;
 696                 rpcauth_lru_add(cred);
 697                 /* Race breaker */
 698                 if (unlikely(!test_bit(RPCAUTH_CRED_HASHED, &cred->cr_flags)))
 699                         rpcauth_lru_remove(cred);
 700         } else if (rpcauth_unhash_cred(cred)) {
 701                 rpcauth_lru_remove(cred);
 702                 if (refcount_dec_and_test(&cred->cr_count))
 703                         goto destroy;
 704         }
 705 out:
 706         rcu_read_unlock();
 707         return;
 708 destroy:
 709         rcu_read_unlock();
 710         cred->cr_ops->crdestroy(cred);
 711 }
 712 EXPORT_SYMBOL_GPL(put_rpccred);
 713
 714 /**
 715  * rpcauth_marshcred - Append RPC credential to end of @xdr
 716  * @task: controlling RPC task
 717  * @xdr: xdr_stream containing initial portion of RPC Call header
 718  *
 719  * On success, an appropriate verifier is added to @xdr, @xdr is
 720  * updated to point past the verifier, and zero is returned.
 721  * Otherwise, @xdr is in an undefined state and a negative errno
 722  * is returned.
 723  */
 724 int rpcauth_marshcred(struct rpc_task *task, struct xdr_stream *xdr)
 725 {
 726         const struct rpc_credops *ops = task->tk_rqstp->rq_cred->cr_ops;
 727
 728         return ops->crmarshal(task, xdr);
 729 }
 730
 731 /**
 732  * rpcauth_wrap_req_encode - XDR encode the RPC procedure
 733  * @task: controlling RPC task
 734  * @xdr: stream where on-the-wire bytes are to be marshalled
 735  *
 736  * On success, @xdr contains the encoded and wrapped message.
 737  * Otherwise, @xdr is in an undefined state.
 738  */
 739 int rpcauth_wrap_req_encode(struct rpc_task *task, struct xdr_stream *xdr)
 740 {
 741         kxdreproc_t encode = task->tk_msg.rpc_proc->p_encode;
 742
 743         encode(task->tk_rqstp, xdr, task->tk_msg.rpc_argp);
 744         return 0;
 745 }
 746 EXPORT_SYMBOL_GPL(rpcauth_wrap_req_encode);
 747
 748 /**
 749  * rpcauth_wrap_req - XDR encode and wrap the RPC procedure
 750  * @task: controlling RPC task
 751  * @xdr: stream where on-the-wire bytes are to be marshalled
 752  *
 753  * On success, @xdr contains the encoded and wrapped message,
 754  * and zero is returned. Otherwise, @xdr is in an undefined
 755  * state and a negative errno is returned.
 756  */
 757 int rpcauth_wrap_req(struct rpc_task *task, struct xdr_stream *xdr)
 758 {
 759         const struct rpc_credops *ops = task->tk_rqstp->rq_cred->cr_ops;
 760
 761         return ops->crwrap_req(task, xdr);
 762 }
 763
 764 /**
 765  * rpcauth_checkverf - Validate verifier in RPC Reply header
 766  * @task: controlling RPC task
 767  * @xdr: xdr_stream containing RPC Reply header
 768  *
 769  * Return values:
 770  *   %0: Verifier is valid. @xdr now points past the verifier.
 771  *   %-EIO: Verifier is corrupted or message ended early.
 772  *   %-EACCES: Verifier is intact but not valid.
 773  *   %-EPROTONOSUPPORT: Server does not support the requested auth type.
 774  *
 775  * When a negative errno is returned, @xdr is left in an undefined
 776  * state.
 777  */
 778 int
 779 rpcauth_checkverf(struct rpc_task *task, struct xdr_stream *xdr)
 780 {
 781         const struct rpc_credops *ops = task->tk_rqstp->rq_cred->cr_ops;
 782
 783         return ops->crvalidate(task, xdr);
 784 }
 785
 786 /**
 787  * rpcauth_unwrap_resp_decode - Invoke XDR decode function
 788  * @task: controlling RPC task
 789  * @xdr: stream where the Reply message resides
 790  *
 791  * Returns zero on success; otherwise a negative errno is returned.
 792  */
 793 int
 794 rpcauth_unwrap_resp_decode(struct rpc_task *task, struct xdr_stream *xdr)
 795 {
 796         kxdrdproc_t decode = task->tk_msg.rpc_proc->p_decode;
 797
 798         return decode(task->tk_rqstp, xdr, task->tk_msg.rpc_resp);
 799 }
 800 EXPORT_SYMBOL_GPL(rpcauth_unwrap_resp_decode);
 801
 802 /**
 803  * rpcauth_unwrap_resp - Invoke unwrap and decode function for the cred
 804  * @task: controlling RPC task
 805  * @xdr: stream where the Reply message resides
 806  *
 807  * Returns zero on success; otherwise a negative errno is returned.
 808  */
 809 int
 810 rpcauth_unwrap_resp(struct rpc_task *task, struct xdr_stream *xdr)
 811 {
 812         const struct rpc_credops *ops = task->tk_rqstp->rq_cred->cr_ops;
 813
 814         return ops->crunwrap_resp(task, xdr);
 815 }
 816
 817 bool
 818 rpcauth_xmit_need_reencode(struct rpc_task *task)
 819 {
 820         struct rpc_cred *cred = task->tk_rqstp->rq_cred;
 821
 822         if (!cred || !cred->cr_ops->crneed_reencode)
 823                 return false;
 824         return cred->cr_ops->crneed_reencode(task);
 825 }
 826
 827 int
 828 rpcauth_refreshcred(struct rpc_task *task)
 829 {
 830         struct rpc_cred *cred;
 831         int err;
 832
 833         cred = task->tk_rqstp->rq_cred;
 834         if (cred == NULL) {
 835                 err = rpcauth_bindcred(task, task->tk_msg.rpc_cred, task->tk_flags);
 836                 if (err < 0)
 837                         goto out;
 838                 cred = task->tk_rqstp->rq_cred;
 839         }
 840
 841         err = cred->cr_ops->crrefresh(task);
 842 out:
 843         if (err < 0)
 844                 task->tk_status = err;
 845         return err;
 846 }
 847
 848 void
 849 rpcauth_invalcred(struct rpc_task *task)
 850 {
 851         struct rpc_cred *cred = task->tk_rqstp->rq_cred;
 852
 853         if (cred)
 854                 clear_bit(RPCAUTH_CRED_UPTODATE, &cred->cr_flags);
 855 }
 856
 857 int
 858 rpcauth_uptodatecred(struct rpc_task *task)
 859 {
 860         struct rpc_cred *cred = task->tk_rqstp->rq_cred;
 861
 862         return cred == NULL ||
 863                 test_bit(RPCAUTH_CRED_UPTODATE, &cred->cr_flags) != 0;
 864 }
 865
 866 static struct shrinker *rpc_cred_shrinker;
 867
 868 int __init rpcauth_init_module(void)
 869 {
 870         int err;
 871
 872         err = rpc_init_authunix();
 873         if (err < 0)
 874                 goto out1;
 875         rpc_cred_shrinker = shrinker_alloc(0, "sunrpc_cred");
 876         if (!rpc_cred_shrinker) {
 877                 err = -ENOMEM;
 878                 goto out2;
 879         }
 880
 881         rpc_cred_shrinker->count_objects = rpcauth_cache_shrink_count;
 882         rpc_cred_shrinker->scan_objects = rpcauth_cache_shrink_scan;
 883
 884         shrinker_register(rpc_cred_shrinker);
 885
 886         return 0;
 887 out2:
 888         rpc_destroy_authunix();
 889 out1:
 890         return err;
 891 }
 892
 893 void rpcauth_remove_module(void)
 894 {
 895         rpc_destroy_authunix();
 896         shrinker_free(rpc_cred_shrinker);
 897 }