|  | @@ -38,9 +38,6 @@
 | 
											
												
													
														|  |  #include <grpc/support/alloc.h>
 |  |  #include <grpc/support/alloc.h>
 | 
											
												
													
														|  |  #include <grpc/support/log.h>
 |  |  #include <grpc/support/log.h>
 | 
											
												
													
														|  |  
 |  |  
 | 
											
												
													
														|  | -#define STATE_BIT_ALIVE 1
 |  | 
 | 
											
												
													
														|  | -#define STATE_BIT_REFS 2
 |  | 
 | 
											
												
													
														|  | -
 |  | 
 | 
											
												
													
														|  |  typedef struct grpc_aelock_qnode {
 |  |  typedef struct grpc_aelock_qnode {
 | 
											
												
													
														|  |    gpr_mpscq_node mpscq_node;
 |  |    gpr_mpscq_node mpscq_node;
 | 
											
												
													
														|  |    grpc_aelock_action action;
 |  |    grpc_aelock_action action;
 | 
											
										
											
												
													
														|  | @@ -53,24 +50,17 @@ struct grpc_aelock {
 | 
											
												
													
														|  |    // state is:
 |  |    // state is:
 | 
											
												
													
														|  |    // lower bit - zero if orphaned
 |  |    // lower bit - zero if orphaned
 | 
											
												
													
														|  |    // other bits - number of items queued on the lock
 |  |    // other bits - number of items queued on the lock
 | 
											
												
													
														|  | -  // see: STATE_BIT_xxx
 |  | 
 | 
											
												
													
														|  |    gpr_atm state;
 |  |    gpr_atm state;
 | 
											
												
													
														|  | -  grpc_aelock_action before_idle_action;
 |  | 
 | 
											
												
													
														|  | -  void *before_idle_action_arg;
 |  | 
 | 
											
												
													
														|  |    grpc_closure continue_finishing;
 |  |    grpc_closure continue_finishing;
 | 
											
												
													
														|  |  };
 |  |  };
 | 
											
												
													
														|  |  
 |  |  
 | 
											
												
													
														|  |  static void continue_finishing(grpc_exec_ctx *exec_ctx, void *arg,
 |  |  static void continue_finishing(grpc_exec_ctx *exec_ctx, void *arg,
 | 
											
												
													
														|  |                                 bool success);
 |  |                                 bool success);
 | 
											
												
													
														|  |  
 |  |  
 | 
											
												
													
														|  | -grpc_aelock *grpc_aelock_create(grpc_workqueue *optional_workqueue,
 |  | 
 | 
											
												
													
														|  | -                                grpc_aelock_action before_idle_action,
 |  | 
 | 
											
												
													
														|  | -                                void *before_idle_action_arg) {
 |  | 
 | 
											
												
													
														|  | 
 |  | +grpc_aelock *grpc_aelock_create(grpc_workqueue *optional_workqueue) {
 | 
											
												
													
														|  |    grpc_aelock *lock = gpr_malloc(sizeof(*lock));
 |  |    grpc_aelock *lock = gpr_malloc(sizeof(*lock));
 | 
											
												
													
														|  | -  lock->before_idle_action = before_idle_action;
 |  | 
 | 
											
												
													
														|  | -  lock->before_idle_action_arg = before_idle_action_arg;
 |  | 
 | 
											
												
													
														|  |    lock->optional_workqueue = optional_workqueue;
 |  |    lock->optional_workqueue = optional_workqueue;
 | 
											
												
													
														|  | -  gpr_atm_no_barrier_store(&lock->state, STATE_BIT_ALIVE);
 |  | 
 | 
											
												
													
														|  | 
 |  | +  gpr_atm_no_barrier_store(&lock->state, 1);
 | 
											
												
													
														|  |    gpr_mpscq_init(&lock->queue);
 |  |    gpr_mpscq_init(&lock->queue);
 | 
											
												
													
														|  |    grpc_closure_init(&lock->continue_finishing, continue_finishing, lock);
 |  |    grpc_closure_init(&lock->continue_finishing, continue_finishing, lock);
 | 
											
												
													
														|  |    return lock;
 |  |    return lock;
 | 
											
										
											
												
													
														|  | @@ -83,8 +73,7 @@ static void really_destroy(grpc_aelock *lock) {
 | 
											
												
													
														|  |  }
 |  |  }
 | 
											
												
													
														|  |  
 |  |  
 | 
											
												
													
														|  |  void grpc_aelock_destroy(grpc_aelock *lock) {
 |  |  void grpc_aelock_destroy(grpc_aelock *lock) {
 | 
											
												
													
														|  | -  if (gpr_atm_full_fetch_add(&lock->state, -STATE_BIT_ALIVE) ==
 |  | 
 | 
											
												
													
														|  | -      STATE_BIT_ALIVE) {
 |  | 
 | 
											
												
													
														|  | 
 |  | +  if (gpr_atm_full_fetch_add(&lock->state, -1) == 1) {
 | 
											
												
													
														|  |      really_destroy(lock);
 |  |      really_destroy(lock);
 | 
											
												
													
														|  |    }
 |  |    }
 | 
											
												
													
														|  |  }
 |  |  }
 | 
											
										
											
												
													
														|  | @@ -92,6 +81,10 @@ void grpc_aelock_destroy(grpc_aelock *lock) {
 | 
											
												
													
														|  |  static bool maybe_finish_one(grpc_exec_ctx *exec_ctx, grpc_aelock *lock) {
 |  |  static bool maybe_finish_one(grpc_exec_ctx *exec_ctx, grpc_aelock *lock) {
 | 
											
												
													
														|  |    gpr_mpscq_node *n = gpr_mpscq_pop(&lock->queue);
 |  |    gpr_mpscq_node *n = gpr_mpscq_pop(&lock->queue);
 | 
											
												
													
														|  |    if (n == NULL) {
 |  |    if (n == NULL) {
 | 
											
												
													
														|  | 
 |  | +    // queue is in an inconsistant state: use this as a cue that we should
 | 
											
												
													
														|  | 
 |  | +    // go off and do something else for a while (and come back later)
 | 
											
												
													
														|  | 
 |  | +    grpc_exec_ctx_enqueue(exec_ctx, &lock->continue_finishing, true,
 | 
											
												
													
														|  | 
 |  | +                          lock->optional_workqueue);
 | 
											
												
													
														|  |      return false;
 |  |      return false;
 | 
											
												
													
														|  |    }
 |  |    }
 | 
											
												
													
														|  |    grpc_aelock_qnode *ln = (grpc_aelock_qnode *)n;
 |  |    grpc_aelock_qnode *ln = (grpc_aelock_qnode *)n;
 | 
											
										
											
												
													
														|  | @@ -101,89 +94,36 @@ static bool maybe_finish_one(grpc_exec_ctx *exec_ctx, grpc_aelock *lock) {
 | 
											
												
													
														|  |  }
 |  |  }
 | 
											
												
													
														|  |  
 |  |  
 | 
											
												
													
														|  |  static void finish(grpc_exec_ctx *exec_ctx, grpc_aelock *lock) {
 |  |  static void finish(grpc_exec_ctx *exec_ctx, grpc_aelock *lock) {
 | 
											
												
													
														|  | -  for (;;) {
 |  | 
 | 
											
												
													
														|  | -    gpr_atm last_state = gpr_atm_full_fetch_add(&lock->state, -STATE_BIT_REFS);
 |  | 
 | 
											
												
													
														|  | -    switch (last_state) {
 |  | 
 | 
											
												
													
														|  | -      default:
 |  | 
 | 
											
												
													
														|  | -      perform_one_step:
 |  | 
 | 
											
												
													
														|  | -        gpr_log(GPR_DEBUG, "ls=%d execute", last_state);
 |  | 
 | 
											
												
													
														|  | -        if (!maybe_finish_one(exec_ctx, lock)) {
 |  | 
 | 
											
												
													
														|  | -          // perform the idle action before going off to do something else
 |  | 
 | 
											
												
													
														|  | -          lock->before_idle_action(exec_ctx, lock->before_idle_action_arg);
 |  | 
 | 
											
												
													
														|  | -          // quick peek to see if we can immediately resume
 |  | 
 | 
											
												
													
														|  | -          if (!maybe_finish_one(exec_ctx, lock)) {
 |  | 
 | 
											
												
													
														|  | -            // queue is in an inconsistant state: use this as a cue that we
 |  | 
 | 
											
												
													
														|  | -            // should
 |  | 
 | 
											
												
													
														|  | -            // go off and do something else for a while (and come back later)
 |  | 
 | 
											
												
													
														|  | -            grpc_exec_ctx_enqueue(exec_ctx, &lock->continue_finishing, true,
 |  | 
 | 
											
												
													
														|  | -                                  lock->optional_workqueue);
 |  | 
 | 
											
												
													
														|  | -            return;
 |  | 
 | 
											
												
													
														|  | -          }
 |  | 
 | 
											
												
													
														|  | -        }
 |  | 
 | 
											
												
													
														|  | -        break;
 |  | 
 | 
											
												
													
														|  | -      case STATE_BIT_ALIVE | (2 * STATE_BIT_REFS):
 |  | 
 | 
											
												
													
														|  | -        gpr_log(GPR_DEBUG, "ls=%d final", last_state);
 |  | 
 | 
											
												
													
														|  | -        lock->before_idle_action(exec_ctx, lock->before_idle_action_arg);
 |  | 
 | 
											
												
													
														|  | -        switch (gpr_atm_full_fetch_add(&lock->state, -STATE_BIT_REFS)) {
 |  | 
 | 
											
												
													
														|  | -          case STATE_BIT_ALIVE | STATE_BIT_REFS:
 |  | 
 | 
											
												
													
														|  | -            return;
 |  | 
 | 
											
												
													
														|  | -          case STATE_BIT_REFS:
 |  | 
 | 
											
												
													
														|  | -            really_destroy(lock);
 |  | 
 | 
											
												
													
														|  | -            return;
 |  | 
 | 
											
												
													
														|  | -          default:
 |  | 
 | 
											
												
													
														|  | -            gpr_log(GPR_DEBUG, "retry");
 |  | 
 | 
											
												
													
														|  | -            // oops: did the before action, but something else came in
 |  | 
 | 
											
												
													
														|  | -            // better add another ref so we remember to do this again
 |  | 
 | 
											
												
													
														|  | -            gpr_atm_full_fetch_add(&lock->state, STATE_BIT_REFS);
 |  | 
 | 
											
												
													
														|  | -            goto perform_one_step;
 |  | 
 | 
											
												
													
														|  | -        }
 |  | 
 | 
											
												
													
														|  | -        break;
 |  | 
 | 
											
												
													
														|  | -      case STATE_BIT_ALIVE | STATE_BIT_REFS:
 |  | 
 | 
											
												
													
														|  | -        gpr_log(GPR_DEBUG, "ls=%d unlock", last_state);
 |  | 
 | 
											
												
													
														|  | 
 |  | +  do {
 | 
											
												
													
														|  | 
 |  | +    switch (gpr_atm_full_fetch_add(&lock->state, -2)) {
 | 
											
												
													
														|  | 
 |  | +      case 3:  // had one count, one unorphaned --> unlocked unorphaned
 | 
											
												
													
														|  |          return;
 |  |          return;
 | 
											
												
													
														|  | -      case 2 * STATE_BIT_REFS:
 |  | 
 | 
											
												
													
														|  | -        gpr_log(GPR_DEBUG, "ls=%d idle", last_state);
 |  | 
 | 
											
												
													
														|  | -        lock->before_idle_action(exec_ctx, lock->before_idle_action_arg);
 |  | 
 | 
											
												
													
														|  | -        GPR_ASSERT(gpr_atm_full_fetch_add(&lock->state, -STATE_BIT_REFS) ==
 |  | 
 | 
											
												
													
														|  | -                   STATE_BIT_REFS);
 |  | 
 | 
											
												
													
														|  | -      case STATE_BIT_REFS:
 |  | 
 | 
											
												
													
														|  | -        gpr_log(GPR_DEBUG, "ls=%d destroy", last_state);
 |  | 
 | 
											
												
													
														|  | 
 |  | +      case 2:  // and one count, one orphaned --> unlocked and orphaned
 | 
											
												
													
														|  |          really_destroy(lock);
 |  |          really_destroy(lock);
 | 
											
												
													
														|  |          return;
 |  |          return;
 | 
											
												
													
														|  | -      case STATE_BIT_ALIVE:
 |  | 
 | 
											
												
													
														|  | 
 |  | +      case 1:
 | 
											
												
													
														|  |        case 0:
 |  |        case 0:
 | 
											
												
													
														|  |          // these values are illegal - representing an already unlocked or
 |  |          // these values are illegal - representing an already unlocked or
 | 
											
												
													
														|  |          // deleted lock
 |  |          // deleted lock
 | 
											
												
													
														|  |          GPR_UNREACHABLE_CODE(return );
 |  |          GPR_UNREACHABLE_CODE(return );
 | 
											
												
													
														|  |      }
 |  |      }
 | 
											
												
													
														|  | -  }
 |  | 
 | 
											
												
													
														|  | -
 |  | 
 | 
											
												
													
														|  | -  // while (maybe_finish_one(exec_ctx, lock));
 |  | 
 | 
											
												
													
														|  | 
 |  | +  } while (maybe_finish_one(exec_ctx, lock));
 | 
											
												
													
														|  |  }
 |  |  }
 | 
											
												
													
														|  |  
 |  |  
 | 
											
												
													
														|  |  static void continue_finishing(grpc_exec_ctx *exec_ctx, void *arg,
 |  |  static void continue_finishing(grpc_exec_ctx *exec_ctx, void *arg,
 | 
											
												
													
														|  |                                 bool success) {
 |  |                                 bool success) {
 | 
											
												
													
														|  | -  grpc_aelock *lock = arg;
 |  | 
 | 
											
												
													
														|  | -  if (maybe_finish_one(exec_ctx, lock)) {
 |  | 
 | 
											
												
													
														|  | -    finish(exec_ctx, lock);
 |  | 
 | 
											
												
													
														|  | -  } else {
 |  | 
 | 
											
												
													
														|  | -    // queue is in an inconsistant state: use this as a cue that we should
 |  | 
 | 
											
												
													
														|  | -    // go off and do something else for a while (and come back later)
 |  | 
 | 
											
												
													
														|  | -    grpc_exec_ctx_enqueue(exec_ctx, &lock->continue_finishing, true,
 |  | 
 | 
											
												
													
														|  | -                          lock->optional_workqueue);
 |  | 
 | 
											
												
													
														|  | -  }
 |  | 
 | 
											
												
													
														|  | 
 |  | +  if (maybe_finish_one(exec_ctx, arg)) finish(exec_ctx, arg);
 | 
											
												
													
														|  |  }
 |  |  }
 | 
											
												
													
														|  |  
 |  |  
 | 
											
												
													
														|  |  void grpc_aelock_execute(grpc_exec_ctx *exec_ctx, grpc_aelock *lock,
 |  |  void grpc_aelock_execute(grpc_exec_ctx *exec_ctx, grpc_aelock *lock,
 | 
											
												
													
														|  |                           grpc_aelock_action action, void *arg,
 |  |                           grpc_aelock_action action, void *arg,
 | 
											
												
													
														|  |                           size_t sizeof_arg) {
 |  |                           size_t sizeof_arg) {
 | 
											
												
													
														|  | -  gpr_atm last = gpr_atm_full_fetch_add(&lock->state, 2 * STATE_BIT_REFS);
 |  | 
 | 
											
												
													
														|  | -  GPR_ASSERT(last & STATE_BIT_ALIVE);  // ensure lock has not been destroyed
 |  | 
 | 
											
												
													
														|  | -  if (last == STATE_BIT_ALIVE) {
 |  | 
 | 
											
												
													
														|  | 
 |  | +  gpr_atm last = gpr_atm_full_fetch_add(&lock->state, 2);
 | 
											
												
													
														|  | 
 |  | +  GPR_ASSERT(last & 1);  // ensure lock has not been destroyed
 | 
											
												
													
														|  | 
 |  | +  if (last == 1) {
 | 
											
												
													
														|  |      action(exec_ctx, arg);
 |  |      action(exec_ctx, arg);
 | 
											
												
													
														|  |      finish(exec_ctx, lock);
 |  |      finish(exec_ctx, lock);
 | 
											
												
													
														|  |    } else {
 |  |    } else {
 | 
											
												
													
														|  | -    gpr_atm_full_fetch_add(&lock->state, -STATE_BIT_REFS);
 |  | 
 | 
											
												
													
														|  |      grpc_aelock_qnode *n = gpr_malloc(sizeof(*n) + sizeof_arg);
 |  |      grpc_aelock_qnode *n = gpr_malloc(sizeof(*n) + sizeof_arg);
 | 
											
												
													
														|  |      n->action = action;
 |  |      n->action = action;
 | 
											
												
													
														|  |      if (sizeof_arg > 0) {
 |  |      if (sizeof_arg > 0) {
 |