This is not the current version of the class.

Lecture 18: Fair locks and RCU

Fairness

2101989 (3156)
2129185 (4436)
2139257 (4579)
2108552 (5182)

Solving fairness

Ticket lock

struct ticket_lock {
    std::atomic<unsigned> now_ = 0;
    std::atomic<unsigned> next_ = 0;

    void lock() {
        unsigned me = next_++;
        while (me != now_) {
            pause();
        }
    }
    void unlock() {
        now_++;
    }
};

MCS (Mellor-Crummey Scott) lock

struct mcs_lock {
    struct qentry {    // user must declare one of these to lock
        std::atomic<qentry*> next;
        std::atomic<bool> blocked;
    };

    std::atomic<qentry*> lk_;   // points at tail

    void lock(qentry& w) {
        w.next = nullptr;
        qentry* prev_tail = lk_.exchange(&w);  // mark self as tail
        if (prev_tail) {           // previous tail exists
            w.blocked = true;
            prev_tail->next = &w;  // link to previous tail
            while (w.blocked) {
                pause();
            }
        }
    }
    void unlock(qentry& w) {
        qentry* expected_tail = &w;
        if (!w.next
            && lk_.compare_exchange_weak(expected_tail, nullptr)) {
            return;  // no one else is waiting
        }
        while (!w.next) {  // wait for next tail to link self
            pause();
        }
        w.next->blocked = false;
    }
}

// some function that uses a lock `l`
f() {
    ...
    mcs_lock::qentry w;
    l.lock(w);
    ...
    l.unlock(w);
    ...
}

About MCS

Blocking in locks

Mutex goals

Futex

Futex example (Drepper’s “Mutex #3”)

struct futex_lock {
    std::atomic<int> val_;
    // 0 = unlocked; 1 = locked, no futex waiters;
    // 2 = locked, maybe futex waiters

    void lock() {
        // phase 1
        for (unsigned i = 0; i < 40; i++) {
            int expected = 0;
            if (val_.compare_exchange_weak(expected, 1)) {
                return;
            }
            sched_yield();
        }

        // phase 2
        int previous = val_.load(std::memory_order_relaxed);
        if (previous != 2) {
            previous = val_.exchange(2);
        }
        while (previous != 0) {
            futex(&val_, FUTEX_WAIT, 2);
            previous = val_.exchange(2);
        }
    }

    void unlock() {
        if (--val_ != 0) {    // atomic decrement
            val_ = 0;
            futex(&val_, FUTEX_WAKE, 1);
        }
    }
};

Readers/writer locks

These are good when reads are much more common than writes. There can be any number of readers or a single writer.

The lock has three states

  1. unlocked (val_ == 0)
  2. read locked (val_ > 0)
  3. write locked (val_ == -1)
struct rw_lock {
    std::atomic<int> val_;

    void lock_read() {
        int expected = val_;
        while (expected < 0
               || !val_.compare_exchange_weak
                        (expected, expected + 1)) {
            pause();
            expected = val_;
        }
    }
    void unlock_read() {
        --val_;
    }

    void lock_write() {
        int expected = 0;
        while (!val_.compare_exchange_weak(expected, -1)) {
            pause();
            expected = 0;
        }
    }
    void unlock_write() {
        val_ = 0;
    }
};

rwlocks: Reducing memory contention

struct rw_lock_2 {
    spinlock f_[NCPU];     // would really want separate cache lines

    void lock_read() {
        f_[this_cpu()].lock();
    }
    void unlock_read() {
        f_[this_cpu()].unlock();
    }

    void lock_write() {
        for (unsigned i = 0; i != NCPU; ++i) {
            f_[i].lock();
        }
    }
    void unlock_write() {
        for (unsigned i = 0; i != NCPU; ++i) {
            f_[i].unlock();
        }
    }
};

Write balance in practice

RCU (Read-Copy Update)

Zero-op read locks: the simple case

an integer

Reading and writing integers safely

std::atomic<int> val;

int read() {
    return val.load(std::memory_order_relaxed);
}

void write(int x) {
    val.store(x, std::memory_order_relaxed);
}

std::atomic<int> val;
spinlock val_lock;

int read() {
    return val.load(std::memory_order_relaxed);
}

void modify() {
    val_lock.lock();
    ... compute with current `val`; other writers are excluded ...
    val.store(x, std::memory_order_relaxed);
    val_lock.unlock();
}

Idea: Use atomics to name versions

struct bigvalue { ... };
std::atomic<bigvalue*> val;
spinlock val_lock;

bigvalue* read_snapshot() {
    return val.load(std::memory_order_relaxed);
}

void modify() {
    val_lock.lock();
    bigvalue* newval = new bigvalue;
    ... compute with `val`; initialize `newval` ...
    val.store(newval, std::memory_order_relaxed);
    val_lock.unlock();
}

Memory allocation and RCU

Idea: Epoch-based reclamation

Sketch

struct bigvalue { ... };
std::atomic<bigvalue*> val;
spinlock val_lock;
std::deque<std::pair<bigvalue*, time_t>> val_garbage;
time_t current_epoch;
time_t read_epochs[NCPU];

void read() {
    // start read-side critical section
    read_epochs[this_cpu()] = current_epoch;

    bigvalue* v = val.load(std::memory_order_relaxed);
    ... use `v` arbitrarily ...

    // mark completion
    read_epochs[this_cpu()] = 0;

    ... MUST NOT refer to `v` (it might be freed!) ...
}

void modify() {
    val_lock.lock();
    bigvalue* oldval = val;
    bigvalue* newval = new bigvalue;
    ... compute, initialize ...
    val.store(newval);
    val_lock.unlock();

    val_garbage.push_back({oldval, current_epoch});
    // NB should lock `val_garbage`
}

void gc() {
    // run periodically
    time_t garbage_epoch = min(read_epochs); // ignoring zeroes
    ++current_epoch;
    free all `val_garbage` older than `garbage_epoch`;
}

Other mechanisms