code review 5279048: runtime: faster and more scalable GC

src/pkg/runtime/arm/atomic.c

 // Copyright 2009 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
 #include "runtime.h"
+#include "arch.h"
+
+static union {
+        Lock l;
+        byte pad [CacheLineSize];
+} locktab[57];
+
+#define LOCK(addr) (&locktab[((uintptr)addr>>3)%nelem(locktab)].l)
 
 // Atomic add and return new value.
 #pragma textflag 7
 uint32
 runtime·xadd(uint32 volatile *val, int32 delta)
 {
         uint32 oval, nval;
 
         for(;;){
                 oval = *val;
@@ skipping 51 matching lines @@
                 if(runtime·casp(addr, old, v))
                         return;
         }
 }
 
 #pragma textflag 7
 void
 runtime·atomicstore(uint32 volatile* addr, uint32 v)
 {
         uint32 old;
-        
+
         for(;;) {
                 old = *addr;
                 if(runtime·cas(addr, old, v))
                         return;
         }
 }
 
 #pragma textflag 7
+bool
+runtime·cas64(uint64 volatile *addr, uint64 *old, uint64 new)
+{
+        bool res;
+········
+        runtime·lock(LOCK(addr));
+        if(*addr == *old) {
+                *addr = new;
+                res = true;
+        } else {
+                *old = *addr;
+                res = false;
+        }
+        runtime·unlock(LOCK(addr));
+        return res;
+}
+
+#pragma textflag 7
 uint64
-runtime·xadd64(uint64 volatile *val, int64 delta)
+runtime·xadd64(uint64 volatile *addr, int64 delta)
 {
-        uint64 old;
-
-        old = *val;
-        for(;;) {
-                if(runtime·cas64(val, &old, old+v))
-                        return old+v;
-        }
+        uint64 res;
+ ·······
+        runtime·lock(LOCK(addr));
+        res = *addr + delta;
+        *addr = res;
+        runtime·unlock(LOCK(addr));
+        return res;
 }
 
 #pragma textflag 7
 uint64
 runtime·atomicload64(uint64 volatile *addr)
 {
-        uint64 old;
+        uint64 res;
 ········
-        old = *val;
-        for(;;) {
-                if(runtime·cas64(val, &old, old))
-                        return old;
-        }
+        runtime·lock(LOCK(addr));
+        res = *addr;
+        runtime·unlock(LOCK(addr));
+        return res;
 }
 
 #pragma textflag 7
 void
 runtime·atomicstore64(uint64 volatile *addr, uint64 v)
 {
-        uint64 old;
- ·······
-        old = *val;
-        for(;;) {
-                if(runtime·cas64(val, &old, v))
-                        return;
-        }
-}
+        runtime·lock(LOCK(addr));
+        *addr = v;
+        runtime·unlock(LOCK(addr));
+}