Skip to content

Commit

Permalink
frv: Rewrite atomic implementation
Browse files Browse the repository at this point in the history 
Mostly complete rewrite of the FRV atomic implementation, instead of
using assembly files, use inline assembler.

The out-of-line CONFIG option makes a bit of a mess of things, but a
little CPP trickery gets that done too.

FRV already had the atomic logic ops but under a non standard name,
the reimplementation provides the generic names and provides the
intermediate form required for the bitops implementation.

The slightly inconsistent __atomic32_fetch_##op naming is because
__atomic_fetch_##op conlicts with GCC builtin functions.

The 64bit atomic ops use the inline assembly %Ln construct to access
the low word register (r+1), afaik this construct was not previously
used in the kernel and is completely undocumented, but I found it in
the FRV GCC code and it seems to work.

FRV had a non-standard definition of atomic_{clear,set}_mask() which
would work types other than atomic_t, the one user relying on that
(arch/frv/kernel/dma.c) got converted to use the new intermediate
form.

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
  • Loading branch information
@KAGA-KOKO
Peter Zijlstra authored and KAGA-KOKO committed Jul 27, 2015
1 parent 7fc1845 commit b0d8003
Show file tree
Hide file tree
Showing 9 changed files with 259 additions and 355 deletions.
119 changes: 66 additions & 53 deletions arch/frv/include/asm/atomic.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,14 +15,15 @@
#define _ASM_ATOMIC_H

#include <linux/types.h>
#include <asm/spr-regs.h>
#include <asm/cmpxchg.h>
#include <asm/barrier.h>

#ifdef CONFIG_SMP
#error not SMP safe
#endif

#include <asm/atomic_defs.h>

/*
* Atomic operations that C can't guarantee us. Useful for
* resource counting etc..
Expand All @@ -34,56 +35,26 @@
#define atomic_read(v) ACCESS_ONCE((v)->counter)
#define atomic_set(v, i) (((v)->counter) = (i))

#ifndef CONFIG_FRV_OUTOFLINE_ATOMIC_OPS
static inline int atomic_add_return(int i, atomic_t *v)
static inline int atomic_inc_return(atomic_t *v)
{
unsigned long val;
return __atomic_add_return(1, &v->counter);
}

asm("0: \n"
" orcc gr0,gr0,gr0,icc3 \n" /* set ICC3.Z */
" ckeq icc3,cc7 \n"
" ld.p %M0,%1 \n" /* LD.P/ORCR must be atomic */
" orcr cc7,cc7,cc3 \n" /* set CC3 to true */
" add%I2 %1,%2,%1 \n"
" cst.p %1,%M0 ,cc3,#1 \n"
" corcc gr29,gr29,gr0 ,cc3,#1 \n" /* clear ICC3.Z if store happens */
" beq icc3,#0,0b \n"
: "+U"(v->counter), "=&r"(val)
: "NPr"(i)
: "memory", "cc7", "cc3", "icc3"
);
static inline int atomic_dec_return(atomic_t *v)
{
return __atomic_sub_return(1, &v->counter);
}

return val;
static inline int atomic_add_return(int i, atomic_t *v)
{
return __atomic_add_return(i, &v->counter);
}

static inline int atomic_sub_return(int i, atomic_t *v)
{
unsigned long val;

asm("0: \n"
" orcc gr0,gr0,gr0,icc3 \n" /* set ICC3.Z */
" ckeq icc3,cc7 \n"
" ld.p %M0,%1 \n" /* LD.P/ORCR must be atomic */
" orcr cc7,cc7,cc3 \n" /* set CC3 to true */
" sub%I2 %1,%2,%1 \n"
" cst.p %1,%M0 ,cc3,#1 \n"
" corcc gr29,gr29,gr0 ,cc3,#1 \n" /* clear ICC3.Z if store happens */
" beq icc3,#0,0b \n"
: "+U"(v->counter), "=&r"(val)
: "NPr"(i)
: "memory", "cc7", "cc3", "icc3"
);

return val;
return __atomic_sub_return(i, &v->counter);
}

#else

extern int atomic_add_return(int i, atomic_t *v);
extern int atomic_sub_return(int i, atomic_t *v);

#endif

static inline int atomic_add_negative(int i, atomic_t *v)
{
return atomic_add_return(i, v) < 0;
Expand All @@ -101,17 +72,14 @@ static inline void atomic_sub(int i, atomic_t *v)

static inline void atomic_inc(atomic_t *v)
{
atomic_add_return(1, v);
atomic_inc_return(v);
}

static inline void atomic_dec(atomic_t *v)
{
atomic_sub_return(1, v);
atomic_dec_return(v);
}

#define atomic_dec_return(v) atomic_sub_return(1, (v))
#define atomic_inc_return(v) atomic_add_return(1, (v))

#define atomic_sub_and_test(i,v) (atomic_sub_return((i), (v)) == 0)
#define atomic_dec_and_test(v) (atomic_sub_return(1, (v)) == 0)
#define atomic_inc_and_test(v) (atomic_add_return(1, (v)) == 0)
Expand All @@ -120,18 +88,19 @@ static inline void atomic_dec(atomic_t *v)
* 64-bit atomic ops
*/
typedef struct {
volatile long long counter;
long long counter;
} atomic64_t;

#define ATOMIC64_INIT(i) { (i) }

static inline long long atomic64_read(atomic64_t *v)
static inline long long atomic64_read(const atomic64_t *v)
{
long long counter;

asm("ldd%I1 %M1,%0"
: "=e"(counter)
: "m"(v->counter));

return counter;
}

Expand All @@ -142,10 +111,25 @@ static inline void atomic64_set(atomic64_t *v, long long i)
: "e"(i));
}

extern long long atomic64_inc_return(atomic64_t *v);
extern long long atomic64_dec_return(atomic64_t *v);
extern long long atomic64_add_return(long long i, atomic64_t *v);
extern long long atomic64_sub_return(long long i, atomic64_t *v);
static inline long long atomic64_inc_return(atomic64_t *v)
{
return __atomic64_add_return(1, &v->counter);
}

static inline long long atomic64_dec_return(atomic64_t *v)
{
return __atomic64_sub_return(1, &v->counter);
}

static inline long long atomic64_add_return(long long i, atomic64_t *v)
{
return __atomic64_add_return(i, &v->counter);
}

static inline long long atomic64_sub_return(long long i, atomic64_t *v)
{
return __atomic64_sub_return(i, &v->counter);
}

static inline long long atomic64_add_negative(long long i, atomic64_t *v)
{
Expand Down Expand Up @@ -176,6 +160,7 @@ static inline void atomic64_dec(atomic64_t *v)
#define atomic64_dec_and_test(v) (atomic64_dec_return((v)) == 0)
#define atomic64_inc_and_test(v) (atomic64_inc_return((v)) == 0)


#define atomic_cmpxchg(v, old, new) (cmpxchg(&(v)->counter, old, new))
#define atomic_xchg(v, new) (xchg(&(v)->counter, new))
#define atomic64_cmpxchg(v, old, new) (__cmpxchg_64(old, new, &(v)->counter))
Expand All @@ -196,5 +181,33 @@ static __inline__ int __atomic_add_unless(atomic_t *v, int a, int u)
return c;
}

#define ATOMIC_OP(op) \
static inline void atomic_##op(int i, atomic_t *v) \
{ \
(void)__atomic32_fetch_##op(i, &v->counter); \
} \
\
static inline void atomic64_##op(long long i, atomic64_t *v) \
{ \
(void)__atomic64_fetch_##op(i, &v->counter); \
}

#define CONFIG_ARCH_HAS_ATOMIC_OR

ATOMIC_OP(or)
ATOMIC_OP(and)
ATOMIC_OP(xor)

#undef ATOMIC_OP

static inline __deprecated void atomic_clear_mask(unsigned int mask, atomic_t *v)
{
atomic_and(~mask, v);
}

static inline __deprecated void atomic_set_mask(unsigned int mask, atomic_t *v)
{
atomic_or(mask, v);
}

#endif /* _ASM_ATOMIC_H */
172 changes: 172 additions & 0 deletions arch/frv/include/asm/atomic_defs.h
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,172 @@

#include <asm/spr-regs.h>

#ifdef __ATOMIC_LIB__

#ifdef CONFIG_FRV_OUTOFLINE_ATOMIC_OPS

#define ATOMIC_QUALS
#define ATOMIC_EXPORT(x) EXPORT_SYMBOL(x)

#else /* !OUTOFLINE && LIB */

#define ATOMIC_OP_RETURN(op)
#define ATOMIC_FETCH_OP(op)

#endif /* OUTOFLINE */

#else /* !__ATOMIC_LIB__ */

#define ATOMIC_EXPORT(x)

#ifdef CONFIG_FRV_OUTOFLINE_ATOMIC_OPS

#define ATOMIC_OP_RETURN(op) \
extern int __atomic_##op##_return(int i, int *v); \
extern long long __atomic64_##op##_return(long long i, long long *v);

#define ATOMIC_FETCH_OP(op) \
extern int __atomic32_fetch_##op(int i, int *v); \
extern long long __atomic64_fetch_##op(long long i, long long *v);

#else /* !OUTOFLINE && !LIB */

#define ATOMIC_QUALS static inline

#endif /* OUTOFLINE */
#endif /* __ATOMIC_LIB__ */


/*
* Note on the 64 bit inline asm variants...
*
* CSTD is a conditional instruction and needs a constrained memory reference.
* Normally 'U' provides the correct constraints for conditional instructions
* and this is used for the 32 bit version, however 'U' does not appear to work
* for 64 bit values (gcc-4.9)
*
* The exact constraint is that conditional instructions cannot deal with an
* immediate displacement in the memory reference, so what we do is we read the
* address through a volatile cast into a local variable in order to insure we
* _have_ to compute the correct address without displacement. This allows us
* to use the regular 'm' for the memory address.
*
* Furthermore, the %Ln operand, which prints the low word register (r+1),
* really only works for registers, this means we cannot allow immediate values
* for the 64 bit versions -- like we do for the 32 bit ones.
*
*/

#ifndef ATOMIC_OP_RETURN
#define ATOMIC_OP_RETURN(op) \
ATOMIC_QUALS int __atomic_##op##_return(int i, int *v) \
{ \
int val; \
\
asm volatile( \
"0: \n" \
" orcc gr0,gr0,gr0,icc3 \n" \
" ckeq icc3,cc7 \n" \
" ld.p %M0,%1 \n" \
" orcr cc7,cc7,cc3 \n" \
" "#op"%I2 %1,%2,%1 \n" \
" cst.p %1,%M0 ,cc3,#1 \n" \
" corcc gr29,gr29,gr0 ,cc3,#1 \n" \
" beq icc3,#0,0b \n" \
: "+U"(*v), "=&r"(val) \
: "NPr"(i) \
: "memory", "cc7", "cc3", "icc3" \
); \
\
return val; \
} \
ATOMIC_EXPORT(__atomic_##op##_return); \
\
ATOMIC_QUALS long long __atomic64_##op##_return(long long i, long long *v) \
{ \
long long *__v = READ_ONCE(v); \
long long val; \
\
asm volatile( \
"0: \n" \
" orcc gr0,gr0,gr0,icc3 \n" \
" ckeq icc3,cc7 \n" \
" ldd.p %M0,%1 \n" \
" orcr cc7,cc7,cc3 \n" \
" "#op"cc %L1,%L2,%L1,icc0 \n" \
" "#op"x %1,%2,%1,icc0 \n" \
" cstd.p %1,%M0 ,cc3,#1 \n" \
" corcc gr29,gr29,gr0 ,cc3,#1 \n" \
" beq icc3,#0,0b \n" \
: "+m"(*__v), "=&e"(val) \
: "e"(i) \
: "memory", "cc7", "cc3", "icc0", "icc3" \
); \
\
return val; \
} \
ATOMIC_EXPORT(__atomic64_##op##_return);
#endif

#ifndef ATOMIC_FETCH_OP
#define ATOMIC_FETCH_OP(op) \
ATOMIC_QUALS int __atomic32_fetch_##op(int i, int *v) \
{ \
int old, tmp; \
\
asm volatile( \
"0: \n" \
" orcc gr0,gr0,gr0,icc3 \n" \
" ckeq icc3,cc7 \n" \
" ld.p %M0,%1 \n" \
" orcr cc7,cc7,cc3 \n" \
" "#op"%I3 %1,%3,%2 \n" \
" cst.p %2,%M0 ,cc3,#1 \n" \
" corcc gr29,gr29,gr0 ,cc3,#1 \n" \
" beq icc3,#0,0b \n" \
: "+U"(*v), "=&r"(old), "=r"(tmp) \
: "NPr"(i) \
: "memory", "cc7", "cc3", "icc3" \
); \
\
return old; \
} \
ATOMIC_EXPORT(__atomic32_fetch_##op); \
\
ATOMIC_QUALS long long __atomic64_fetch_##op(long long i, long long *v) \
{ \
long long *__v = READ_ONCE(v); \
long long old, tmp; \
\
asm volatile( \
"0: \n" \
" orcc gr0,gr0,gr0,icc3 \n" \
" ckeq icc3,cc7 \n" \
" ldd.p %M0,%1 \n" \
" orcr cc7,cc7,cc3 \n" \
" "#op" %L1,%L3,%L2 \n" \
" "#op" %1,%3,%2 \n" \
" cstd.p %2,%M0 ,cc3,#1 \n" \
" corcc gr29,gr29,gr0 ,cc3,#1 \n" \
" beq icc3,#0,0b \n" \
: "+m"(*__v), "=&e"(old), "=e"(tmp) \
: "e"(i) \
: "memory", "cc7", "cc3", "icc3" \
); \
\
return old; \
} \
ATOMIC_EXPORT(__atomic64_fetch_##op);
#endif

ATOMIC_FETCH_OP(or)
ATOMIC_FETCH_OP(and)
ATOMIC_FETCH_OP(xor)

ATOMIC_OP_RETURN(add)
ATOMIC_OP_RETURN(sub)

#undef ATOMIC_FETCH_OP
#undef ATOMIC_OP_RETURN
#undef ATOMIC_QUALS
#undef ATOMIC_EXPORT
Loading

0 comments on commit b0d8003

Please sign in to comment.