Quelle  atomic64-arcv2.h   Sprache: C

/* SPDX-License-Identifier: GPL-2.0-only */

/*
 * ARCv2 supports 64-bit exclusive load (LLOCKD) / store (SCONDD)
 *  - The address HAS to be 64-bit aligned
 */

#ifndef _ASM_ARC_ATOMIC64_ARCV2_H
#define _ASM_ARC_ATOMIC64_ARCV2_H

typedef struct {
 s64 __aligned(8) counter;
} atomic64_t;

#define ATOMIC64_INIT(a) { (a) }

static inline s64 arch_atomic64_read(const atomic64_t *v)
{
 s64 val;

 __asm__ __volatile__(
 " ldd %0, [%1] \n"
 : "=r"(val)
 : "r"(&v->counter));

 return val;
}

static inline void arch_atomic64_set(atomic64_t *v, s64 a)
{
 /*
 * This could have been a simple assignment in "C" but would need
 * explicit volatile. Otherwise gcc optimizers could elide the store
 * which borked atomic64 self-test
 * In the inline asm version, memory clobber needed for exact same
 * reason, to tell gcc about the store.
 *
 * This however is not needed for sibling atomic64_add() etc since both
 * load/store are explicitly done in inline asm. As long as API is used
 * for each access, gcc has no way to optimize away any load/store
 */
 __asm__ __volatile__(
 " std %0, [%1] \n"
 :
 : "r"(a), "r"(&v->counter)
 : "memory");
}

#define ATOMIC64_OP(op, op1, op2)     \
static inline void arch_atomic64_##op(s64 a, atomic64_t *v)  \
{         \
 s64 val;       \
         \
 __asm__ __volatile__(      \
 "1: \n"    \
 " llockd %0, [%1] \n"    \
 " " #op1 " %L0, %L0, %L2 \n"    \
 " " #op2 " %H0, %H0, %H2 \n"    \
 " scondd %0, [%1] \n"    \
 " bnz 1b \n"    \
 : "=&r"(val)       \
 : "r"(&v->counter), "ir"(a)     \
 : "cc", "memory");      \
}         \

#define ATOMIC64_OP_RETURN(op, op1, op2)           \
static inline s64 arch_atomic64_##op##_return_relaxed(s64 a, atomic64_t *v) \
{         \
 s64 val;       \
         \
 __asm__ __volatile__(      \
 "1: \n"    \
 " llockd %0, [%1] \n"    \
 " " #op1 " %L0, %L0, %L2 \n"    \
 " " #op2 " %H0, %H0, %H2 \n"    \
 " scondd %0, [%1] \n"    \
 " bnz 1b \n"    \
 : [val] "=&r"(val)      \
 : "r"(&v->counter), "ir"(a)     \
 : "cc", "memory");      \
         \
 return val;       \
}

#define arch_atomic64_add_return_relaxed arch_atomic64_add_return_relaxed
#define arch_atomic64_sub_return_relaxed arch_atomic64_sub_return_relaxed

#define ATOMIC64_FETCH_OP(op, op1, op2)            \
static inline s64 arch_atomic64_fetch_##op##_relaxed(s64 a, atomic64_t *v) \
{         \
 s64 val, orig;       \
         \
 __asm__ __volatile__(      \
 "1: \n"    \
 " llockd %0, [%2] \n"    \
 " " #op1 " %L1, %L0, %L3 \n"    \
 " " #op2 " %H1, %H0, %H3 \n"    \
 " scondd %1, [%2] \n"    \
 " bnz 1b \n"    \
 : "=&r"(orig), "=&r"(val)     \
 : "r"(&v->counter), "ir"(a)     \
 : "cc", "memory");      \
         \
 return orig;       \
}

#define arch_atomic64_fetch_add_relaxed  arch_atomic64_fetch_add_relaxed
#define arch_atomic64_fetch_sub_relaxed  arch_atomic64_fetch_sub_relaxed

#define arch_atomic64_fetch_and_relaxed  arch_atomic64_fetch_and_relaxed
#define arch_atomic64_fetch_andnot_relaxed arch_atomic64_fetch_andnot_relaxed
#define arch_atomic64_fetch_or_relaxed  arch_atomic64_fetch_or_relaxed
#define arch_atomic64_fetch_xor_relaxed  arch_atomic64_fetch_xor_relaxed

#define ATOMIC64_OPS(op, op1, op2)     \
 ATOMIC64_OP(op, op1, op2)     \
 ATOMIC64_OP_RETURN(op, op1, op2)    \
 ATOMIC64_FETCH_OP(op, op1, op2)

ATOMIC64_OPS(add, add.f, adc)
ATOMIC64_OPS(sub, sub.f, sbc)

#undef ATOMIC64_OPS
#define ATOMIC64_OPS(op, op1, op2)     \
 ATOMIC64_OP(op, op1, op2)     \
 ATOMIC64_FETCH_OP(op, op1, op2)

ATOMIC64_OPS(and, and, and)
ATOMIC64_OPS(andnot, bic, bic)
ATOMIC64_OPS(or, or, or)
ATOMIC64_OPS(xor, xor, xor)

#define arch_atomic64_andnot  arch_atomic64_andnot

#undef ATOMIC64_OPS
#undef ATOMIC64_FETCH_OP
#undef ATOMIC64_OP_RETURN
#undef ATOMIC64_OP

static inline u64 __arch_cmpxchg64_relaxed(volatile void *ptr, u64 old, u64 new)
{
 u64 prev;

 __asm__ __volatile__(
 "1: llockd %0, [%1] \n"
 " brne %L0, %L2, 2f \n"
 " brne %H0, %H2, 2f \n"
 " scondd %3, [%1] \n"
 " bnz 1b \n"
 "2: \n"
 : "=&r"(prev)
 : "r"(ptr), "ir"(old), "r"(new)
 : "memory", "cc");

 return prev;
}
#define arch_cmpxchg64_relaxed __arch_cmpxchg64_relaxed

static inline s64 arch_atomic64_xchg(atomic64_t *ptr, s64 new)
{
 s64 prev;

 smp_mb();

 __asm__ __volatile__(
 "1: llockd %0, [%1] \n"
 " scondd %2, [%1] \n"
 " bnz 1b \n"
 "2: \n"
 : "=&r"(prev)
 : "r"(ptr), "r"(new)
 : "cc"); /* memory clobber comes from smp_mb() */

 smp_mb();

 return prev;
}
#define arch_atomic64_xchg arch_atomic64_xchg

static inline s64 arch_atomic64_dec_if_positive(atomic64_t *v)
{
 s64 val;

 smp_mb();

 __asm__ __volatile__(
 "1: llockd %0, [%1] \n"
 " sub.f %L0, %L0, 1 # w0 - 1, set C on borrow\n"
 " sub.c %H0, %H0, 1 # if C set, w1 - 1\n"
 " brlt %H0, 0, 2f \n"
 " scondd %0, [%1] \n"
 " bnz 1b \n"
 "2: \n"
 : "=&r"(val)
 : "r"(&v->counter)
 : "cc"); /* memory clobber comes from smp_mb() */

 smp_mb();

 return val;
}
#define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive

static inline s64 arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
{
 s64 old, temp;

 smp_mb();

 __asm__ __volatile__(
 "1: llockd %0, [%2] \n"
 " brne %L0, %L4, 2f # continue to add since v != u \n"
 " breq.d %H0, %H4, 3f # return since v == u \n"
 "2: \n"
 " add.f %L1, %L0, %L3 \n"
 " adc %H1, %H0, %H3 \n"
 " scondd %1, [%2] \n"
 " bnz 1b \n"
 "3: \n"
 : "=&r"(old), "=&r" (temp)
 : "r"(&v->counter), "r"(a), "r"(u)
 : "cc"); /* memory clobber comes from smp_mb() */

 smp_mb();

 return old;
}
#define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless

#endif