Quelle  copy_x86.hpp   Sprache: C

/*
 * Copyright (c) 2003, 2022, Oracle and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License version 2 only, as
 * published by the Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 *
 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 * or visit www.oracle.com if you need additional information or have any
 * questions.
 *
 */

#ifndef CPU_X86_COPY_X86_HPP
#define CPU_X86_COPY_X86_HPP

#include OS_CPU_HEADER(copy)

static void pd_fill_to_words(HeapWord* tohw, size_t count, juint value) {
#ifdef AMD64
  julong* to = (julong*) tohw;
  julong  v  = ((julong) value << 32) | value;
  while (count-- > 0) {
    *to++ = v;
  }
#else
  juint* to = (juint*)tohw;
  count *= HeapWordSize / BytesPerInt;
  while (count-- > 0) {
    *to++ = value;
  }
#endif // AMD64
}

static void pd_fill_to_aligned_words(HeapWord* tohw, size_t count, juint value) {
  pd_fill_to_words(tohw, count, value);
}

static void pd_fill_to_bytes(void* to, size_t count, jubyte value) {
  (void)memset(to, value, count);
}

static void pd_zero_to_words(HeapWord* tohw, size_t count) {
  pd_fill_to_words(tohw, count, 0);
}

static void pd_zero_to_bytes(void* to, size_t count) {
  (void)memset(to, 0, count);
}

static void pd_conjoint_words(const HeapWord* from, HeapWord* to, size_t count) {
#if defined AMD64 || defined _WINDOWS
  (void)memmove(to, from, count * HeapWordSize);
#else
  // Includes a zero-count check.
  intx temp = 0;
  __asm__ volatile(" testl %6,%6 ;"
                   " jz 7f ;"
                   " cmpl %4,%5 ;"
                   " leal -4(%4,%6,4),%3;"
                   " jbe 1f ;"
                   " cmpl %7,%5 ;"
                   " jbe 4f ;"
                   "1: cmpl $32,%6 ;"
                   " ja 3f ;"
                   " subl %4,%1 ;"
                   "2: movl (%4),%3 ;"
                   " movl %7,(%5,%4,1) ;"
                   " addl $4,%0 ;"
                   " subl $1,%2 ;"
                   " jnz 2b ;"
                   " jmp 7f ;"
                   "3: rep; smovl ;"
                   " jmp 7f ;"
                   "4: cmpl $32,%2 ;"
                   " movl %7,%0 ;"
                   " leal -4(%5,%6,4),%1;"
                   " ja 6f ;"
                   " subl %4,%1 ;"
                   "5: movl (%4),%3 ;"
                   " movl %7,(%5,%4,1) ;"
                   " subl $4,%0 ;"
                   " subl $1,%2 ;"
                   " jnz 5b ;"
                   " jmp 7f ;"
                   "6: std ;"
                   " rep; smovl ;"
                   " cld ;"
                   "7: nop "
                   : "=S" (from), "=D" (to), "=c" (count), "=r" (temp)
                   : "0"  (from), "1"  (to), "2"  (count), "3"  (temp)
                   : "memory", "flags");
#endif // AMD64
}

static void pd_disjoint_words(const HeapWord* from, HeapWord* to, size_t count) {
#ifdef AMD64
  switch (count) {
  case 8:  to[7] = from[7];
  case 7:  to[6] = from[6];
  case 6:  to[5] = from[5];
  case 5:  to[4] = from[4];
  case 4:  to[3] = from[3];
  case 3:  to[2] = from[2];
  case 2:  to[1] = from[1];
  case 1:  to[0] = from[0];
  case 0:  break;
  default:
    (void)memcpy(to, from, count * HeapWordSize);
    break;
  }
#else
#if defined _WINDOWS
  (void)memcpy(to, from, count * HeapWordSize);
#else
  // Includes a zero-count check.
  intx temp = 0;
  __asm__ volatile(" testl %6,%6 ;"
                   " jz 3f ;"
                   " cmpl $32,%6 ;"
                   " ja 2f ;"
                   " subl %4,%1 ;"
                   "1: movl (%4),%3 ;"
                   " movl %7,(%5,%4,1);"
                   " addl $4,%0 ;"
                   " subl $1,%2 ;"
                   " jnz 1b ;"
                   " jmp 3f ;"
                   "2: rep; smovl ;"
                   "3: nop "
                   : "=S" (from), "=D" (to), "=c" (count), "=r" (temp)
                   : "0"  (from), "1"  (to), "2"  (count), "3"  (temp)
                   : "memory", "cc");
#endif // _WINDOWS
#endif // AMD64
}

static void pd_disjoint_words_atomic(const HeapWord* from, HeapWord* to, size_t count) {
#ifdef AMD64
  shared_disjoint_words_atomic(from, to, count);
#else
  // pd_disjoint_words is word-atomic in this implementation.
  pd_disjoint_words(from, to, count);
#endif // AMD64
}

static void pd_aligned_conjoint_words(const HeapWord* from, HeapWord* to, size_t count) {
  pd_conjoint_words(from, to, count);
}

static void pd_aligned_disjoint_words(const HeapWord* from, HeapWord* to, size_t count) {
  pd_disjoint_words(from, to, count);
}

static void pd_conjoint_bytes(const void* from, void* to, size_t count) {
#if defined AMD64 || defined _WINDOWS
  (void)memmove(to, from, count);
#else
  // Includes a zero-count check.
  intx temp = 0;
  __asm__ volatile(" testl %6,%6 ;"
                   " jz 13f ;"
                   " cmpl %4,%5 ;"
                   " leal -1(%4,%6),%3 ;"
                   " jbe 1f ;"
                   " cmpl %7,%5 ;"
                   " jbe 8f ;"
                   "1: cmpl $3,%6 ;"
                   " jbe 6f ;"
                   " movl %6,%3 ;"
                   " movl $4,%2 ;"
                   " subl %4,%2 ;"
                   " andl $3,%2 ;"
                   " jz 2f ;"
                   " subl %6,%3 ;"
                   " rep; smovb ;"
                   "2: movl %7,%2 ;"
                   " shrl $2,%2 ;"
                   " jz 5f ;"
                   " cmpl $32,%2 ;"
                   " ja 4f ;"
                   " subl %4,%1 ;"
                   "3: movl (%4),%%edx ;"
                   " movl %%edx,(%5,%4,1);"
                   " addl $4,%0 ;"
                   " subl $1,%2 ;"
                   " jnz 3b ;"
                   " addl %4,%1 ;"
                   " jmp 5f ;"
                   "4: rep; smovl ;"
                   "5: movl %7,%2 ;"
                   " andl $3,%2 ;"
                   " jz 13f ;"
                   "6: xorl %7,%3 ;"
                   "7: movb (%4,%7,1),%%dl ;"
                   " movb %%dl,(%5,%7,1) ;"
                   " addl $1,%3 ;"
                   " subl $1,%2 ;"
                   " jnz 7b ;"
                   " jmp 13f ;"
                   "8: std ;"
                   " cmpl $12,%2 ;"
                   " ja 9f ;"
                   " movl %7,%0 ;"
                   " leal -1(%6,%5),%1 ;"
                   " jmp 11f ;"
                   "9: xchgl %3,%2 ;"
                   " movl %6,%0 ;"
                   " addl $1,%2 ;"
                   " leal -1(%7,%5),%1 ;"
                   " andl $3,%2 ;"
                   " jz 10f ;"
                   " subl %6,%3 ;"
                   " rep; smovb ;"
                   "10: movl %7,%2 ;"
                   " subl $3,%0 ;"
                   " shrl $2,%2 ;"
                   " subl $3,%1 ;"
                   " rep; smovl ;"
                   " andl $3,%3 ;"
                   " jz 12f ;"
                   " movl %7,%2 ;"
                   " addl $3,%0 ;"
                   " addl $3,%1 ;"
                   "11: rep; smovb ;"
                   "12: cld ;"
                   "13: nop ;"
                   : "=S" (from), "=D" (to), "=c" (count), "=r" (temp)
                   : "0"  (from), "1"  (to), "2"  (count), "3"  (temp)
                   : "memory", "flags", "%edx");
#endif // AMD64
}

static void pd_conjoint_bytes_atomic(const void* from, void* to, size_t count) {
  pd_conjoint_bytes(from, to, count);
}

// Windows has a different implementation
#ifndef _WINDOWS
static void pd_conjoint_jshorts_atomic(const jshort* from, jshort* to, size_t count) {
  _Copy_conjoint_jshorts_atomic(from, to, count);
}

static void pd_conjoint_jints_atomic(const jint* from, jint* to, size_t count) {
#ifdef AMD64
  _Copy_conjoint_jints_atomic(from, to, count);
#else
  assert(HeapWordSize == BytesPerInt, "heapwords and jints must be the same size");
  // pd_conjoint_words is word-atomic in this implementation.
  pd_conjoint_words((const HeapWord*)from, (HeapWord*)to, count);
#endif // AMD64
}

static void pd_conjoint_jlongs_atomic(const jlong* from, jlong* to, size_t count) {
#ifdef AMD64
  _Copy_conjoint_jlongs_atomic(from, to, count);
#else
  // Guarantee use of fild/fistp or xmm regs via some asm code, because compilers won't.
  if (from > to) {
    while (count-- > 0) {
      __asm__ volatile("fildll (%0); fistpll (%1)"
                       :
                       : "r" (from), "r" (to)
                       : "memory" );
      ++from;
      ++to;
    }
  } else {
    while (count-- > 0) {
      __asm__ volatile("fildll (%0,%2,8); fistpll (%1,%2,8)"
                       :
                       : "r" (from), "r" (to), "r" (count)
                       : "memory" );
    }
  }
#endif // AMD64
}

static void pd_conjoint_oops_atomic(const oop* from, oop* to, size_t count) {
#ifdef AMD64
  assert(BytesPerLong == BytesPerOop, "jlongs and oops must be the same size");
  _Copy_conjoint_jlongs_atomic((const jlong*)from, (jlong*)to, count);
#else
  assert(HeapWordSize == BytesPerOop, "heapwords and oops must be the same size");
  // pd_conjoint_words is word-atomic in this implementation.
  pd_conjoint_words((const HeapWord*)from, (HeapWord*)to, count);
#endif // AMD64
}

static void pd_arrayof_conjoint_bytes(const HeapWord* from, HeapWord* to, size_t count) {
  _Copy_arrayof_conjoint_bytes(from, to, count);
}

static void pd_arrayof_conjoint_jshorts(const HeapWord* from, HeapWord* to, size_t count) {
  _Copy_arrayof_conjoint_jshorts(from, to, count);
}

static void pd_arrayof_conjoint_jints(const HeapWord* from, HeapWord* to, size_t count) {
#ifdef AMD64
   _Copy_arrayof_conjoint_jints(from, to, count);
#else
  pd_conjoint_jints_atomic((const jint*)from, (jint*)to, count);
#endif // AMD64
}

static void pd_arrayof_conjoint_jlongs(const HeapWord* from, HeapWord* to, size_t count) {
#ifdef AMD64
  _Copy_arrayof_conjoint_jlongs(from, to, count);
#else
  pd_conjoint_jlongs_atomic((const jlong*)from, (jlong*)to, count);
#endif // AMD64
}

static void pd_arrayof_conjoint_oops(const HeapWord* from, HeapWord* to, size_t count) {
#ifdef AMD64
  assert(BytesPerLong == BytesPerOop, "jlongs and oops must be the same size");
  _Copy_arrayof_conjoint_jlongs(from, to, count);
#else
  pd_conjoint_oops_atomic((const oop*)from, (oop*)to, count);
#endif // AMD64
}

#endif // _WINDOWS

#endif // CPU_X86_COPY_X86_HPP