Quelle aarch64_vector_ad.m4 Sprache: Shell

//
// Copyright (c) 2020, 2022, Oracle and/or its affiliates. All rights reserved.
// Copyright (c) 2020, 2022, Arm Limited. All rights reserved.
// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
//
// This code is free software; you can redistribute it and/or modify it
// under the terms of the GNU General Public License version 2 only, as
// published by the Free Software Foundation.
//
// This code is distributed in the hope that it will be useful, but WITHOUT
// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
// FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
// version 2 for more details (a copy is included in the LICENSE file that
// accompanied this code).
//
// You should have received a copy of the GNU General Public License version
// 2 along with this work; if not, write to the Free Software Foundation,
// Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
//
// Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
// or visit www.oracle.com if you need additional information or have any
// questions.
//
//

dnl Generate the warning
// This file is automatically generated by running "m4 aarch64_vector_ad.m4". Do not edit!
dnl

// AArch64 VECTOR Architecture Description File

dnl
dnl OPERAND_VMEMORYA_IMMEDIATE_OFFSET($1,            $2      )
dnl OPERAND_VMEMORYA_IMMEDIATE_OFFSET(imm_type_abbr, imm_type)
define(`OPERAND_VMEMORYA_IMMEDIATE_OFFSET', `
operand vmemA_imm$1Offset4() %{
  // (esize / msize) = 1
  predicate(Address::offset_ok_for_sve_immed(n->get_$2(), 4,
            Matcher::scalable_vector_reg_size(T_BYTE)));
  match(Con$1);
  op_cost(0);
  format %{ %}
  interface(CONST_INTER);
%}')dnl
dnl
dnl OPERAND_VMEMORYA_INDIRECT_OFFSET($1           )
dnl OPERAND_VMEMORYA_INDIRECT_OFFSET(imm_type_abbr)
define(`OPERAND_VMEMORYA_INDIRECT_OFFSET', `
operand vmemA_indOff$1`'4(iRegP reg, vmemA_imm$1Offset4 off) %{
  constraint(ALLOC_IN_RC(ptr_reg));
  match(AddP reg off);
  op_cost(0);
  format %{ "[$reg, $off]" %}
  interface(MEMORY_INTER) %{
    base($reg);
    `index'(0xffffffff);
    scale(0x0);
    disp($off);
  %}
%}')dnl
dnl
// 4 bit signed offset -- for predicated load/store
OPERAND_VMEMORYA_IMMEDIATE_OFFSET(I, int)
OPERAND_VMEMORYA_IMMEDIATE_OFFSET(L, long)
OPERAND_VMEMORYA_INDIRECT_OFFSET(I)
OPERAND_VMEMORYA_INDIRECT_OFFSET(L)

// The indOff of vmemA is valid only when the vector element (load to/store from)
// size equals to memory element (load from/store to) size.
opclass vmemA(indirect, vmemA_indOffI4, vmemA_indOffL4);

source_hpp %{
  // Assert that the given node is not a variable shift.
  bool assert_not_var_shift(const Node* n);

  Assembler::SIMD_Arrangement get_arrangement(const Node* n);
%}

source %{

  typedef void (C2_MacroAssembler::* sve_mem_insn_predicate)(FloatRegister Rt, Assembler::SIMD_RegVariant T,
                                                             PRegister Pg, const Address &adr);

  // Predicated load/store, with optional ptrue to all elements of given predicate register.
  static void loadStoreA_predicated(C2_MacroAssembler masm, bool is_store, FloatRegister reg,
                                    PRegister pg, BasicType mem_elem_bt, BasicType vector_elem_bt,
                                    int opcode, Register base, int index, int size, int disp) {
    sve_mem_insn_predicate insn;
    int mesize = type2aelembytes(mem_elem_bt);
    if (index == -1) {
      assert(size == 0, "unsupported address mode: scale size = %d", size);
      switch(mesize) {
      case 1:
        insn = is_store ? &C2_MacroAssembler::sve_st1b : &C2_MacroAssembler::sve_ld1b;
        break;
      case 2:
        insn = is_store ? &C2_MacroAssembler::sve_st1h : &C2_MacroAssembler::sve_ld1h;
        break;
      case 4:
        insn = is_store ? &C2_MacroAssembler::sve_st1w : &C2_MacroAssembler::sve_ld1w;
        break;
      case 8:
        insn = is_store ? &C2_MacroAssembler::sve_st1d : &C2_MacroAssembler::sve_ld1d;
        break;
      default:
        assert(false, "unsupported");
        ShouldNotReachHere();
      }
      int imm4 = disp / mesize / Matcher::scalable_vector_reg_size(vector_elem_bt);
      (masm.*insn)(reg, Assembler::elemType_to_regVariant(vector_elem_bt), pg, Address(base, imm4));
    } else {
      assert(false, "unimplemented");
      ShouldNotReachHere();
    }
  }

  const bool Matcher::match_rule_supported_superword(int opcode, int vlen, BasicType bt) {
    if (UseSVE == 0) {
      // These operations are not profitable to be vectorized on NEON, because no direct
      // NEON instructions support them. But the match rule support for them is profitable for
      // Vector API intrinsics.
      if ((opcode == Op_VectorCastD2X && bt == T_INT) ||
          (opcode == Op_VectorCastL2X && bt == T_FLOAT) ||
          (opcode == Op_CountLeadingZerosV && bt == T_LONG) ||
          (opcode == Op_CountTrailingZerosV && bt == T_LONG) ||
          opcode == Op_AddReductionVD || opcode == Op_AddReductionVF ||
          opcode == Op_MulReductionVD || opcode == Op_MulReductionVF ||
          opcode == Op_MulVL) {
        return false;
      }
    }
    return match_rule_supported_vector(opcode, vlen, bt);
  }

  // Identify extra cases that we might want to provide match rules for vector nodes and
  // other intrinsics guarded with vector length (vlen) and element type (bt).
  const bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType bt) {
    if (!match_rule_supported(opcode)) {
      return false;
    }

    int length_in_bytes = vlen * type2aelembytes(bt);
    if (UseSVE == 0 && length_in_bytes > 16) {
      return false;
    }

    // Check whether specific Op is supported.
    // Fail fast, otherwise fall through to common vector_size_supported() check.
    switch (opcode) {
      case Op_AndVMask:
      case Op_OrVMask:
      case Op_XorVMask:
      case Op_MaskAll:
      case Op_VectorMaskGen:
      case Op_LoadVectorMasked:
      case Op_StoreVectorMasked:
      case Op_LoadVectorGather:
      case Op_StoreVectorScatter:
      case Op_LoadVectorGatherMasked:
      case Op_StoreVectorScatterMasked:
      case Op_PopulateIndex:
      case Op_CompressM:
      case Op_CompressV:
        if (UseSVE == 0) {
          return false;
        }
        break;
      case Op_MulAddVS2VI:
        if (length_in_bytes != 16) {
          return false;
        }
        break;
      case Op_MulReductionVD:
      case Op_MulReductionVF:
      case Op_MulReductionVI:
      case Op_MulReductionVL:
        // No vector multiply reduction instructions, but we do
        // emit scalar instructions for 64/128-bit vectors.
        if (length_in_bytes != 8 && length_in_bytes != 16) {
          return false;
        }
        break;
      case Op_VectorMaskCmp:
        if (length_in_bytes < 8) {
          return false;
        }
        break;
      case Op_VectorLoadShuffle:
      case Op_VectorRearrange:
        if (vlen < 4) {
          return false;
        }
        break;
      case Op_ExpandV:
        if (UseSVE < 2 || is_subword_type(bt)) {
          return false;
        }
        break;
      case Op_VectorMaskToLong:
        if (UseSVE > 0 && vlen > 64) {
          return false;
        }
        break;
      case Op_VectorLongToMask:
        if (UseSVE < 2 || vlen > 64 || !VM_Version::supports_svebitperm()) {
          return false;
        }
        break;
      default:
        break;
    }
    return vector_size_supported(bt, vlen);
  }

  const bool Matcher::match_rule_supported_vector_masked(int opcode, int vlen, BasicType bt) {
    // Only SVE supports masked operations.
    if (UseSVE == 0) {
      return false;
    }

    // If an opcode does not support the masked version,
    // unpredicated node with VectorBlend node will be used instead.
    switch(opcode) {
      case Op_VectorRearrange:
      case Op_MulReductionVD:
      case Op_MulReductionVF:
      case Op_MulReductionVI:
      case Op_MulReductionVL:
        return false;
      // We use Op_LoadVectorMasked to implement the predicated Op_LoadVector.
      // Hence we turn to check whether Op_LoadVectorMasked is supported. The
      // same as vector store/gather/scatter.
      case Op_LoadVector:
        opcode = Op_LoadVectorMasked;
        break;
      case Op_StoreVector:
        opcode = Op_StoreVectorMasked;
        break;
      case Op_LoadVectorGather:
        opcode = Op_LoadVectorGatherMasked;
        break;
      case Op_StoreVectorScatter:
        opcode = Op_StoreVectorScatterMasked;
        break;
      default:
        break;
    }

    return match_rule_supported_vector(opcode, vlen, bt);
  }

  const bool Matcher::vector_needs_partial_operations(Node* node, const TypeVect* vt) {
    // Only SVE has partial vector operations
    if (UseSVE == 0) {
      return false;
    }

    switch(node->Opcode()) {
      case Op_VectorLoadMask:
      case Op_VectorMaskCmp:
      case Op_LoadVectorGather:
      case Op_StoreVectorScatter:
      case Op_AddReductionVF:
      case Op_AddReductionVD:
      case Op_AndReductionV:
      case Op_OrReductionV:
      case Op_XorReductionV:
      // Mask is needed for partial Op_VectorMaskFirstTrue, because when the
      // input predicate is all-false, the result should be the vector length
      // instead of the vector register size.
      case Op_VectorMaskFirstTrue:
        return true;
      case Op_MaskAll:
        return !node->in(1)->is_Con();
      case Op_LoadVector:
      case Op_StoreVector:
        // We use NEON load/store instructions if the vector length is <= 128 bits.
        return vt->length_in_bytes() > 16;
      case Op_AddReductionVI:
      case Op_AddReductionVL:
        // We may prefer using NEON instructions rather than SVE partial operations.
        return !VM_Version::use_neon_for_vector(vt->length_in_bytes());
      case Op_MinReductionV:
      case Op_MaxReductionV:
        // For BYTE/SHORT/INT/FLOAT/DOUBLE types, we may prefer using NEON
        // instructions rather than SVE partial operations.
        return vt->element_basic_type() == T_LONG ||
               !VM_Version::use_neon_for_vector(vt->length_in_bytes());
      default:
        // For other ops whose vector size is smaller than the max vector size, a
        // full-sized unpredicated operation does not impact the final vector result.
        return false;
    }
  }

  // Assert that the given node is not a variable shift.
  bool assert_not_var_shift(const Node* n) {
    assert(!n->as_ShiftV()->is_var_shift(), "illegal variable shift");
    return true;
  }

  Assembler::SIMD_Arrangement get_arrangement(const Node* n) {
    BasicType bt = Matcher::vector_element_basic_type(n);
    uint length_in_bytes = Matcher::vector_length_in_bytes(n);
    return Assembler::esize2arrangement((uint)type2aelembytes(bt),
                                        /* isQ */ length_in_bytes == 16);
  }
%}

// All VECTOR instructions

// ------------------------------ Vector load/store ----------------------------
dnl
dnl VECTOR_LOAD_STORE($1,   $2,     $3,       $4,    $5  )
dnl VECTOR_LOAD_STORE(type, nbytes, arg_name, nbits, size)
define(`VECTOR_LOAD_STORE', `
// ifelse(load, $1, Load, Store) Vector ($4 bits)
instruct $1V$2(vReg $3, vmem$2 mem) %{
  predicate(`n->as_'ifelse(load, $1, Load, Store)Vector()->memory_size() == $2);
  match(Set ifelse(load, $1, dst (LoadVector mem), mem (StoreVector mem src)));
  format %{ "$1V$2 ifelse(load, $1, `$dst, $mem', `$mem, $src')\t# vector ($4 bits)" %}
  ins_encode( `aarch64_enc_'ifelse(load, $1, ldr, str)v$5($3, mem) );
  ins_pipe(pipe_slow);
%}')dnl
dnl
VECTOR_LOAD_STORE(load,  2,  dst, 16,  H)
VECTOR_LOAD_STORE(store, 2,  src, 16,  H)
VECTOR_LOAD_STORE(load,  4,  dst, 32,  S)
VECTOR_LOAD_STORE(store, 4,  src, 32,  S)
VECTOR_LOAD_STORE(load,  8,  dst, 64,  D)
VECTOR_LOAD_STORE(store, 8,  src, 64,  D)
VECTOR_LOAD_STORE(load,  16, dst, 128, Q)
VECTOR_LOAD_STORE(store, 16, src, 128, Q)

// Load Vector (> 128 bits)
instruct loadV(vReg dst, vmemA mem) %{
  predicate(n->as_LoadVector()->memory_size() > 16);
  match(Set dst (LoadVector mem));
  format %{ "loadV $dst, $mem\t# vector (sve)" %}
  ins_encode %{
    assert(UseSVE > 0, "must be sve");
    BasicType bt = Matcher::vector_element_basic_type(this);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    assert(length_in_bytes == MaxVectorSize, "invalid vector length");
    loadStoreA_predicated(C2_MacroAssembler(&cbuf), /* is_store */ false,
                          $dst$$FloatRegister, ptrue, bt, bt, $mem->opcode(),
                          as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
  %}
  ins_pipe(pipe_slow);
%}

// Store Vector (> 128 bits)
instruct storeV(vReg src, vmemA mem) %{
  predicate(n->as_StoreVector()->memory_size() > 16);
  match(Set mem (StoreVector mem src));
  format %{ "storeV $mem, $src\t# vector (sve)" %}
  ins_encode %{
    assert(UseSVE > 0, "must be sve");
    BasicType bt = Matcher::vector_element_basic_type(this, $src);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src);
    assert(length_in_bytes == MaxVectorSize, "invalid vector length");
    loadStoreA_predicated(C2_MacroAssembler(&cbuf), /* is_store */ true,
                          $src$$FloatRegister, ptrue, bt, bt, $mem->opcode(),
                          as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
  %}
  ins_pipe(pipe_slow);
%}

// vector load/store - predicated

instruct loadV_masked(vReg dst, vmemA mem, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst (LoadVectorMasked mem pg));
  format %{ "loadV_masked $dst, $pg, $mem" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    loadStoreA_predicated(C2_MacroAssembler(&cbuf), /* is_store */ false, $dst$$FloatRegister,
                          $pg$$PRegister, bt, bt, $mem->opcode(),
                          as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
  %}
  ins_pipe(pipe_slow);
%}

instruct storeV_masked(vReg src, vmemA mem, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set mem (StoreVectorMasked mem (Binary src pg)));
  format %{ "storeV_masked $mem, $pg, $src" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this, $src);
    loadStoreA_predicated(C2_MacroAssembler(&cbuf), /* is_store */ true, $src$$FloatRegister,
                          $pg$$PRegister, bt, bt, $mem->opcode(),
                          as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
  %}
  ins_pipe(pipe_slow);
%}

// vector load const

instruct vloadcon(vReg dst, immI0 src) %{
  match(Set dst (VectorLoadConst src));
  format %{ "vloadcon $dst, $src\t# load/generate iota indices" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    if (UseSVE == 0) {
      uint length_in_bytes = Matcher::vector_length_in_bytes(this);
      assert(length_in_bytes <= 16, "must be");
      // The iota indices are ordered by type B/S/I/L/F/D, and the offset between two types is 16.
      int offset = exact_log2(type2aelembytes(bt)) << 4;
      if (is_floating_point_type(bt)) {
        offset += 32;
      }
      __ lea(rscratch1, ExternalAddress(StubRoutines::aarch64::vector_iota_indices() + offset));
      if (length_in_bytes == 16) {
        __ ldrq($dst$$FloatRegister, rscratch1);
      } else {
        __ ldrd($dst$$FloatRegister, rscratch1);
      }
    } else {
      Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
      __ sve_index($dst$$FloatRegister, size, 0, 1);
      if (is_floating_point_type(bt)) {
        __ sve_scvtf($dst$$FloatRegister, size, ptrue, $dst$$FloatRegister, size);
      }
    }
  %}
  ins_pipe(pipe_slow);
%}

dnl
dnl BINARY_OP($1,        $2,      $3,        $4,       $5  )
dnl BINARY_OP(rule_name, op_name, insn_neon, insn_sve, size)
define(`BINARY_OP', `
instruct $1(vReg dst, vReg src1, vReg src2) %{
  match(Set dst ($2 src1 src2));
  format %{ "$1 $dst, $src1, $src2" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ $3($dst$$FloatRegister, get_arrangement(this),
              $src1$$FloatRegister, $src2$$FloatRegister);
    } else {
      assert(UseSVE > 0, "must be sve");
      __ $4($dst$$FloatRegister, __ $5, $src1$$FloatRegister, $src2$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}')dnl
dnl
dnl BINARY_OP_PREDICATE($1,        $2,      $3,   $4  )
dnl BINARY_OP_PREDICATE(rule_name, op_name, insn, size)
define(`BINARY_OP_PREDICATE', `
instruct $1_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 ($2 (Binary dst_src1 src2) pg));
  format %{ "$1_masked $dst_src1, $pg, $dst_src1, $src2" %}
  ins_encode %{
    __ $3($dst_src1$$FloatRegister, __ $4, $pg$$PRegister, $src2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}')dnl
dnl
dnl VADD_IMM($1,   $2,       $3  )
dnl VADD_IMM(type, imm_type, size)
define(`VADD_IMM', `
instruct vaddImm$1(vReg dst_src, $2 con) %{
  predicate(UseSVE > 0);
  match(Set dst_src (AddV$1 dst_src (Replicate$1 con)));
  format %{ "vaddImm$1 $dst_src, $dst_src, $con" %}
  ins_encode %{
    int val = (int)$con$$constant;
    if (val > 0) {
      __ sve_add($dst_src$$FloatRegister, __ $3, val);
    } else {
      __ sve_sub($dst_src$$FloatRegister, __ $3, -val);
    }
  %}
  ins_pipe(pipe_slow);
%}')dnl
dnl
// ------------------------------ Vector add -----------------------------------

// vector add
BINARY_OP(vaddB, AddVB, addv, sve_add,  B)
BINARY_OP(vaddS, AddVS, addv, sve_add,  H)
BINARY_OP(vaddI, AddVI, addv, sve_add,  S)
BINARY_OP(vaddL, AddVL, addv, sve_add,  D)
BINARY_OP(vaddF, AddVF, fadd, sve_fadd, S)
BINARY_OP(vaddD, AddVD, fadd, sve_fadd, D)

// vector add - predicated
BINARY_OP_PREDICATE(vaddB, AddVB, sve_add,  B)
BINARY_OP_PREDICATE(vaddS, AddVS, sve_add,  H)
BINARY_OP_PREDICATE(vaddI, AddVI, sve_add,  S)
BINARY_OP_PREDICATE(vaddL, AddVL, sve_add,  D)
BINARY_OP_PREDICATE(vaddF, AddVF, sve_fadd, S)
BINARY_OP_PREDICATE(vaddD, AddVD, sve_fadd, D)

// vector add reg imm (unpredicated)
VADD_IMM(B, immBAddSubV, B)
VADD_IMM(S, immIAddSubV, H)
VADD_IMM(I, immIAddSubV, S)
VADD_IMM(L, immLAddSubV, D)

// ------------------------------ Vector sub -----------------------------------

// vector sub
BINARY_OP(vsubB, SubVB, subv, sve_sub,  B)
BINARY_OP(vsubS, SubVS, subv, sve_sub,  H)
BINARY_OP(vsubI, SubVI, subv, sve_sub,  S)
BINARY_OP(vsubL, SubVL, subv, sve_sub,  D)
BINARY_OP(vsubF, SubVF, fsub, sve_fsub, S)
BINARY_OP(vsubD, SubVD, fsub, sve_fsub, D)

// vector sub - predicated
BINARY_OP_PREDICATE(vsubB, SubVB, sve_sub,  B)
BINARY_OP_PREDICATE(vsubS, SubVS, sve_sub,  H)
BINARY_OP_PREDICATE(vsubI, SubVI, sve_sub,  S)
BINARY_OP_PREDICATE(vsubL, SubVL, sve_sub,  D)
BINARY_OP_PREDICATE(vsubF, SubVF, sve_fsub, S)
BINARY_OP_PREDICATE(vsubD, SubVD, sve_fsub, D)

dnl
dnl BINARY_OP_NEON_SVE_PAIRWISE($1,        $2,      $3,        $4,       $5  )
dnl BINARY_OP_NEON_SVE_PAIRWISE(rule_name, op_name, insn_neon, insn_sve, size)
define(`BINARY_OP_NEON_SVE_PAIRWISE', `
instruct $1_neon(vReg dst, vReg src1, vReg src2) %{
  predicate(VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
  match(Set dst ($2 src1 src2));
  format %{ "$1_neon $dst, $src1, $src2" %}
  ins_encode %{
    __ $3($dst$$FloatRegister, get_arrangement(this),
            $src1$$FloatRegister, $src2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct $1_sve(vReg dst_src1, vReg src2) %{
  predicate(!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
  match(Set dst_src1 ($2 dst_src1 src2));
  format %{ "$1_sve $dst_src1, $dst_src1, $src2" %}
  ins_encode %{
    assert(UseSVE > 0, "must be sve");
    __ $4($dst_src1$$FloatRegister, __ $5, ptrue, $src2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}')dnl
dnl
// ------------------------------ Vector mul -----------------------------------

// vector mul - BYTE, CHAR, SHORT, INT
BINARY_OP_NEON_SVE_PAIRWISE(vmulB, MulVB, mulv, sve_mul, B)
BINARY_OP_NEON_SVE_PAIRWISE(vmulS, MulVS, mulv, sve_mul, H)
BINARY_OP_NEON_SVE_PAIRWISE(vmulI, MulVI, mulv, sve_mul, S)

// vector mul - LONG

instruct vmulL_neon(vReg dst, vReg src1, vReg src2) %{
  predicate(UseSVE == 0);
  match(Set dst (MulVL src1 src2));
  format %{ "vmulL_neon $dst, $src1, $src2\t# 2L" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    assert(length_in_bytes == 16, "must be");
    __ umov(rscratch1, $src1$$FloatRegister, __ D, 0);
    __ umov(rscratch2, $src2$$FloatRegister, __ D, 0);
    __ mul(rscratch2, rscratch2, rscratch1);
    __ mov($dst$$FloatRegister, __ D, 0, rscratch2);
    __ umov(rscratch1, $src1$$FloatRegister, __ D, 1);
    __ umov(rscratch2, $src2$$FloatRegister, __ D, 1);
    __ mul(rscratch2, rscratch2, rscratch1);
    __ mov($dst$$FloatRegister, __ D, 1, rscratch2);
  %}
  ins_pipe(pipe_slow);
%}

instruct vmulL_sve(vReg dst_src1, vReg src2) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (MulVL dst_src1 src2));
  format %{ "vmulL_sve $dst_src1, $dst_src1, $src2" %}
  ins_encode %{
    __ sve_mul($dst_src1$$FloatRegister, __ D, ptrue, $src2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// vector mul - floating-point
BINARY_OP(vmulF, MulVF, fmul, sve_fmul, S)
BINARY_OP(vmulD, MulVD, fmul, sve_fmul, D)

// vector mul - predicated
BINARY_OP_PREDICATE(vmulB, MulVB, sve_mul,  B)
BINARY_OP_PREDICATE(vmulS, MulVS, sve_mul,  H)
BINARY_OP_PREDICATE(vmulI, MulVI, sve_mul,  S)
BINARY_OP_PREDICATE(vmulL, MulVL, sve_mul,  D)
BINARY_OP_PREDICATE(vmulF, MulVF, sve_fmul, S)
BINARY_OP_PREDICATE(vmulD, MulVD, sve_fmul, D)

// ------------------------------ Vector float div -----------------------------

// vector float div
BINARY_OP_NEON_SVE_PAIRWISE(vdivF, DivVF, fdiv, sve_fdiv, S)
BINARY_OP_NEON_SVE_PAIRWISE(vdivD, DivVD, fdiv, sve_fdiv, D)

// vector float div - predicated
BINARY_OP_PREDICATE(vdivF, DivVF, sve_fdiv, S)
BINARY_OP_PREDICATE(vdivD, DivVD, sve_fdiv, D)
dnl
dnl BITWISE_OP($1,        $2,      $3,        $4      )
dnl BITWISE_OP(rule_name, op_name, insn_neon, insn_sve)
define(`BITWISE_OP', `
instruct $1(vReg dst, vReg src1, vReg src2) %{
  match(Set dst ($2 src1 src2));
  format %{ "$1 $dst, $src1, $src2" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ $3($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
              $src1$$FloatRegister, $src2$$FloatRegister);
    } else {
      assert(UseSVE > 0, "must be sve");
      __ $4($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}')dnl
dnl
dnl BITWISE_OP_PREDICATE($1,        $2,      $3  )
dnl BITWISE_OP_PREDICATE(rule_name, op_name, insn)
define(`BITWISE_OP_PREDICATE', `
instruct $1_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 ($2 (Binary dst_src1 src2) pg));
  format %{ "$1_masked $dst_src1, $pg, $dst_src1, $src2" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ $3($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
               $pg$$PRegister, $src2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}')dnl
dnl
dnl BITWISE_OP_IMM($1,        $2,   $3,      $4,   $5  )
dnl BITWISE_OP_IMM(rule_name, type, op_name, insn, size)
define(`BITWISE_OP_IMM', `
instruct $1(vReg dst_src, imm$2Log con) %{
  predicate(UseSVE > 0);
  match(Set dst_src ($3 dst_src (Replicate$2 con)));
  format %{ "$1 $dst_src, $dst_src, $con" %}
  ins_encode %{
    __ $4($dst_src$$FloatRegister, __ $5, (uint64_t)($con$$constant));
  %}
  ins_pipe(pipe_slow);
%}')dnl
dnl

// ------------------------------ Vector and -----------------------------------

// vector and
BITWISE_OP(vand, AndV, andr, sve_and)

// vector and - predicated
BITWISE_OP_PREDICATE(vand, AndV, sve_and)

// vector and reg imm (unpredicated)
BITWISE_OP_IMM(vandImmB, B, AndV, sve_and, B)
BITWISE_OP_IMM(vandImmS, S, AndV, sve_and, H)
BITWISE_OP_IMM(vandImmI, I, AndV, sve_and, S)
BITWISE_OP_IMM(vandImmL, L, AndV, sve_and, D)

// ------------------------------ Vector or ------------------------------------

// vector or
BITWISE_OP(vor, OrV, orr, sve_orr)

// vector or - predicated
BITWISE_OP_PREDICATE(vor, OrV, sve_orr)

// vector or reg imm (unpredicated)
BITWISE_OP_IMM(vorImmB, B, OrV, sve_orr, B)
BITWISE_OP_IMM(vorImmS, S, OrV, sve_orr, H)
BITWISE_OP_IMM(vorImmI, I, OrV, sve_orr, S)
BITWISE_OP_IMM(vorImmL, L, OrV, sve_orr, D)

// ------------------------------ Vector xor -----------------------------------

// vector xor
BITWISE_OP(vxor, XorV, eor, sve_eor)

// vector xor - predicated
BITWISE_OP_PREDICATE(vxor, XorV, sve_eor)

// vector xor reg imm (unpredicated)
BITWISE_OP_IMM(vxorImmB, B, XorV, sve_eor, B)
BITWISE_OP_IMM(vxorImmS, S, XorV, sve_eor, H)
BITWISE_OP_IMM(vxorImmI, I, XorV, sve_eor, S)
BITWISE_OP_IMM(vxorImmL, L, XorV, sve_eor, D)

// vector eor3 (unpredicated)

instruct veor3_neon(vReg dst, vReg src1, vReg src2, vReg src3) %{
  predicate(VM_Version::supports_sha3() &&
            VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
  match(Set dst (XorV src1 (XorV src2 src3)));
  format %{ "veor3_neon $dst, $src1, $src2, $src3" %}
  ins_encode %{
    __ eor3($dst$$FloatRegister, __ T16B, $src1$$FloatRegister,
            $src2$$FloatRegister, $src3$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct veor3_sve(vReg dst_src1, vReg src2, vReg src3) %{
  predicate(UseSVE == 2 && !VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
  match(Set dst_src1 (XorV dst_src1 (XorV src2 src3)));
  format %{ "veor3_sve $dst_src1, $dst_src1, $src2, $src3" %}
  ins_encode %{
    __ sve_eor3($dst_src1$$FloatRegister, $src2$$FloatRegister, $src3$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ Vector not -----------------------------------

dnl
define(`MATCH_RULE', `ifelse($1, I,
`match(Set dst (XorV src (ReplicateB m1)));
  match(Set dst (XorV src (ReplicateS m1)));
  match(Set dst (XorV src (ReplicateI m1)));',
`match(Set dst (XorV src (ReplicateL m1)));')')dnl
dnl
dnl VECTOR_NOT($1  )
dnl VECTOR_NOT(type)
define(`VECTOR_NOT', `
instruct vnot$1`'(vReg dst, vReg src, imm$1_M1 m1) %{
  MATCH_RULE($1)
  format %{ "vnot$1 $dst, $src" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ notr($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
              $src$$FloatRegister);
    } else {
      assert(UseSVE > 0, "must be sve");
      __ sve_not($dst$$FloatRegister, __ D, ptrue, $src$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}')dnl
dnl
// vector not
VECTOR_NOT(I)
VECTOR_NOT(L)
undefine(MATCH_RULE)
dnl
define(`MATCH_RULE', `ifelse($1, I,
`match(Set dst_src (XorV (Binary dst_src (ReplicateB m1)) pg));
  match(Set dst_src (XorV (Binary dst_src (ReplicateS m1)) pg));
  match(Set dst_src (XorV (Binary dst_src (ReplicateI m1)) pg));',
`match(Set dst_src (XorV (Binary dst_src (ReplicateL m1)) pg));')')dnl
dnl
dnl VECTOR_NOT_PREDICATE($1  )
dnl VECTOR_NOT_PREDICATE(type)
define(`VECTOR_NOT_PREDICATE', `
instruct vnot$1_masked`'(vReg dst_src, imm$1_M1 m1, pRegGov pg) %{
  predicate(UseSVE > 0);
  MATCH_RULE($1)
  format %{ "vnot$1_masked $dst_src, $pg, $dst_src" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_not($dst_src$$FloatRegister, __ elemType_to_regVariant(bt),
               $pg$$PRegister, $dst_src$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}')dnl
dnl
// vector not - predicated
VECTOR_NOT_PREDICATE(I)
VECTOR_NOT_PREDICATE(L)
undefine(MATCH_RULE)
dnl
// ------------------------------ Vector and_not -------------------------------

dnl
define(`MATCH_RULE', `ifelse($1, I,
`match(Set dst (AndV src1 (XorV src2 (ReplicateB m1))));
  match(Set dst (AndV src1 (XorV src2 (ReplicateS m1))));
  match(Set dst (AndV src1 (XorV src2 (ReplicateI m1))));',
`match(Set dst (AndV src1 (XorV src2 (ReplicateL m1))));')')dnl
dnl
dnl VECTOR_AND_NOT($1  )
dnl VECTOR_AND_NOT(type)
define(`VECTOR_AND_NOT', `
instruct vand_not$1`'(vReg dst, vReg src1, vReg src2, imm$1_M1 m1) %{
  MATCH_RULE($1)
  format %{ "vand_not$1 $dst, $src1, $src2" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ bic($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
             $src1$$FloatRegister, $src2$$FloatRegister);
    } else {
      assert(UseSVE > 0, "must be sve");
      __ sve_bic($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}')dnl
dnl
// vector and_not
VECTOR_AND_NOT(I)
VECTOR_AND_NOT(L)
undefine(MATCH_RULE)
dnl
define(`MATCH_RULE', `ifelse($1, I,
`match(Set dst_src1 (AndV (Binary dst_src1 (XorV src2 (ReplicateB m1))) pg));
  match(Set dst_src1 (AndV (Binary dst_src1 (XorV src2 (ReplicateS m1))) pg));
  match(Set dst_src1 (AndV (Binary dst_src1 (XorV src2 (ReplicateI m1))) pg));',
`match(Set dst_src1 (AndV (Binary dst_src1 (XorV src2 (ReplicateL m1))) pg));')')dnl
dnl
dnl VECTOR_AND_NOT_PREDICATE($1  )
dnl VECTOR_AND_NOT_PREDICATE(type)
define(`VECTOR_AND_NOT_PREDICATE', `
instruct vand_not$1_masked`'(vReg dst_src1, vReg src2, imm$1_M1 m1, pRegGov pg) %{
  predicate(UseSVE > 0);
  MATCH_RULE($1)
  format %{ "vand_not$1_masked $dst_src1, $pg, $dst_src1, $src2" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_bic($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
               $pg$$PRegister, $src2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}')dnl
dnl
// vector and_not - predicated
VECTOR_AND_NOT_PREDICATE(I)
VECTOR_AND_NOT_PREDICATE(L)
undefine(MATCH_RULE)
dnl
dnl UNARY_OP($1,        $2,      $3,        $4,       $5  )
dnl UNARY_OP(rule_name, op_name, insn_neon, insn_sve, size)
define(`UNARY_OP', `
instruct $1(vReg dst, vReg src) %{
  match(Set dst ($2 src));
  format %{ "$1 $dst, $src" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ $3($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister);
    } else {
      assert(UseSVE > 0, "must be sve");
      __ $4($dst$$FloatRegister, __ $5, ptrue, $src$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}')dnl
dnl
dnl UNARY_OP_PREDICATE($1,        $2,      $3  )
dnl UNARY_OP_PREDICATE(rule_name, op_name, insn)
define(`UNARY_OP_PREDICATE', `
instruct $1_masked(vReg dst_src, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src ($2 dst_src pg));
  format %{ "$1_masked $dst_src, $pg, $dst_src" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ $3($dst_src$$FloatRegister, __ elemType_to_regVariant(bt),
               $pg$$PRegister, $dst_src$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}')dnl
dnl
dnl UNARY_OP_PREDICATE_WITH_SIZE($1,        $2,      $3,   $4  )
dnl UNARY_OP_PREDICATE_WITH_SIZE(rule_name, op_name, insn, size)
define(`UNARY_OP_PREDICATE_WITH_SIZE', `
instruct $1_masked(vReg dst_src, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src ($2 dst_src pg));
  format %{ "$1_masked $dst_src, $pg, $dst_src" %}
  ins_encode %{
    __ $3($dst_src$$FloatRegister, __ $4, $pg$$PRegister, $dst_src$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}')dnl
dnl
// ------------------------------ Vector abs -----------------------------------

// vector abs
UNARY_OP(vabsB, AbsVB, absr, sve_abs,  B)
UNARY_OP(vabsS, AbsVS, absr, sve_abs,  H)
UNARY_OP(vabsI, AbsVI, absr, sve_abs,  S)
UNARY_OP(vabsL, AbsVL, absr, sve_abs,  D)
UNARY_OP(vabsF, AbsVF, fabs, sve_fabs, S)
UNARY_OP(vabsD, AbsVD, fabs, sve_fabs, D)

// vector abs - predicated
UNARY_OP_PREDICATE_WITH_SIZE(vabsB, AbsVB, sve_abs,  B)
UNARY_OP_PREDICATE_WITH_SIZE(vabsS, AbsVS, sve_abs,  H)
UNARY_OP_PREDICATE_WITH_SIZE(vabsI, AbsVI, sve_abs,  S)
UNARY_OP_PREDICATE_WITH_SIZE(vabsL, AbsVL, sve_abs,  D)
UNARY_OP_PREDICATE_WITH_SIZE(vabsF, AbsVF, sve_fabs, S)
UNARY_OP_PREDICATE_WITH_SIZE(vabsD, AbsVD, sve_fabs, D)

// ------------------------------ Vector fabd ----------------------------------

// vector fabs diff

instruct vfabd_neon(vReg dst, vReg src1, vReg src2) %{
  predicate(VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
  match(Set dst (AbsVF (SubVF src1 src2)));
  match(Set dst (AbsVD (SubVD src1 src2)));
  format %{ "vfabd_neon $dst, $src1, $src2" %}
  ins_encode %{
    __ fabd($dst$$FloatRegister, get_arrangement(this),
            $src1$$FloatRegister, $src2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vfabd_sve(vReg dst_src1, vReg src2) %{
  predicate(!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
  match(Set dst_src1 (AbsVF (SubVF dst_src1 src2)));
  match(Set dst_src1 (AbsVD (SubVD dst_src1 src2)));
  format %{ "vfabd_sve $dst_src1, $dst_src1, $src2" %}
  ins_encode %{
    assert(UseSVE > 0, "must be sve");
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_fabd($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
                ptrue, $src2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// vector fabs diff - predicated

instruct vfabd_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (AbsVF (SubVF (Binary dst_src1 src2) pg) pg));
  match(Set dst_src1 (AbsVD (SubVD (Binary dst_src1 src2) pg) pg));
  format %{ "vfabd_masked $dst_src1, $pg, $dst_src1, $src2" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_fabd($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
                $pg$$PRegister, $src2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ Vector neg -----------------------------------

// vector neg

instruct vnegI(vReg dst, vReg src) %{
  match(Set dst (NegVI src));
  format %{ "vnegI $dst, $src" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ negr($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister);
    } else {
      assert(UseSVE > 0, "must be sve");
      BasicType bt = Matcher::vector_element_basic_type(this);
      __ sve_neg($dst$$FloatRegister, __ elemType_to_regVariant(bt),
                 ptrue, $src$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}
UNARY_OP(vnegL, NegVL, negr, sve_neg,  D)
UNARY_OP(vnegF, NegVF, fneg, sve_fneg, S)
UNARY_OP(vnegD, NegVD, fneg, sve_fneg, D)

// vector neg - predicated
UNARY_OP_PREDICATE(vnegI, NegVI, sve_neg)
UNARY_OP_PREDICATE_WITH_SIZE(vnegL, NegVL, sve_neg,  D)
UNARY_OP_PREDICATE_WITH_SIZE(vnegF, NegVF, sve_fneg, S)
UNARY_OP_PREDICATE_WITH_SIZE(vnegD, NegVD, sve_fneg, D)

// ------------------------------ Vector sqrt ----------------------------------

// vector sqrt
UNARY_OP(vsqrtF, SqrtVF, fsqrt, sve_fsqrt, S)
UNARY_OP(vsqrtD, SqrtVD, fsqrt, sve_fsqrt, D)

// vector sqrt - predicated
UNARY_OP_PREDICATE_WITH_SIZE(vsqrtF, SqrtVF, sve_fsqrt, S)
UNARY_OP_PREDICATE_WITH_SIZE(vsqrtD, SqrtVD, sve_fsqrt, D)

dnl
dnl VMINMAX_L_NEON($1,   $2     )
dnl VMINMAX_L_NEON(type, op_name)
define(`VMINMAX_L_NEON', `
instruct v$1L_neon(vReg dst, vReg src1, vReg src2) %{
  predicate(UseSVE == 0 && Matcher::vector_element_basic_type(n) == T_LONG);
  match(Set dst ($2 src1 src2));
  effect(TEMP_DEF dst);
  format %{ "v$1L_neon $dst, $src1, $src2\t# 2L" %}
  ins_encode %{
    __ cmgt($dst$$FloatRegister, __ T2D, $src1$$FloatRegister, $src2$$FloatRegister);
    __ bsl($dst$$FloatRegister, __ T16B, ifelse(min, $1, $src2, $src1)$$FloatRegister, ifelse(min, $1, $src1, $src2)$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}')dnl
dnl
dnl VMINMAX_L_SVE($1,   $2,      $3  )
dnl VMINMAX_L_SVE(type, op_name, insn)
define(`VMINMAX_L_SVE', `
instruct v$1L_sve(vReg dst_src1, vReg src2) %{
  predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_LONG);
  match(Set dst_src1 ($2 dst_src1 src2));
  format %{ "v$1L_sve $dst_src1, $dst_src1, $src2" %}
  ins_encode %{
    __ $3($dst_src1$$FloatRegister, __ D, ptrue, $src2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}')dnl
dnl
dnl VMINMAX_NEON($1,   $2,      $3,      $4           )
dnl VMINMAX_NEON(type, op_name, insn_fp, insn_integral)
define(`VMINMAX_NEON', `
instruct v$1_neon(vReg dst, vReg src1, vReg src2) %{
  predicate(Matcher::vector_element_basic_type(n) != T_LONG &&
            VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
  match(Set dst ($2 src1 src2));
  format %{ "v$1_neon $dst, $src1, $src2\t# B/S/I/F/D" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    if (is_floating_point_type(bt)) {
      __ $3($dst$$FloatRegister, get_arrangement(this),
              $src1$$FloatRegister, $src2$$FloatRegister);
    } else {
      assert(is_integral_type(bt) && bt != T_LONG, "unsupported type");
      __ $4($dst$$FloatRegister, get_arrangement(this),
              $src1$$FloatRegister, $src2$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}')dnl
dnl
dnl VMINMAX_SVE($1,   $2,      $3,      $4           )
dnl VMINMAX_SVE(type, op_name, insn_fp, insn_integral)
define(`VMINMAX_SVE', `
instruct v$1_sve(vReg dst_src1, vReg src2) %{
  predicate(Matcher::vector_element_basic_type(n) != T_LONG &&
            !VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
  match(Set dst_src1 ($2 dst_src1 src2));
  format %{ "v$1_sve $dst_src1, $dst_src1, $src2\t# B/S/I/F/D" %}
  ins_encode %{
    assert(UseSVE > 0, "must be sve");
    BasicType bt = Matcher::vector_element_basic_type(this);
    if (is_floating_point_type(bt)) {
      __ $3($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
                  ptrue, $src2$$FloatRegister);
    } else {
      assert(is_integral_type(bt) && bt != T_LONG, "unsupported type");
      __ $4($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
                  ptrue, $src2$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}')dnl
dnl
dnl VMINMAX_PREDICATE($1,   $2,      $3,      $4           )
dnl VMINMAX_PREDICATE(type, op_name, insn_fp, insn_integral)
define(`VMINMAX_PREDICATE', `
instruct v$1_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 ($2 (Binary dst_src1 src2) pg));
  format %{ "v$1_masked $dst_src1, $pg, $dst_src1, $src2" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    if (is_floating_point_type(bt)) {
      __ $3($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
                  $pg$$PRegister, $src2$$FloatRegister);
    } else {
      assert(is_integral_type(bt), "unsupported type");
      __ $4($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
                  $pg$$PRegister, $src2$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}')dnl
dnl
// ------------------------------ Vector min -----------------------------------

// vector min - LONG
VMINMAX_L_NEON(min, MinV)
VMINMAX_L_SVE(min, MinV, sve_smin)

// vector min - B/S/I/F/D
VMINMAX_NEON(min, MinV, fmin, minv)
VMINMAX_SVE(min, MinV, sve_fmin, sve_smin)

// vector min - predicated
VMINMAX_PREDICATE(min, MinV, sve_fmin, sve_smin)

// ------------------------------ Vector max -----------------------------------

// vector max - LONG
VMINMAX_L_NEON(max, MaxV)
VMINMAX_L_SVE(max, MaxV, sve_smax)

// vector max - B/S/I/F/D
VMINMAX_NEON(max, MaxV, fmax, maxv)
VMINMAX_SVE(max, MaxV, sve_fmax, sve_smax)

// vector max - predicated
VMINMAX_PREDICATE(max, MaxV, sve_fmax, sve_smax)

// ------------------------------ MLA RELATED ----------------------------------

// vector mla
// dst_src1 = dst_src1 + src2 * src3

instruct vmla(vReg dst_src1, vReg src2, vReg src3) %{
  match(Set dst_src1 (AddVB dst_src1 (MulVB src2 src3)));
  match(Set dst_src1 (AddVS dst_src1 (MulVS src2 src3)));
  match(Set dst_src1 (AddVI dst_src1 (MulVI src2 src3)));
  format %{ "vmla $dst_src1, $src2, $src3" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ mlav($dst_src1$$FloatRegister, get_arrangement(this),
              $src2$$FloatRegister, $src3$$FloatRegister);
    } else {
      assert(UseSVE > 0, "must be sve");
      __ sve_mla($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
                 ptrue, $src2$$FloatRegister, $src3$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct vmlaL(vReg dst_src1, vReg src2, vReg src3) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (AddVL dst_src1 (MulVL src2 src3)));
  format %{ "vmlaL $dst_src1, $src2, $src3" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_mla($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
               ptrue, $src2$$FloatRegister, $src3$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vmla_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (AddVB (Binary dst_src1 (MulVB src2 src3)) pg));
  match(Set dst_src1 (AddVS (Binary dst_src1 (MulVS src2 src3)) pg));
  match(Set dst_src1 (AddVI (Binary dst_src1 (MulVI src2 src3)) pg));
  match(Set dst_src1 (AddVL (Binary dst_src1 (MulVL src2 src3)) pg));
  format %{ "vmla_masked $dst_src1, $pg, $src2, $src3" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_mla($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
               $pg$$PRegister, $src2$$FloatRegister, $src3$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// vector fmla
// dst_src1 = dst_src1 + src2 * src3

instruct vfmla(vReg dst_src1, vReg src2, vReg src3) %{
  predicate(UseFMA);
  match(Set dst_src1 (FmaVF dst_src1 (Binary src2 src3)));
  match(Set dst_src1 (FmaVD dst_src1 (Binary src2 src3)));
  format %{ "vfmla $dst_src1, $src2, $src3" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ fmla($dst_src1$$FloatRegister, get_arrangement(this),
              $src2$$FloatRegister, $src3$$FloatRegister);
    } else {
      assert(UseSVE > 0, "must be sve");
      BasicType bt = Matcher::vector_element_basic_type(this);
      __ sve_fmla($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
                  ptrue, $src2$$FloatRegister, $src3$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}

// vector fmad - predicated
// dst_src1 = dst_src1 * src2 + src3

instruct vfmad_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
  predicate(UseFMA && UseSVE > 0);
  match(Set dst_src1 (FmaVF (Binary dst_src1 src2) (Binary src3 pg)));
  match(Set dst_src1 (FmaVD (Binary dst_src1 src2) (Binary src3 pg)));
  format %{ "vfmad_masked $dst_src1, $pg, $src2, $src3" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_fmad($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
                $pg$$PRegister, $src2$$FloatRegister, $src3$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// vector mls
// dst_src1 = dst_src1 - src2 * src3

instruct vmls(vReg dst_src1, vReg src2, vReg src3) %{
  match(Set dst_src1 (SubVB dst_src1 (MulVB src2 src3)));
  match(Set dst_src1 (SubVS dst_src1 (MulVS src2 src3)));
  match(Set dst_src1 (SubVI dst_src1 (MulVI src2 src3)));
  format %{ "vmls $dst_src1, $src2, $src3" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ mlsv($dst_src1$$FloatRegister, get_arrangement(this),
              $src2$$FloatRegister, $src3$$FloatRegister);
    } else {
      assert(UseSVE > 0, "must be sve");
      __ sve_mls($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
                 ptrue, $src2$$FloatRegister, $src3$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct vmlsL(vReg dst_src1, vReg src2, vReg src3) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (SubVL dst_src1 (MulVL src2 src3)));
  format %{ "vmlsL $dst_src1, $src2, $src3" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_mls($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
                 ptrue, $src2$$FloatRegister, $src3$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vmls_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (SubVB (Binary dst_src1 (MulVB src2 src3)) pg));
  match(Set dst_src1 (SubVS (Binary dst_src1 (MulVS src2 src3)) pg));
  match(Set dst_src1 (SubVI (Binary dst_src1 (MulVI src2 src3)) pg));
  match(Set dst_src1 (SubVL (Binary dst_src1 (MulVL src2 src3)) pg));
  format %{ "vmls_masked $dst_src1, $pg, $src2, $src3" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_mls($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
               $pg$$PRegister, $src2$$FloatRegister, $src3$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// vector fmls

// dst_src1 = dst_src1 + -src2 * src3
instruct vfmls1(vReg dst_src1, vReg src2, vReg src3) %{
  predicate(UseFMA);
  match(Set dst_src1 (FmaVF dst_src1 (Binary (NegVF src2) src3)));
  match(Set dst_src1 (FmaVD dst_src1 (Binary (NegVD src2) src3)));
  format %{ "vfmls1 $dst_src1, $src2, $src3" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ fmls($dst_src1$$FloatRegister, get_arrangement(this),
              $src2$$FloatRegister, $src3$$FloatRegister);
    } else {
      assert(UseSVE > 0, "must be sve");
      BasicType bt = Matcher::vector_element_basic_type(this);
      __ sve_fmls($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
                  ptrue, $src2$$FloatRegister, $src3$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}

// dst_src1 = dst_src1 + src2 * -src3
instruct vfmls2(vReg dst_src1, vReg src2, vReg src3) %{
  predicate(UseFMA);
  match(Set dst_src1 (FmaVF dst_src1 (Binary src2 (NegVF src3))));
  match(Set dst_src1 (FmaVD dst_src1 (Binary src2 (NegVD src3))));
  format %{ "vfmls2 $dst_src1, $src2, $src3" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ fmls($dst_src1$$FloatRegister, get_arrangement(this),
              $src2$$FloatRegister, $src3$$FloatRegister);
    } else {
      assert(UseSVE > 0, "must be sve");
      BasicType bt = Matcher::vector_element_basic_type(this);
      __ sve_fmls($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
                  ptrue, $src2$$FloatRegister, $src3$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}

// vector fmsb - predicated

// dst_src1 = dst_src1 * -src2 + src3
instruct vfmsb_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
  predicate(UseFMA && UseSVE > 0);
  match(Set dst_src1 (FmaVF (Binary dst_src1 (NegVF src2)) (Binary src3 pg)));
  match(Set dst_src1 (FmaVD (Binary dst_src1 (NegVD src2)) (Binary src3 pg)));
  format %{ "vfmsb_masked $dst_src1, $pg, $src2, $src3" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_fmsb($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
                $pg$$PRegister, $src2$$FloatRegister, $src3$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// vector fnmla (sve)

// dst_src1 = -dst_src1 + -src2 * src3
instruct vfnmla1(vReg dst_src1, vReg src2, vReg src3) %{
  predicate(UseFMA && UseSVE > 0);
  match(Set dst_src1 (FmaVF (NegVF dst_src1) (Binary (NegVF src2) src3)));
  match(Set dst_src1 (FmaVD (NegVD dst_src1) (Binary (NegVD src2) src3)));
  format %{ "vfnmla1 $dst_src1, $src2, $src3" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_fnmla($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
                 ptrue, $src2$$FloatRegister, $src3$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// dst_src1 = -dst_src1 + src2 * -src3
instruct vfnmla2(vReg dst_src1, vReg src2, vReg src3) %{
  predicate(UseFMA && UseSVE > 0);
  match(Set dst_src1 (FmaVF (NegVF dst_src1) (Binary src2 (NegVF src3))));
  match(Set dst_src1 (FmaVD (NegVD dst_src1) (Binary src2 (NegVD src3))));
  format %{ "vfnmla2 $dst_src1, $src2, $src3" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_fnmla($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
                 ptrue, $src2$$FloatRegister, $src3$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// vector fnmad - predicated

// dst_src1 = -src3 + dst_src1 * -src2
instruct vfnmad_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
  predicate(UseFMA && UseSVE > 0);
  match(Set dst_src1 (FmaVF (Binary dst_src1 (NegVF src2)) (Binary (NegVF src3) pg)));
  match(Set dst_src1 (FmaVD (Binary dst_src1 (NegVD src2)) (Binary (NegVD src3) pg)));
  format %{ "vfnmad_masked $dst_src1, $pg, $src2, $src3" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_fnmad($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
                 $pg$$PRegister, $src2$$FloatRegister, $src3$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// vector fnmls (sve)

// dst_src1 = -dst_src1 + src2 * src3
instruct vfnmls(vReg dst_src1, vReg src2, vReg src3) %{
  predicate(UseFMA && UseSVE > 0);
  match(Set dst_src1 (FmaVF (NegVF dst_src1) (Binary src2 src3)));
  match(Set dst_src1 (FmaVD (NegVD dst_src1) (Binary src2 src3)));
  format %{ "vfnmls $dst_src1, $src2, $src3" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_fnmls($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
                 ptrue, $src2$$FloatRegister, $src3$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// vector fnmsb - predicated

// dst_src1 = -src3 + dst_src1 * src2
instruct vfnmsb_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
  predicate(UseFMA && UseSVE > 0);
  match(Set dst_src1 (FmaVF (Binary dst_src1 src2) (Binary (NegVF src3) pg)));
  match(Set dst_src1 (FmaVD (Binary dst_src1 src2) (Binary (NegVD src3) pg)));
  format %{ "vfnmsb_masked $dst_src1, $pg, $src2, $src3" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_fnmsb($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
                 $pg$$PRegister, $src2$$FloatRegister, $src3$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// MulAddVS2VI
// Vector Multiply-Add Shorts into Integer

instruct vmuladdS2I(vReg dst, vReg src1, vReg src2, vReg tmp) %{
  predicate(Matcher::vector_length_in_bytes(n) == 16 &&
            Matcher::vector_element_basic_type(n->in(1)) == T_SHORT);
  match(Set dst (MulAddVS2VI src1 src2));
  effect(TEMP_DEF dst, TEMP tmp);
  format %{ "vmuladdS2I $dst, $src1, $src2\t# KILL $tmp" %}
  ins_encode %{
    __ smullv($tmp$$FloatRegister, __ T4H, $src1$$FloatRegister, $src2$$FloatRegister);
    __ smullv($dst$$FloatRegister, __ T8H, $src1$$FloatRegister, $src2$$FloatRegister);
    __ addpv($dst$$FloatRegister, __ T4S, $tmp$$FloatRegister, $dst$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ Vector shift ---------------------------------

// Vector right shift in AArch64 ASIMD
//
// Right shifts with vector shift count on AArch64 ASIMD are implemented
// as left shift by negative shift count.
// There are two cases for vector shift count.
//
// Case 1: The vector shift count is from replication.
//        |            |
//    LoadVector  RShiftCntV
//        |       /
//     RShiftVI
//
// Case 2: The vector shift count is from loading.
// This case isn't supported by middle-end now. But it's supported by
// panama/vectorIntrinsics(JEP 338: Vector API).
//        |            |
//    LoadVector  LoadVector
//        |       /
//     RShiftVI
//
// The negate is conducted in RShiftCntV rule for case 1, whereas it's done in
// RShiftV* rules for case 2. Because there exists an optimization opportunity
// for case 1, that is, multiple neg instructions in inner loop can be hoisted
// to outer loop and merged into one neg instruction.
//
// Note that ShiftVNode::is_var_shift() indicates whether the vector shift
// count is a variable vector(case 2) or not(a vector generated by RShiftCntV,
// i.e. case 1).

// vector shift count

instruct vshiftcntL(vReg dst, iRegIorL2I cnt) %{
  match(Set dst (LShiftCntV cnt));
  format %{ "vshiftcntL $dst, $cnt" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ dup($dst$$FloatRegister, get_arrangement(this), $cnt$$Register);
    } else {
      assert(UseSVE > 0, "must be sve");
      BasicType bt = Matcher::vector_element_basic_type(this);
      __ sve_dup($dst$$FloatRegister, __ elemType_to_regVariant(bt), $cnt$$Register);
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct vshiftcntR(vReg dst, iRegIorL2I cnt) %{
  match(Set dst (RShiftCntV cnt));
  format %{ "vshiftcntR $dst, $cnt" %}
  ins_encode %{
    if (UseSVE == 0) {
      uint length_in_bytes = Matcher::vector_length_in_bytes(this);
      assert(length_in_bytes <= 16, "must be");
      __ negw(rscratch1, $cnt$$Register);
      __ dup($dst$$FloatRegister, get_arrangement(this), rscratch1);
    } else {
      BasicType bt = Matcher::vector_element_basic_type(this);
      __ sve_dup($dst$$FloatRegister, __ elemType_to_regVariant(bt), $cnt$$Register);
    }
  %}
  ins_pipe(pipe_slow);
%}

// vector shift left

instruct vlsl_neon(vReg dst, vReg src, vReg shift) %{
  predicate(VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
  match(Set dst (LShiftVB src shift));
  match(Set dst (LShiftVS src shift));
  match(Set dst (LShiftVI src shift));
  match(Set dst (LShiftVL src shift));
  format %{ "vlsl_neon $dst, $src, $shift" %}
  ins_encode %{
    __ sshl($dst$$FloatRegister, get_arrangement(this),
            $src$$FloatRegister, $shift$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vlsl_sve(vReg dst_src, vReg shift) %{
  predicate(!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
  match(Set dst_src (LShiftVB dst_src shift));
  match(Set dst_src (LShiftVS dst_src shift));
  match(Set dst_src (LShiftVI dst_src shift));
  match(Set dst_src (LShiftVL dst_src shift));
  format %{ "vlsl_sve $dst_src, $dst_src, $shift" %}
  ins_encode %{
    assert(UseSVE > 0, "must be sve");
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_lsl($dst_src$$FloatRegister, __ elemType_to_regVariant(bt),
               ptrue, $shift$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

dnl
dnl VRSHIFT_NEON($1,   $2,      $3  )
dnl VRSHIFT_NEON(type, op_name, insn)
define(`VRSHIFT_NEON', `
instruct v$1_neon(vReg dst, vReg src, vReg shift) %{
  predicate(UseSVE == 0 && !n->as_ShiftV()->is_var_shift());
  match(Set dst ($2VB src shift));
  match(Set dst ($2VS src shift));
  match(Set dst ($2VI src shift));
  match(Set dst ($2VL src shift));
  format %{ "v$1_neon $dst, $src, $shift\t# not variable shift" %}
  ins_encode %{
    __ $3($dst$$FloatRegister, get_arrangement(this),
            $src$$FloatRegister, $shift$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}')dnl
dnl
dnl VRSHIFT_NEON_VAR($1,   $2,      $3  )
dnl VRSHIFT_NEON_VAR(type, op_name, insn)
define(`VRSHIFT_NEON_VAR', `
instruct v$1_neon_var(vReg dst, vReg src, vReg shift) %{
  predicate(UseSVE == 0 && n->as_ShiftV()->is_var_shift());
  match(Set dst ($2VB src shift));
  match(Set dst ($2VS src shift));
  match(Set dst ($2VI src shift));
  match(Set dst ($2VL src shift));
  effect(TEMP_DEF dst);
  format %{ "v$1_neon_var $dst, $src, $shift\t# variable shift" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    __ negr($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
            $shift$$FloatRegister);
    __ $3($dst$$FloatRegister, get_arrangement(this),
            $src$$FloatRegister, $dst$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}')dnl
dnl
dnl VRSHIFT_SVE($1,   $2,      $3  )
dnl VRSHIFT_SVE(type, op_name, insn)
define(`VRSHIFT_SVE', `
instruct v$1_sve(vReg dst_src, vReg shift) %{
  predicate(UseSVE > 0);
  match(Set dst_src ($2VB dst_src shift));
  match(Set dst_src ($2VS dst_src shift));
  match(Set dst_src ($2VI dst_src shift));
  match(Set dst_src ($2VL dst_src shift));
  format %{ "v$1_sve $dst_src, $dst_src, $shift" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ $3($dst_src$$FloatRegister, __ elemType_to_regVariant(bt),
               ptrue, $shift$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}')dnl
dnl
// vector shift right (arithmetic)
VRSHIFT_NEON(asr, RShift, sshl)
VRSHIFT_NEON_VAR(asr, RShift, sshl)
VRSHIFT_SVE(asr, RShift, sve_asr)

// vector shift right (logical)
VRSHIFT_NEON(lsr, URShift, ushl)
VRSHIFT_NEON_VAR(lsr, URShift, ushl)
VRSHIFT_SVE(lsr, URShift, sve_lsr)

// vector shift with imm

instruct vlsl_imm(vReg dst, vReg src, immI shift) %{
  predicate(assert_not_var_shift(n));
  match(Set dst (LShiftVB src (LShiftCntV shift)));
  match(Set dst (LShiftVS src (LShiftCntV shift)));
  match(Set dst (LShiftVI src (LShiftCntV shift)));
  match(Set dst (LShiftVL src (LShiftCntV shift)));
  format %{ "vlsl_imm $dst, $src, $shift" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    int con = (int)$shift$$constant;
    if (is_subword_type(bt)) {
      // Optimize for B/S
      int esize_in_bits = type2aelembytes(bt) * BitsPerByte;
      if (con >= esize_in_bits) {
        if (VM_Version::use_neon_for_vector(length_in_bytes)) {
          __ eor($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
                 $src$$FloatRegister, $src$$FloatRegister);
        } else {
          assert(UseSVE > 0, "must be sve");
          __ sve_eor($dst$$FloatRegister, $src$$FloatRegister, $src$$FloatRegister);
        }
        return;
      }
    }
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ shl($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister, con);
    } else {
      assert(UseSVE > 0, "must be sve");
      __ sve_lsl($dst$$FloatRegister, __ elemType_to_regVariant(bt), $src$$FloatRegister, con);
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct vasr_imm(vReg dst, vReg src, immI_positive shift) %{
  predicate(assert_not_var_shift(n));
  match(Set dst (RShiftVB src (RShiftCntV shift)));
  match(Set dst (RShiftVS src (RShiftCntV shift)));
  match(Set dst (RShiftVI src (RShiftCntV shift)));
  match(Set dst (RShiftVL src (RShiftCntV shift)));
  format %{ "vasr_imm $dst, $src, $shift" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    int con = (int)$shift$$constant;
    if (is_subword_type(bt)) {
      // Refine con for B/S
      int esize_in_bits = type2aelembytes(bt) * BitsPerByte;
      if (con >= esize_in_bits) con = esize_in_bits - 1;
    }
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ sshr($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister, con);
    } else {
      assert(UseSVE > 0, "must be sve");
      __ sve_asr($dst$$FloatRegister, __ elemType_to_regVariant(bt), $src$$FloatRegister, con);
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct vlsr_imm(vReg dst, vReg src, immI_positive shift) %{
  predicate(assert_not_var_shift(n));
  match(Set dst (URShiftVB src (RShiftCntV shift)));
  match(Set dst (URShiftVS src (RShiftCntV shift)));
  match(Set dst (URShiftVI src (RShiftCntV shift)));
  match(Set dst (URShiftVL src (RShiftCntV shift)));
  format %{ "vlsr_imm $dst, $src, $shift" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    int con = (int)$shift$$constant;
    if (is_subword_type(bt)) {
      // Optimize for B/S
      int esize_in_bits = type2aelembytes(bt) * BitsPerByte;
      if (con >= esize_in_bits) {
        if (VM_Version::use_neon_for_vector(length_in_bytes)) {
          __ eor($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
                 $src$$FloatRegister, $src$$FloatRegister);
        } else {
          assert(UseSVE > 0, "must be sve");
          __ sve_eor($dst$$FloatRegister, $src$$FloatRegister, $src$$FloatRegister);
        }
        return;
      }
    }
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ ushr($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister, con);
    } else {
      assert(UseSVE > 0, "must be sve");
      __ sve_lsr($dst$$FloatRegister, __ elemType_to_regVariant(bt), $src$$FloatRegister, con);
    }
  %}
  ins_pipe(pipe_slow);
%}

// shift right add with imm (vector length <= 128 bits only)

instruct vasra_imm(vReg dst, vReg src, immI_positive shift) %{
  predicate(Matcher::vector_length_in_bytes(n) <= 16);
  match(Set dst (AddVB dst (RShiftVB src (RShiftCntV shift))));
  match(Set dst (AddVS dst (RShiftVS src (RShiftCntV shift))));
  match(Set dst (AddVI dst (RShiftVI src (RShiftCntV shift))));
  match(Set dst (AddVL dst (RShiftVL src (RShiftCntV shift))));
  format %{ "vasra_imm $dst, $src, $shift" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    int con = (int)$shift$$constant;
    if (is_subword_type(bt)) {
      // Refine con for B/S
      int esize_in_bits = type2aelembytes(bt) * BitsPerByte;
      if (con >= esize_in_bits) con = esize_in_bits - 1;
    }
    __ ssra($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister, con);
  %}
  ins_pipe(pipe_slow);
%}

instruct vlsra_imm(vReg dst, vReg src, immI_positive shift) %{
  predicate(Matcher::vector_length_in_bytes(n) <= 16);
  match(Set dst (AddVB dst (URShiftVB src (RShiftCntV shift))));
  match(Set dst (AddVS dst (URShiftVS src (RShiftCntV shift))));
  match(Set dst (AddVI dst (URShiftVI src (RShiftCntV shift))));
  match(Set dst (AddVL dst (URShiftVL src (RShiftCntV shift))));
  format %{ "vlsra_imm $dst, $src, $shift" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    int con = (int)$shift$$constant;
    if (is_subword_type(bt)) { // for B/H
      int esize_in_bits = type2aelembytes(bt) * BitsPerByte;
      if (con < esize_in_bits) {
        __ usra($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister, con);
      }
    } else { // for S/D
      assert(type2aelembytes(bt) == 4 || type2aelembytes(bt) == 8, "unsupported type");
      __ usra($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister, con);
    }
  %}
  ins_pipe(pipe_slow);
%}

dnl
dnl VSHIFT_PREDICATE($1,   $2,      $3  )
dnl VSHIFT_PREDICATE(type, op_name, insn)
define(`VSHIFT_PREDICATE', `
instruct v$1_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 ($2VB (Binary dst_src1 src2) pg));
  match(Set dst_src1 ($2VS (Binary dst_src1 src2) pg));
  match(Set dst_src1 ($2VI (Binary dst_src1 src2) pg));
  match(Set dst_src1 ($2VL (Binary dst_src1 src2) pg));
  format %{ "v$1_masked $dst_src1, $pg, $dst_src1, $src2" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ $3($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
               $pg$$PRegister, $src2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}')dnl
dnl
dnl VSHIFT_IMM_PREDICATE($1,   $2,       $3,       $4,       $5  )
dnl VSHIFT_IMM_PREDICATE(type, arg_type, op_name1, op_name2, insn)
define(`VSHIFT_IMM_PREDICATE', `
instruct v$1_imm_masked(vReg dst_src, $2 shift, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src ($3VB (Binary dst_src ($4 shift)) pg));
  match(Set dst_src ($3VS (Binary dst_src ($4 shift)) pg));
  match(Set dst_src ($3VI (Binary dst_src ($4 shift)) pg));
  match(Set dst_src ($3VL (Binary dst_src ($4 shift)) pg));
  format %{ "v$1_imm_masked $dst_src, $pg, $dst_src, $shift" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    int esize_in_bits = type2aelembytes(bt) * BitsPerByte;
    int con = (int)$shift$$constant;
    assert(con ifelse($1, lsl, >=, >) 0 && con < esize_in_bits, "invalid shift immediate");
    __ $5($dst_src$$FloatRegister, __ elemType_to_regVariant(bt),
               $pg$$PRegister, con);
  %}
  ins_pipe(pipe_slow);
%}')dnl
dnl
// vector shift - predicated
VSHIFT_PREDICATE(lsl, LShift,  sve_lsl)
VSHIFT_PREDICATE(asr, RShift,  sve_asr)
VSHIFT_PREDICATE(lsr, URShift, sve_lsr)

// vector shift with imm - predicated
VSHIFT_IMM_PREDICATE(lsl, immI,          LShift,  LShiftCntV, sve_lsl)
VSHIFT_IMM_PREDICATE(asr, immI_positive, RShift,  RShiftCntV, sve_asr)
VSHIFT_IMM_PREDICATE(lsr, immI_positive, URShift, RShiftCntV, sve_lsr)

// ------------------------------ Vector reduction add -------------------------

dnl
dnl REDUCE_ADD_INT_NEON_SVE_PAIRWISE($1,   $2      )
dnl REDUCE_ADD_INT_NEON_SVE_PAIRWISE(type, arg_type)
define(`REDUCE_ADD_INT_NEON_SVE_PAIRWISE', `
--> --------------------

--> maximum size reached

--> --------------------

quality79%

¤ Dauer der Verarbeitung: 0.63 Sekunden (vorverarbeitet) ¤

Wurzel

Suchen

Beweissystem der NASA

Beweissystem Isabelle

NIST Cobol Testsuite

Cephes Mathematical Library

Wiener Entwicklungsmethode

Haftungshinweis

Die Informationen auf dieser Webseite wurden nach bestem Wissen sorgfältig zusammengestellt. Es wird jedoch weder Vollständigkeit, noch Richtigkeit, noch Qualität der bereit gestellten Informationen zugesichert.

Bemerkung:

Die farbliche Syntaxdarstellung ist noch experimentell.